From 6274e7986fe158730bd2e6550bb702127bb1dde4 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 8 Sep 2026 16:13:12 -0700 Subject: [PATCH 1/6] feat(knowledge): export knowledge bases as bundle archives Adds a streaming zip export for a knowledge base across the v2 API, the internal API, the CLI, and the UI. The bundle carries the base's configuration and tag definitions, each workspace-visible document's original file and chunk text, and optionally the chunk vectors so an import into a deployment with the same embedding model can reuse them. Access-control lists, connector links, credentials, uploader identity, and storage keys never leave. - knowledge.export operation gated by a new Knowledge Base Export permission group setting; exports record an audit event - entries are appended one at a time so a large base never fans out storage reads and the manifest is written last with the counts actually produced - sim knowledge export --output-file kb.zip [--no-vectors] - Export in the knowledge base row menu and base page header --- apps/docs/content/docs/cli/knowledge.mdx | 28 ++ apps/docs/content/docs/cli/reference.mdx | 30 ++ .../docs/knowledgebase/export-import.mdx | 63 ++++ .../docs/content/docs/knowledgebase/meta.json | 3 +- apps/docs/openapi-v2-knowledge.json | 124 ++++++++ .../api/knowledge/[id]/export/route.test.ts | 129 ++++++++ .../app/api/knowledge/[id]/export/route.ts | 35 +++ .../[knowledgeBaseId]/export/route.test.ts | 172 +++++++++++ .../[knowledgeBaseId]/export/route.ts | 40 +++ .../[workspaceId]/knowledge/[id]/base.tsx | 8 + .../knowledge-base-context-menu.tsx | 13 +- .../[workspaceId]/knowledge/knowledge.tsx | 7 + apps/sim/hooks/queries/kb/knowledge.ts | 22 +- apps/sim/lib/api/client/request.ts | 23 +- apps/sim/lib/api/contracts/knowledge/base.ts | 15 + .../v2/__tests__/cross-cutting.test.ts | 7 +- apps/sim/lib/api/contracts/v2/knowledge.ts | 22 ++ .../lib/api/contracts/v2/openapi/knowledge.ts | 36 ++- .../lib/copilot/generated/docs-manifest.ts | 1 + apps/sim/lib/knowledge/api/route-policies.ts | 1 + .../lib/knowledge/application/exports.test.ts | 197 +++++++++++++ apps/sim/lib/knowledge/application/exports.ts | 81 +++++ .../lib/knowledge/application/operations.ts | 15 + apps/sim/lib/knowledge/constants.ts | 16 + .../sim/lib/knowledge/transfer/bundle.test.ts | 250 ++++++++++++++++ apps/sim/lib/knowledge/transfer/bundle.ts | 276 ++++++++++++++++++ .../knowledge/transfer/export-archive.test.ts | 200 +++++++++++++ .../lib/knowledge/transfer/export-archive.ts | 146 +++++++++ .../lib/knowledge/transfer/export-source.ts | 190 ++++++++++++ .../permission-groups/capabilities.test.ts | 13 + .../sim/lib/permission-groups/capabilities.ts | 9 + apps/sim/lib/permission-groups/fields.test.ts | 2 + apps/sim/lib/permission-groups/fields.ts | 7 + apps/sim/lib/uploads/zip-entry-path.ts | 5 + packages/audit/src/types.ts | 1 + packages/sim-cli/README.md | 1 + .../sim-cli/src/commands/protocol/index.ts | 2 + .../protocol/knowledge-export.test.ts | 239 +++++++++++++++ .../src/commands/protocol/knowledge-export.ts | 93 ++++++ packages/sim-cli/src/generated/v2-api.ts | 33 +++ packages/sim-cli/src/program.ts | 1 + packages/testing/src/mocks/audit.mock.ts | 1 + scripts/openapi/documents.test.ts | 4 +- 43 files changed, 2545 insertions(+), 16 deletions(-) create mode 100644 apps/docs/content/docs/knowledgebase/export-import.mdx create mode 100644 apps/sim/app/api/knowledge/[id]/export/route.test.ts create mode 100644 apps/sim/app/api/knowledge/[id]/export/route.ts create mode 100644 apps/sim/app/api/v2/knowledge/[knowledgeBaseId]/export/route.test.ts create mode 100644 apps/sim/app/api/v2/knowledge/[knowledgeBaseId]/export/route.ts create mode 100644 apps/sim/lib/knowledge/application/exports.test.ts create mode 100644 apps/sim/lib/knowledge/application/exports.ts create mode 100644 apps/sim/lib/knowledge/transfer/bundle.test.ts create mode 100644 apps/sim/lib/knowledge/transfer/bundle.ts create mode 100644 apps/sim/lib/knowledge/transfer/export-archive.test.ts create mode 100644 apps/sim/lib/knowledge/transfer/export-archive.ts create mode 100644 apps/sim/lib/knowledge/transfer/export-source.ts create mode 100644 packages/sim-cli/src/commands/protocol/knowledge-export.test.ts create mode 100644 packages/sim-cli/src/commands/protocol/knowledge-export.ts diff --git a/apps/docs/content/docs/cli/knowledge.mdx b/apps/docs/content/docs/cli/knowledge.mdx index e8ff47e9717..08f919e125f 100644 --- a/apps/docs/content/docs/cli/knowledge.mdx +++ b/apps/docs/content/docs/cli/knowledge.mdx @@ -1099,6 +1099,34 @@ sim knowledge mv +## Export a knowledge base as a .simkb.zip bundle + +```bash +sim knowledge export [options] +``` + +**Arguments** + + + +| Argument | Required | Description | +| --- | --- | --- | +| `knowledgeBaseId` | Yes | Knowledge base to export | + + + +**Options** + + + +| Option | Required | Description | +| --- | --- | --- | +| `-o, --output-file ` | No | Write the bundle to this path instead of the name the server suggests; pass - to stream it to stdout. | +| `--force` | No | Overwrite --output-file if it already exists. | +| `--no-vectors` | No | Leave chunk vectors out of the bundle, so an import re-embeds every chunk. | + + + ## List knowledge resources and child folders together ```bash diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index eee728763ca..19cdae57d7f 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -2405,6 +2405,36 @@ sim knowledge mv +### sim knowledge export + +Export a knowledge base as a .simkb.zip bundle + +```bash +sim knowledge export [options] +``` + +**Arguments** + + + +| Argument | Required | Description | +| --- | --- | --- | +| `knowledgeBaseId` | Yes | Knowledge base to export | + + + +**Options** + + + +| Option | Required | Description | +| --- | --- | --- | +| `-o, --output-file ` | No | Write the bundle to this path instead of the name the server suggests; pass - to stream it to stdout. | +| `--force` | No | Overwrite --output-file if it already exists. | +| `--no-vectors` | No | Leave chunk vectors out of the bundle, so an import re-embeds every chunk. | + + + ### sim knowledge ls List knowledge resources and child folders together diff --git a/apps/docs/content/docs/knowledgebase/export-import.mdx b/apps/docs/content/docs/knowledgebase/export-import.mdx new file mode 100644 index 00000000000..ae591c3aad9 --- /dev/null +++ b/apps/docs/content/docs/knowledgebase/export-import.mdx @@ -0,0 +1,63 @@ +--- +title: Export +description: Download a knowledge base as one archive of its documents, chunks, tags, and settings. +--- + +An **export** is a single archive that holds a knowledge base: every document, the chunks Sim split it into, the tag definitions and values, and the chunking settings. Use it to keep a copy of a base outside Sim. + +To export, right-click a base in the knowledge base list and choose **Export**, or open the base and click the **Export** button in the header. The download is named `.simkb.zip`. + +## What the archive contains + +```text +.simkb.zip +├── manifest.json +├── files/ +│ └── / +└── chunks/ + └── .ndjson +``` + +- **`manifest.json`** describes the export (format version 1): the base name, description, and chunking config; the embedding model and dimension and whether vectors are included; the tag definitions; and a document list with each document's filename, MIME type, size, enabled flag, tag values, entry paths, and chunk, token, and character counts. +- **`files/`** holds each document's original file, when it has one. +- **`chunks/`** holds one NDJSON file per document, with one JSON line per chunk: its index, content, token count, start and end offsets, enabled flag, and the vector when vectors are included. + +The export includes every document a workspace member can read, its file and its chunk text. Documents synced from a [connector](/knowledgebase/connectors) export as plain documents: their text and, where Sim stored it, their file. + +## What stays behind + +An export never includes: + +- Access-control lists +- Connector links and credentials +- Who uploaded each document +- Internal storage keys + +Organization-wide search indexes cannot be exported. + +## Vectors + +Vectors are embeddings, the numbers a model produces so search can compare chunks. They are included by default. Each chunk's vector is stored as base64-encoded float32 values in its NDJSON line. + +Vectors are only valid for a deployment that uses the same embedding model and dimension. The manifest records both for the base you exported. To skip vectors, set `vectors=false` on the API or pass `--no-vectors` to the CLI. + +## Limits and governance + +A base with more than 2,000 documents cannot be exported. The request returns `413`. + +Organization admins can withhold export through the permission group setting **Knowledge Base Export** (enterprise). Every export records an audit event. + +## API + +```http +GET /api/v2/knowledge/{knowledgeBaseId}/export?workspaceId={workspaceId}&vectors=true +``` + +The response is the archive. See the [API reference](/api-reference/getting-started) for authentication. + +## CLI + +```bash +sim knowledge export --output-file kb.zip +sim knowledge export --output-file kb.zip --no-vectors +``` diff --git a/apps/docs/content/docs/knowledgebase/meta.json b/apps/docs/content/docs/knowledgebase/meta.json index d7cfa580e0a..226ffc9c4ea 100644 --- a/apps/docs/content/docs/knowledgebase/meta.json +++ b/apps/docs/content/docs/knowledgebase/meta.json @@ -5,6 +5,7 @@ "connectors", "tags", "debugging-retrieval", - "chunking-strategies" + "chunking-strategies", + "export-import" ] } diff --git a/apps/docs/openapi-v2-knowledge.json b/apps/docs/openapi-v2-knowledge.json index 244dd69c4c6..e36b58becc8 100644 --- a/apps/docs/openapi-v2-knowledge.json +++ b/apps/docs/openapi-v2-knowledge.json @@ -502,6 +502,105 @@ } } }, + "/api/v2/knowledge/{knowledgeBaseId}/export": { + "get": { + "operationId": "exportKnowledgeBase", + "summary": "Export Knowledge Base", + "description": "Stream a knowledge base as a bundle archive: its configuration, tag definitions, each workspace-visible document's file and chunk text, and optionally chunk vectors. Access-control lists, connector links, and credentials never leave. More than 2000 documents returns `413`. Exports record an audit event. `HEAD` checks access with the same authorization as `GET` but skips side effects, returning an empty `200` without payload headers on success. `HEAD` omits `Content-Length`; use file metadata to size downloads.\n\nOAuth scope: `api:read`.", + "x-sim-operation": "knowledge.export", + "x-oauth-scope": "api:read", + "tags": ["Knowledge Bases"], + "parameters": [ + { + "name": "knowledgeBaseId", + "in": "path", + "required": true, + "description": "Unique knowledge base identifier.", + "schema": { + "type": "string", + "minLength": 1, + "description": "Unique knowledge base identifier." + } + }, + { + "name": "workspaceId", + "in": "query", + "required": true, + "description": "Workspace that owns the knowledge base.", + "schema": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "description": "Workspace that owns the knowledge base." + } + }, + { + "name": "vectors", + "in": "query", + "required": false, + "description": "Include chunk vectors so an import into a deployment with the same embedding model reuses them instead of re-embedding.", + "schema": { + "description": "Include chunk vectors so an import into a deployment with the same embedding model reuses them instead of re-embedding.", + "type": "boolean" + } + } + ], + "responses": { + "200": { + "description": "The knowledge base as a zip archive.", + "headers": { + "Content-Type": { + "$ref": "#/components/headers/Content-Type" + }, + "Content-Disposition": { + "$ref": "#/components/headers/Content-Disposition" + }, + "X-RateLimit-Limit": { + "$ref": "#/components/headers/X-RateLimit-Limit" + }, + "X-RateLimit-Remaining": { + "$ref": "#/components/headers/X-RateLimit-Remaining" + }, + "X-RateLimit-Reset": { + "$ref": "#/components/headers/X-RateLimit-Reset" + } + }, + "content": { + "application/zip": { + "schema": { + "type": "string", + "format": "binary" + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "403": { + "$ref": "#/components/responses/Forbidden" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "413": { + "$ref": "#/components/responses/PayloadTooLarge" + }, + "429": { + "$ref": "#/components/responses/RateLimited" + }, + "500": { + "$ref": "#/components/responses/InternalError" + }, + "503": { + "$ref": "#/components/responses/ServiceUnavailable" + } + } + } + }, "/api/v2/knowledge/{knowledgeBaseId}/connectors": { "get": { "operationId": "listKnowledgeConnectors", @@ -4339,6 +4438,31 @@ } }, "headers": { + "Content-Type": { + "description": "MIME type of the file, defaulting to application/octet-stream when the stored type is unavailable.", + "schema": { + "type": "string", + "title": "Content type", + "description": "MIME type of the file, defaulting to application/octet-stream when the stored type is unavailable." + } + }, + "Content-Disposition": { + "description": "Attachment disposition containing sanitized and RFC 5987 encoded filenames.", + "schema": { + "type": "string", + "title": "Content disposition", + "description": "Attachment disposition containing sanitized and RFC 5987 encoded filenames." + } + }, + "Content-Length": { + "description": "File size in bytes.", + "schema": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)$", + "title": "Content length", + "description": "File size in bytes." + } + }, "X-RateLimit-Limit": { "description": "Maximum requests allowed in the current window.", "schema": { diff --git a/apps/sim/app/api/knowledge/[id]/export/route.test.ts b/apps/sim/app/api/knowledge/[id]/export/route.test.ts new file mode 100644 index 00000000000..ce2a8cf0ac4 --- /dev/null +++ b/apps/sim/app/api/knowledge/[id]/export/route.test.ts @@ -0,0 +1,129 @@ +/** + * @vitest-environment node + */ +import { Readable } from 'node:stream' +import { createMockRequest } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockGetSession, + mockExportBundle, + mockBuildKnowledgeBundleArchive, + mockKnowledgeBundleFileName, +} = vi.hoisted(() => ({ + mockGetSession: vi.fn(), + mockExportBundle: vi.fn(), + mockBuildKnowledgeBundleArchive: vi.fn(), + mockKnowledgeBundleFileName: vi.fn(), +})) + +vi.mock('@/lib/auth', () => ({ + auth: { api: { getSession: vi.fn() } }, + getSession: mockGetSession, +})) + +vi.mock('@/lib/knowledge/application/exports', () => ({ + exportKnowledgeBase: { + operation: { id: 'knowledge.export', minimumRole: 'read', workspaceApiKey: 'allow' }, + execute: mockExportBundle, + }, +})) + +vi.mock('@/lib/knowledge/transfer/export-archive', () => ({ + buildKnowledgeBundleArchive: mockBuildKnowledgeBundleArchive, + knowledgeBundleFileName: mockKnowledgeBundleFileName, +})) + +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { GET } from '@/app/api/knowledge/[id]/export/route' + +const KNOWLEDGE_BASE_ID = 'kb-1' +const FILE_NAME = 'Support docs.simkb.zip' +const context = { params: Promise.resolve({ id: KNOWLEDGE_BASE_ID }) } + +const BUNDLE = { + knowledgeBase: { name: 'Support docs', description: null, chunkingConfig: null }, + embedding: { model: 'text-embedding-3-small', dimension: 1536 }, + tags: [], + documents: [], + chunks: () => Readable.from([]), +} + +function requestFor(query = '') { + return createMockRequest( + 'GET', + undefined, + {}, + `http://localhost:3000/api/knowledge/${KNOWLEDGE_BASE_ID}/export${query ? `?${query}` : ''}` + ) +} + +describe('GET /api/knowledge/[id]/export', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetSession.mockResolvedValue({ user: { id: 'user-1' }, session: { id: 'session-1' } }) + mockExportBundle.mockResolvedValue(BUNDLE) + mockBuildKnowledgeBundleArchive.mockImplementation(() => Readable.from([Buffer.from('zip')])) + mockKnowledgeBundleFileName.mockReturnValue(FILE_NAME) + }) + + it('streams the bundle the use case returns as a zip', async () => { + const response = await GET(requestFor(), context) + + expect(response.status).toBe(200) + expect(response.headers.get('Content-Type')).toBe('application/zip') + expect(response.headers.get('Content-Disposition')).toBe(`attachment; filename="${FILE_NAME}"`) + expect(response.headers.get('Cache-Control')).toBe('no-store') + expect(await response.text()).toBe('zip') + expect(mockBuildKnowledgeBundleArchive).toHaveBeenCalledWith(BUNDLE) + expect(mockKnowledgeBundleFileName).toHaveBeenCalledWith('Support docs') + }) + + it('maps the route param and defaults vectors to true', async () => { + await GET(requestFor(), context) + + expect(mockExportBundle).toHaveBeenCalledWith( + expect.objectContaining({ + input: { knowledgeBaseId: KNOWLEDGE_BASE_ID, vectors: true }, + }) + ) + }) + + it('passes vectors=false through to the use case', async () => { + await GET(requestFor('vectors=false'), context) + + expect(mockExportBundle).toHaveBeenCalledWith( + expect.objectContaining({ input: expect.objectContaining({ vectors: false }) }) + ) + }) + + it('authenticates before dispatching the use case', async () => { + mockGetSession.mockResolvedValue(null) + + const response = await GET(requestFor(), context) + + expect(response.status).toBe(401) + expect(mockExportBundle).not.toHaveBeenCalled() + }) + + it('answers 404 for a missing knowledge base', async () => { + mockExportBundle.mockRejectedValue( + new OrchestrationError('not_found', 'Knowledge base not found') + ) + + const response = await GET(requestFor(), context) + + expect(response.status).toBe(404) + expect(await response.json()).toEqual({ error: 'Knowledge base not found' }) + }) + + it('answers 413 when the bundle exceeds the export ceiling', async () => { + mockExportBundle.mockRejectedValue( + new OrchestrationError('payload_too_large', 'Knowledge base is too large to export') + ) + + const response = await GET(requestFor(), context) + + expect(response.status).toBe(413) + }) +}) diff --git a/apps/sim/app/api/knowledge/[id]/export/route.ts b/apps/sim/app/api/knowledge/[id]/export/route.ts new file mode 100644 index 00000000000..b7c9c25e560 --- /dev/null +++ b/apps/sim/app/api/knowledge/[id]/export/route.ts @@ -0,0 +1,35 @@ +import { exportKnowledgeBaseContract } from '@/lib/api/contracts/knowledge' +import { + defineInternalBinaryRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { nodeReadableToWebStream } from '@/lib/core/utils/node-stream' +import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' +import { exportKnowledgeBase } from '@/lib/knowledge/application/exports' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { + buildKnowledgeBundleArchive, + knowledgeBundleFileName, +} from '@/lib/knowledge/transfer/export-archive' +import { encodeFilenameForHeader } from '@/app/api/files/utils' + +/** GET /api/knowledge/[id]/export — stream a knowledge base as a bundle archive. */ +export const GET = defineInternalBinaryRoute({ + contract: exportKnowledgeBaseContract, + auth: internalSessionAuth, + operation: knowledgeOperations.export, + rateLimit: internalRateLimits.none({ reason: 'Internal knowledge base bundle download' }), + errorPolicy: internalKnowledgeErrorPolicies.export, + mapInput: ({ params, query }) => ({ + knowledgeBaseId: params.id, + vectors: query.vectors, + }), + useCase: exportKnowledgeBase, + present: (bundle) => ({ + body: nodeReadableToWebStream(buildKnowledgeBundleArchive(bundle)), + contentType: 'application/zip', + contentDisposition: `attachment; ${encodeFilenameForHeader(knowledgeBundleFileName(bundle.knowledgeBase.name))}`, + headers: { 'Cache-Control': 'no-store' }, + }), +}) diff --git a/apps/sim/app/api/v2/knowledge/[knowledgeBaseId]/export/route.test.ts b/apps/sim/app/api/v2/knowledge/[knowledgeBaseId]/export/route.test.ts new file mode 100644 index 00000000000..1f610672904 --- /dev/null +++ b/apps/sim/app/api/v2/knowledge/[knowledgeBaseId]/export/route.test.ts @@ -0,0 +1,172 @@ +/** + * @vitest-environment node + */ +import { + MockV2ApiKeyUnauthenticatedError, + V2_OPERATION_RATE_LIMIT_ALLOWED, + V2_PREAUTH_RATE_LIMIT_ALLOWED, + v2ApiKeyAuthModuleMock, + v2RateLimiterModuleMock, + v2RouteMocks, +} from '@sim/testing' +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + exportBundle: vi.fn(), + authorizeExport: vi.fn(), + buildKnowledgeBundleArchive: vi.fn(), + knowledgeBundleFileName: vi.fn(), +})) + +vi.mock('@/lib/knowledge/application/exports', () => ({ + exportKnowledgeBase: { + operation: { id: 'knowledge.export', minimumRole: 'read', workspaceApiKey: 'allow' }, + execute: mocks.exportBundle, + authorize: mocks.authorizeExport, + }, +})) + +vi.mock('@/lib/knowledge/transfer/export-archive', () => ({ + buildKnowledgeBundleArchive: mocks.buildKnowledgeBundleArchive, + knowledgeBundleFileName: mocks.knowledgeBundleFileName, +})) + +vi.mock('@/lib/api/server/routes/v2-api-key-auth', () => v2ApiKeyAuthModuleMock) +vi.mock('@/lib/core/rate-limiter', () => v2RateLimiterModuleMock) + +import { Readable } from 'node:stream' +import { NoWorkspaceAccessError } from '@/lib/core/application' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { GET } from '@/app/api/v2/knowledge/[knowledgeBaseId]/export/route' + +const WORKSPACE_ID = '6fc7631d-88cd-46f8-9f0a-d4764daef7f8' +const KNOWLEDGE_BASE_ID = 'e0d2c0c8-3b4c-4a9f-9a3e-2f1d5c7b8a90' +const FILE_NAME = 'Support docs.simkb.zip' +const context = { params: Promise.resolve({ knowledgeBaseId: KNOWLEDGE_BASE_ID }) } + +const AUTH = { + principal: { kind: 'workspace_api_key' as const, workspaceId: WORKSPACE_ID, keyId: 'key-1' }, + rateLimitSubjectIds: ['api-key:key-1', `workspace:${WORKSPACE_ID}`] as const, + rateLimitSubscription: null, + keyType: 'workspace' as const, +} + +const BUNDLE = { + knowledgeBase: { name: 'Support docs', description: null, chunkingConfig: null }, + embedding: { model: 'text-embedding-3-small', dimension: 1536 }, + tags: [], + documents: [], + chunks: () => Readable.from([]), +} + +function exportRequest(query = `workspaceId=${WORKSPACE_ID}`) { + return new NextRequest( + `http://localhost:3000/api/v2/knowledge/${KNOWLEDGE_BASE_ID}/export?${query}`, + { headers: { 'x-api-key': 'secret' } } + ) +} + +describe('GET /api/v2/knowledge/[knowledgeBaseId]/export', () => { + beforeEach(() => { + vi.clearAllMocks() + v2RouteMocks.authenticate.mockResolvedValue(AUTH) + v2RouteMocks.preauthRate.mockResolvedValue(V2_PREAUTH_RATE_LIMIT_ALLOWED) + v2RouteMocks.operationRate.mockResolvedValue(V2_OPERATION_RATE_LIMIT_ALLOWED) + mocks.authorizeExport.mockResolvedValue(undefined) + mocks.exportBundle.mockResolvedValue(BUNDLE) + mocks.buildKnowledgeBundleArchive.mockImplementation(() => Readable.from([Buffer.from('zip')])) + mocks.knowledgeBundleFileName.mockReturnValue(FILE_NAME) + }) + + it('streams the knowledge base as a zip bundle', async () => { + const response = await GET(exportRequest(), context) + + expect(response.status).toBe(200) + expect(response.headers.get('Content-Type')).toBe('application/zip') + expect(response.headers.get('Content-Disposition')).toBe(`attachment; filename="${FILE_NAME}"`) + expect(response.headers.get('Cache-Control')).toContain('no-store') + expect(await response.text()).toBe('zip') + expect(mocks.buildKnowledgeBundleArchive).toHaveBeenCalledWith(BUNDLE) + expect(mocks.knowledgeBundleFileName).toHaveBeenCalledWith('Support docs') + }) + + it('maps the knowledge base id and asserted workspace into the use case input', async () => { + await GET(exportRequest(), context) + + expect(mocks.exportBundle).toHaveBeenCalledWith( + expect.objectContaining({ + principal: AUTH.principal, + input: { + knowledgeBaseId: KNOWLEDGE_BASE_ID, + assertedWorkspaceId: WORKSPACE_ID, + vectors: true, + }, + }) + ) + }) + + it('defaults the vectors flag to true when omitted', async () => { + await GET(exportRequest(), context) + + expect(mocks.exportBundle).toHaveBeenCalledWith( + expect.objectContaining({ input: expect.objectContaining({ vectors: true }) }) + ) + }) + + it('passes vectors=false through to the use case', async () => { + await GET(exportRequest(`workspaceId=${WORKSPACE_ID}&vectors=false`), context) + + expect(mocks.exportBundle).toHaveBeenCalledWith( + expect.objectContaining({ input: expect.objectContaining({ vectors: false }) }) + ) + }) + + it('rejects a request without a workspaceId', async () => { + const response = await GET(exportRequest(''), context) + + expect(response.status).toBe(400) + expect(mocks.exportBundle).not.toHaveBeenCalled() + }) + + it('rejects an unauthenticated request', async () => { + v2RouteMocks.authenticate.mockRejectedValueOnce(new MockV2ApiKeyUnauthenticatedError()) + + const response = await GET(exportRequest(), context) + + expect(response.status).toBe(401) + expect(mocks.exportBundle).not.toHaveBeenCalled() + }) + + it('answers 404 for a missing knowledge base', async () => { + mocks.exportBundle.mockRejectedValueOnce( + new OrchestrationError('not_found', 'Knowledge base not found') + ) + + const response = await GET(exportRequest(), context) + + expect(response.status).toBe(404) + expect((await response.json()).error.code).toBe('NOT_FOUND') + }) + + it('answers 413 when the bundle exceeds the export ceiling', async () => { + mocks.exportBundle.mockRejectedValueOnce( + new OrchestrationError('payload_too_large', 'Knowledge base is too large to export') + ) + + const response = await GET(exportRequest(), context) + + expect(response.status).toBe(413) + expect((await response.json()).error.code).toBe('PAYLOAD_TOO_LARGE') + }) + + /** Cross-tenant denials are concealed as an absent knowledge base. */ + it('conceals a cross-tenant workspace as 404', async () => { + mocks.exportBundle.mockRejectedValueOnce(new NoWorkspaceAccessError()) + + const response = await GET(exportRequest(), context) + + expect(response.status).toBe(404) + expect((await response.json()).error.message).toBe('Knowledge base not found') + }) +}) diff --git a/apps/sim/app/api/v2/knowledge/[knowledgeBaseId]/export/route.ts b/apps/sim/app/api/v2/knowledge/[knowledgeBaseId]/export/route.ts new file mode 100644 index 00000000000..4d8b8a2cc8f --- /dev/null +++ b/apps/sim/app/api/v2/knowledge/[knowledgeBaseId]/export/route.ts @@ -0,0 +1,40 @@ +import { v2ExportKnowledgeBaseContract } from '@/lib/api/contracts/v2/knowledge' +import { defineV2BinaryRoute, v2ApiKeyAuth, v2RateLimits } from '@/lib/api/server/routes' +import { nodeReadableToWebStream } from '@/lib/core/utils/node-stream' +import { v2KnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' +import { exportKnowledgeBase } from '@/lib/knowledge/application/exports' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { + buildKnowledgeBundleArchive, + knowledgeBundleFileName, +} from '@/lib/knowledge/transfer/export-archive' +import { encodeFilenameForHeader } from '@/app/api/files/utils' + +export const dynamic = 'force-dynamic' +export const revalidate = 0 + +/** + * GET /api/v2/knowledge/[knowledgeBaseId]/export — stream a knowledge base as a bundle archive. + * + * `headSafe: false` because the export records a `KNOWLEDGE_BASE_EXPORTED` + * audit event and pulls bytes out of object storage. + */ +export const GET = defineV2BinaryRoute({ + contract: v2ExportKnowledgeBaseContract, + auth: v2ApiKeyAuth, + headSafe: false, + operation: knowledgeOperations.export, + rateLimit: v2RateLimits.publicApi, + errorPolicy: v2KnowledgeErrorPolicies.concealKnowledgeBaseAuthorization, + mapInput: ({ params, query }) => ({ + knowledgeBaseId: params.knowledgeBaseId, + assertedWorkspaceId: query.workspaceId, + vectors: query.vectors, + }), + useCase: exportKnowledgeBase, + present: (bundle) => ({ + body: nodeReadableToWebStream(buildKnowledgeBundleArchive(bundle)), + contentType: 'application/zip', + contentDisposition: `attachment; ${encodeFilenameForHeader(knowledgeBundleFileName(bundle.knowledgeBase.name))}`, + }), +}) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx index 5224b2d1d19..9cd9a351589 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx @@ -26,6 +26,7 @@ import { CircleAlert, Database, DatabaseX, + Download, Loader, Pencil, Plus, @@ -118,6 +119,7 @@ import type { ConnectorData } from '@/hooks/queries/kb/connectors' import { isConnectorSyncingOrPending, useConnectorList } from '@/hooks/queries/kb/connectors' import type { DocumentTagFilter } from '@/hooks/queries/kb/knowledge' import { + downloadKnowledgeBaseExport, useBulkDocumentOperation, useDeleteDocument, useDeleteKnowledgeBase, @@ -1009,6 +1011,11 @@ export function KnowledgeBase({ const headerActions: ResourceAction[] = useMemo( () => [ + { + text: 'Export', + icon: Download, + onSelect: () => downloadKnowledgeBaseExport(id), + }, ...(userPermissions.canEdit || userPermissions.isLoading ? [ { @@ -1028,6 +1035,7 @@ export function KnowledgeBase({ }, ], [ + id, userPermissions.canEdit, userPermissions.isLoading, setShowAddConnectorModal, diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/components/knowledge-base-context-menu/knowledge-base-context-menu.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/components/knowledge-base-context-menu/knowledge-base-context-menu.tsx index 83ce9100c4d..dfbf4e37995 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/components/knowledge-base-context-menu/knowledge-base-context-menu.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/components/knowledge-base-context-menu/knowledge-base-context-menu.tsx @@ -12,6 +12,7 @@ import { DropdownMenuTrigger, } from '@sim/emcn' import { + Download, Duplicate, FolderInput, Pencil, @@ -31,6 +32,7 @@ interface KnowledgeBaseContextMenuProps { onOpenInNewTab?: () => void onViewTags?: () => void onCopyId?: () => void + onExport?: () => void onTogglePin?: () => void /** Pin state of the right-clicked base, driving the Pin/Unpin label. */ pinned?: boolean @@ -50,7 +52,7 @@ interface KnowledgeBaseContextMenuProps { /** * Context menu component for knowledge base cards. - * Displays open in new tab, view tags, edit, and delete options. + * Displays open in new tab, view tags, export, edit, and delete options. */ export const KnowledgeBaseContextMenu = memo(function KnowledgeBaseContextMenu({ isOpen, @@ -59,6 +61,7 @@ export const KnowledgeBaseContextMenu = memo(function KnowledgeBaseContextMenu({ onOpenInNewTab, onViewTags, onCopyId, + onExport, onTogglePin, pinned = false, onEdit, @@ -76,7 +79,7 @@ export const KnowledgeBaseContextMenu = memo(function KnowledgeBaseContextMenu({ const isMultiSelect = selectedCount > 1 const hasNavigationSection = !isMultiSelect && showOpenInNewTab && !!onOpenInNewTab const hasInfoSection = - !isMultiSelect && ((showViewTags && !!onViewTags) || !!onCopyId || !!onTogglePin) + !isMultiSelect && ((showViewTags && !!onViewTags) || !!onCopyId || !!onExport || !!onTogglePin) const hasMoveSection = !disableEdit && !!onMove && !!moveOptions && moveOptions.length > 0 const hasEditSection = (!isMultiSelect && showEdit && !!onEdit) || hasMoveSection const hasDestructiveSection = showDelete && !!onDelete @@ -122,6 +125,12 @@ export const KnowledgeBaseContextMenu = memo(function KnowledgeBaseContextMenu({ Copy ID )} + {!isMultiSelect && onExport && ( + + + Export + + )} {!isMultiSelect && onTogglePin && ( diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx index 6825988de30..6d5c1f8e390 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx @@ -87,6 +87,7 @@ import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' import { useKnowledgeBasesList } from '@/hooks/kb/use-knowledge' import { useCreateFolder, useDeleteFolderMutation, useUpdateFolder } from '@/hooks/queries/folders' import { + downloadKnowledgeBaseExport, useBulkDeleteKnowledgeBases, useBulkMoveKnowledgeBases, useDeleteKnowledgeBase, @@ -877,6 +878,11 @@ export function Knowledge() { } }, []) + const handleExport = useCallback(() => { + const kb = activeKnowledgeBaseRef.current + if (kb) downloadKnowledgeBaseExport(kb.id) + }, []) + const handleEdit = useCallback(() => { setIsEditModalOpen(true) }, []) @@ -1539,6 +1545,7 @@ export function Knowledge() { onOpenInNewTab={handleOpenInNewTab} onViewTags={handleViewTags} onCopyId={handleCopyId} + onExport={handleExport} onTogglePin={handleToggleBasePin} pinned={pinnedBaseIds.has(activeKnowledgeBase.id)} onEdit={handleEdit} diff --git a/apps/sim/hooks/queries/kb/knowledge.ts b/apps/sim/hooks/queries/kb/knowledge.ts index a53086b6dec..750436ff322 100644 --- a/apps/sim/hooks/queries/kb/knowledge.ts +++ b/apps/sim/hooks/queries/kb/knowledge.ts @@ -2,7 +2,7 @@ import { toast } from '@sim/emcn' import { createLogger } from '@sim/logger' import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import { ApiClientError } from '@/lib/api/client/errors' -import { requestJson } from '@/lib/api/client/request' +import { contractUrl, requestJson } from '@/lib/api/client/request' import { type BulkChunkOperationData, type BulkDeleteKnowledgeItemsBody, @@ -25,6 +25,7 @@ import { deleteKnowledgeChunkContract, deleteKnowledgeDocumentContract, deleteTagDefinitionContract, + exportKnowledgeBaseContract, getKnowledgeBaseContract, getKnowledgeDocumentContract, getTagUsageContract, @@ -741,6 +742,25 @@ async function deleteKnowledgeBase({ knowledgeBaseId }: DeleteKnowledgeBaseParam }) } +/** + * Starts a browser download of a knowledge base's bundle archive. + * + * An anchor navigation rather than a fetch: the archive is streamed and can run + * to gigabytes, and the browser saving it straight to disk is what keeps it out + * of page memory. The session cookie authenticates the same-origin request. + */ +export function downloadKnowledgeBaseExport(knowledgeBaseId: string): void { + const anchor = document.createElement('a') + anchor.href = contractUrl(exportKnowledgeBaseContract, { + params: { id: knowledgeBaseId }, + query: {}, + }) + anchor.download = '' + document.body.appendChild(anchor) + anchor.click() + document.body.removeChild(anchor) +} + export function useDeleteKnowledgeBase() { const queryClient = useQueryClient() diff --git a/apps/sim/lib/api/client/request.ts b/apps/sim/lib/api/client/request.ts index ba03bb45068..cef9207cb16 100644 --- a/apps/sim/lib/api/client/request.ts +++ b/apps/sim/lib/api/client/request.ts @@ -192,12 +192,10 @@ export async function requestJson( throw new Error(`Contract ${contract.method} ${contract.path} does not declare a JSON response`) } - const parsedParams = parseOptionalSchema(contract.params, input.params) - const parsedQuery = parseOptionalSchema(contract.query, input.query) const parsedBody = parseOptionalSchema(contract.body, input.body) const parsedHeaders = parseOptionalSchema(contract.headers, input.headers) - const url = appendQuery(replacePathParams(contract.path, parsedParams), parsedQuery) + const url = contractUrl(contract, input) const hasBody = parsedBody !== undefined && contract.method !== 'GET' const response = await fetch(url, { @@ -236,17 +234,30 @@ export async function requestJson( } } +/** + * The URL a contract resolves to for `input`, validated the same way a request + * would be. For a download the browser should stream itself — an anchor + * navigation rather than a fetch that buffers the body — so the caller needs + * the address, not the response. + */ +export function contractUrl( + contract: C, + input: ApiClientRequest +): string { + const parsedParams = parseOptionalSchema(contract.params, input.params) + const parsedQuery = parseOptionalSchema(contract.query, input.query) + return appendQuery(replacePathParams(contract.path, parsedParams), parsedQuery) +} + export async function requestRaw( contract: C, input: ApiClientRequest, options: ApiRawRequestOptions = {} ): Promise { - const parsedParams = parseOptionalSchema(contract.params, input.params) - const parsedQuery = parseOptionalSchema(contract.query, input.query) const parsedBody = parseOptionalSchema(contract.body, input.body) const parsedHeaders = parseOptionalSchema(contract.headers, input.headers) - const url = appendQuery(replacePathParams(contract.path, parsedParams), parsedQuery) + const url = contractUrl(contract, input) const hasBody = parsedBody !== undefined && contract.method !== 'GET' const headers = { ...buildHeaders(parsedHeaders, hasBody), diff --git a/apps/sim/lib/api/contracts/knowledge/base.ts b/apps/sim/lib/api/contracts/knowledge/base.ts index dc84effbdeb..fd09ef0f463 100644 --- a/apps/sim/lib/api/contracts/knowledge/base.ts +++ b/apps/sim/lib/api/contracts/knowledge/base.ts @@ -6,6 +6,7 @@ import { wireDateSchema, } from '@/lib/api/contracts/knowledge/shared' import { + booleanQueryFlagSchema, folderIdSchema, requiredFieldSchema, workspaceIdSchema, @@ -238,6 +239,20 @@ export const getKnowledgeBaseContract = defineRouteContract({ }, }) +export const exportKnowledgeBaseQuerySchema = z.object({ + vectors: booleanQueryFlagSchema.optional().default(true), +}) + +export type ExportKnowledgeBaseQuery = z.input + +export const exportKnowledgeBaseContract = defineRouteContract({ + method: 'GET', + path: '/api/knowledge/[id]/export', + params: knowledgeBaseParamsSchema, + query: exportKnowledgeBaseQuerySchema, + response: { mode: 'binary' }, +}) + export const updateKnowledgeBaseContract = defineRouteContract({ method: 'PUT', path: '/api/knowledge/[id]', diff --git a/apps/sim/lib/api/contracts/v2/__tests__/cross-cutting.test.ts b/apps/sim/lib/api/contracts/v2/__tests__/cross-cutting.test.ts index 25149ed980f..4c432cca258 100644 --- a/apps/sim/lib/api/contracts/v2/__tests__/cross-cutting.test.ts +++ b/apps/sim/lib/api/contracts/v2/__tests__/cross-cutting.test.ts @@ -180,11 +180,12 @@ describe('knowledge and files request-slice strictness', () => { * read, archive extraction, file-text read, folder restore, bulk zip * download, and permanent delete. It falls when two routes become one: 106 → * 105 when the archived knowledge-base list folded into `GET /knowledge` as - * `scope=archived`, and 105 → 108 with the file content-search query and the - * in-place content edit's query and body. + * `scope=archived`, 105 → 108 with the file content-search query and the + * in-place content edit's query and body, and 108 → 109 with the knowledge + * base export query. */ it('sweeps every documented query and body slice', () => { - expect(slices.length).toBe(108) + expect(slices.length).toBe(109) }) it.each(slices)('%s rejects an undeclared key', (_name, schema) => { diff --git a/apps/sim/lib/api/contracts/v2/knowledge.ts b/apps/sim/lib/api/contracts/v2/knowledge.ts index 227306f7055..1c539902810 100644 --- a/apps/sim/lib/api/contracts/v2/knowledge.ts +++ b/apps/sim/lib/api/contracts/v2/knowledge.ts @@ -772,6 +772,28 @@ export const v2GetKnowledgeBaseContract = defineRouteContract({ }, }) +export const v2ExportKnowledgeBaseQuerySchema = z + .object({ + workspaceId: workspaceIdSchema.describe('Workspace that owns the knowledge base.'), + vectors: booleanQueryFlagSchema + .optional() + .default(true) + .describe( + 'Include chunk vectors so an import into a deployment with the same embedding model reuses them instead of re-embedding.' + ), + }) + .strict() + +export type V2ExportKnowledgeBaseQuery = z.input + +export const v2ExportKnowledgeBaseContract = defineRouteContract({ + method: 'GET', + path: '/api/v2/knowledge/[knowledgeBaseId]/export', + params: v2KnowledgeBaseParamsSchema, + query: v2ExportKnowledgeBaseQuerySchema, + response: { mode: 'binary' }, +}) + /** * PATCH, not PUT: every mutable field is optional and a `superRefine` requires * at least one, so this is a partial update rather than a replacement. diff --git a/apps/sim/lib/api/contracts/v2/openapi/knowledge.ts b/apps/sim/lib/api/contracts/v2/openapi/knowledge.ts index 9831dd9cd4e..5bf3b052979 100644 --- a/apps/sim/lib/api/contracts/v2/openapi/knowledge.ts +++ b/apps/sim/lib/api/contracts/v2/openapi/knowledge.ts @@ -12,6 +12,7 @@ import { v2DeleteKnowledgeConnectorContract, v2DeleteKnowledgeDocumentContract, v2DeleteKnowledgeFolderContract, + v2ExportKnowledgeBaseContract, v2GetKnowledgeBaseContract, v2GetKnowledgeConnectorContract, v2GetKnowledgeDocumentContract, @@ -39,11 +40,14 @@ import { type ErrorResponseId, FOLDER_TREE_TOO_LARGE, FULL_SET_LIST, + HEAD_MIRRORS_GET, + HEAD_OMITS_PAYLOAD_HEADERS, RATE_LIMIT_HEADERS, RESOURCE_CONFLICT_ERRORS, RESOURCE_ERRORS, V2_AUTH_SECURITY, V2_AUTH_SECURITY_SCHEMES, + V2_BINARY_DOWNLOAD_HEADERS, V2_COMMON_HEADERS, V2_ERROR_SCHEMA, WORKSPACE_API_KEY_DENIED, @@ -58,6 +62,7 @@ import { type OpenApiSuccessMetadata, } from '@/lib/api/openapi/types' import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { MAX_KNOWLEDGE_BUNDLE_DOCUMENTS } from '@/lib/knowledge/constants' const WORKSPACE_ID = 'a91c4b2e-6d3f-4e8a-b5c7-0d9e2f1a8c64' const KNOWLEDGE_BASE_ID = '7c9e6679-7425-40de-944b-e07fc1f90ae7' @@ -198,6 +203,35 @@ const declaredRoutes = [ ), } ), + defineOpenApiRoute( + v2ExportKnowledgeBaseContract, + knowledgeOperation({ + applicationOperation: knowledgeOperations.export, + operationId: 'exportKnowledgeBase', + summary: 'Export Knowledge Base', + description: `Stream a knowledge base as a bundle archive: its configuration, tag definitions, each workspace-visible document's file and chunk text, and optionally chunk vectors. Access-control lists, connector links, and credentials never leave. More than ${MAX_KNOWLEDGE_BUNDLE_DOCUMENTS} documents returns \`413\`. Exports record an audit event. ${HEAD_MIRRORS_GET} ${HEAD_OMITS_PAYLOAD_HEADERS}`, + errors: [...RESOURCE_ERRORS, 'PayloadTooLarge'], + success: { + description: 'The knowledge base as a zip archive.', + headers: ['Content-Type', 'Content-Disposition'], + contentTypes: ['application/zip'], + }, + }), + { + params: documentedSchema( + v2ExportKnowledgeBaseContract.params, + 'ExportKnowledgeBaseParams', + 'Export knowledge base path parameters', + 'Knowledge base selected for export.' + ), + query: documentedSchema( + v2ExportKnowledgeBaseContract.query, + 'ExportKnowledgeBaseQuery', + 'Export knowledge base query', + 'Workspace scope and whether chunk vectors are included.' + ), + } + ), defineOpenApiRoute( v2UpdateKnowledgeBaseContract, knowledgeOperation({ @@ -1241,7 +1275,7 @@ export const knowledgeOpenApiDocument = defineOpenApiDocument({ ], security: V2_AUTH_SECURITY, securitySchemes: V2_AUTH_SECURITY_SCHEMES, - headers: V2_COMMON_HEADERS, + headers: { ...V2_BINARY_DOWNLOAD_HEADERS, ...V2_COMMON_HEADERS }, errorSchema: V2_ERROR_SCHEMA, errorResponses: withErrorExamples({ Conflict: { message: 'Upload has already been completed' }, diff --git a/apps/sim/lib/copilot/generated/docs-manifest.ts b/apps/sim/lib/copilot/generated/docs-manifest.ts index 6421f1a32e2..0d8f03d9ec8 100644 --- a/apps/sim/lib/copilot/generated/docs-manifest.ts +++ b/apps/sim/lib/copilot/generated/docs-manifest.ts @@ -360,6 +360,7 @@ export const DOCS_MANIFEST: readonly string[] = [ 'knowledgebase/chunking-strategies.mdx', 'knowledgebase/connectors.mdx', 'knowledgebase/debugging-retrieval.mdx', + 'knowledgebase/export-import.mdx', 'knowledgebase/tags.mdx', 'knowledgebase/using-in-workflows.mdx', 'logs-debugging.mdx', diff --git a/apps/sim/lib/knowledge/api/route-policies.ts b/apps/sim/lib/knowledge/api/route-policies.ts index 07bce348043..a0f47466ceb 100644 --- a/apps/sim/lib/knowledge/api/route-policies.ts +++ b/apps/sim/lib/knowledge/api/route-policies.ts @@ -73,6 +73,7 @@ function concealKnowledgeBase(base: InternalErrorPolicy): InternalErrorPolicy { export const internalKnowledgeErrorPolicies = { list: internalKnowledgeErrorPolicy('Failed to fetch knowledge bases'), read: concealKnowledgeBase(internalKnowledgeErrorPolicy('Failed to fetch knowledge base')), + export: concealKnowledgeBase(internalKnowledgeErrorPolicy('Failed to export knowledge base')), create: internalKnowledgeErrorPolicy('Failed to create knowledge base'), update: concealKnowledgeBase(internalKnowledgeErrorPolicy('Failed to update knowledge base')), delete: concealKnowledgeBase(internalKnowledgeErrorPolicy('Failed to delete knowledge base')), diff --git a/apps/sim/lib/knowledge/application/exports.test.ts b/apps/sim/lib/knowledge/application/exports.test.ts new file mode 100644 index 00000000000..85a153ebc68 --- /dev/null +++ b/apps/sim/lib/knowledge/application/exports.test.ts @@ -0,0 +1,197 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + resolveKnowledgeBase: vi.fn(), + resolvePermission: vi.fn(), + resolveAccess: vi.fn(), + listTags: vi.fn(), + listDocuments: vi.fn(), + iterateChunks: vi.fn(), + recordAudit: vi.fn(), +})) + +vi.mock('@sim/audit', () => ({ + AuditAction: { KNOWLEDGE_BASE_EXPORTED: 'knowledge_base.exported' }, + AuditResourceType: { KNOWLEDGE_BASE: 'knowledge_base' }, + recordAudit: mocks.recordAudit, +})) + +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: (actual: string | null, required: string) => { + const rank = { read: 1, write: 2, admin: 3 } as const + return ( + actual !== null && rank[actual as keyof typeof rank] >= rank[required as keyof typeof rank] + ) + }, + resolveEffectiveWorkspacePermission: mocks.resolvePermission, +})) + +vi.mock('@/lib/knowledge/access/scope', () => ({ + resolveKnowledgeAccessScope: mocks.resolveAccess, +})) + +vi.mock('@/lib/knowledge/application/contexts', () => ({ + resolveActiveKnowledgeBaseContext: mocks.resolveKnowledgeBase, +})) + +vi.mock('@/lib/knowledge/transfer/export-source', () => ({ + listExportableTags: mocks.listTags, + listExportableDocuments: mocks.listDocuments, + iterateDocumentChunks: mocks.iterateChunks, +})) + +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { exportKnowledgeBase } from '@/lib/knowledge/application/exports' + +const context = { + workspaceId: 'workspace-1', + workspaceOrganizationId: 'organization-1', + allowPersonalApiKeys: true, + billedAccountUserId: 'billing-owner-1', +} + +const knowledgeBase = { + id: 'knowledge-1', + userId: 'billing-owner-1', + name: 'Support docs', + description: 'Everything support knows', + tokenCount: 0, + embeddingModel: 'text-embedding-3-small', + embeddingDimension: 1536, + chunkingConfig: { maxSize: 1024, minSize: 100, overlap: 200 }, + createdAt: new Date('2026-01-01T00:00:00Z'), + updatedAt: new Date('2026-01-01T00:00:00Z'), + deletedAt: null, + workspaceId: 'workspace-1', + folderId: null, + docCount: 2, + connectorTypes: [], + hasPermissionScopedConnector: false, +} + +const principal = { kind: 'session', userId: 'user-1', sessionId: 'session-1' } as const + +const documents = [ + { + id: 'doc-1', + filename: 'handbook.pdf', + mimeType: 'application/pdf', + fileSize: 3, + enabled: true, + tokenCount: 12, + characterCount: 40, + tags: {}, + file: { kind: 'storage', key: 'kb/handbook.pdf' }, + hasChunks: true, + }, +] + +describe('exportKnowledgeBase', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.resolveAccess.mockResolvedValue({ kind: 'workspace', tokens: ['ws', 'pub'] }) + mocks.resolveKnowledgeBase.mockResolvedValue({ + ...context, + knowledgeBaseId: knowledgeBase.id, + knowledgeBase, + access: { get: mocks.resolveAccess }, + }) + mocks.resolvePermission.mockResolvedValue('read') + mocks.listTags.mockResolvedValue([{ slot: 'tag1', displayName: 'Product', fieldType: 'text' }]) + mocks.listDocuments.mockResolvedValue(documents) + mocks.iterateChunks.mockReturnValue((async function* () {})()) + }) + + it('lets a read-role principal export and describes the bundle from the stored base', async () => { + const result = await exportKnowledgeBase.execute({ + principal, + input: { knowledgeBaseId: 'knowledge-1', assertedWorkspaceId: 'workspace-1', vectors: true }, + }) + + expect(mocks.resolveKnowledgeBase).toHaveBeenCalledWith( + { knowledgeBaseId: 'knowledge-1', assertedWorkspaceId: 'workspace-1', vectors: true }, + principal + ) + expect(result.knowledgeBase).toEqual({ + name: 'Support docs', + description: 'Everything support knows', + chunkingConfig: { maxSize: 1024, minSize: 100, overlap: 200 }, + }) + expect(result.embedding).toEqual({ + model: 'text-embedding-3-small', + dimension: 1536, + vectorsIncluded: true, + }) + expect(result.tags).toEqual([{ slot: 'tag1', displayName: 'Product', fieldType: 'text' }]) + expect(result.documents).toBe(documents) + }) + + it('reads the vector column only when vectors are requested', async () => { + const withVectors = await exportKnowledgeBase.execute({ + principal, + input: { knowledgeBaseId: 'knowledge-1', vectors: true }, + }) + withVectors.chunks('doc-1') + expect(mocks.iterateChunks).toHaveBeenLastCalledWith('doc-1', 1536) + + const textOnly = await exportKnowledgeBase.execute({ + principal, + input: { knowledgeBaseId: 'knowledge-1', vectors: false }, + }) + textOnly.chunks('doc-1') + expect(mocks.iterateChunks).toHaveBeenLastCalledWith('doc-1', null) + expect(textOnly.embedding.vectorsIncluded).toBe(false) + }) + + it('records the export as an audit event', async () => { + await exportKnowledgeBase.execute({ + principal, + input: { knowledgeBaseId: 'knowledge-1', vectors: false }, + }) + + expect(mocks.recordAudit).toHaveBeenCalledTimes(1) + expect(mocks.recordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + action: 'knowledge_base.exported', + resourceType: 'knowledge_base', + resourceId: 'knowledge-1', + resourceName: 'Support docs', + metadata: expect.objectContaining({ + workspaceId: 'workspace-1', + vectors: false, + documentCount: 1, + }), + }) + ) + }) + + it('conceals a base the principal cannot read', async () => { + mocks.resolvePermission.mockResolvedValue(null) + + await expect( + exportKnowledgeBase.execute({ + principal, + input: { knowledgeBaseId: 'knowledge-1', vectors: true }, + }) + ).rejects.toMatchObject({ name: 'NoWorkspaceAccessError' }) + expect(mocks.listDocuments).not.toHaveBeenCalled() + expect(mocks.recordAudit).not.toHaveBeenCalled() + }) + + it('propagates an oversized base without recording audit', async () => { + mocks.listDocuments.mockRejectedValueOnce( + new OrchestrationError('payload_too_large', 'Knowledge base has 2001 documents') + ) + + await expect( + exportKnowledgeBase.execute({ + principal, + input: { knowledgeBaseId: 'knowledge-1', vectors: true }, + }) + ).rejects.toMatchObject({ code: 'payload_too_large' }) + expect(mocks.recordAudit).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/application/exports.ts b/apps/sim/lib/knowledge/application/exports.ts new file mode 100644 index 00000000000..57a4bcfee20 --- /dev/null +++ b/apps/sim/lib/knowledge/application/exports.ts @@ -0,0 +1,81 @@ +import { AuditAction, AuditResourceType } from '@sim/audit' +import type { Principal } from '@sim/auth/principal' +import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' +import { resolveActiveKnowledgeBaseContext } from '@/lib/knowledge/application/contexts' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { toKbEmbeddingDimensions } from '@/lib/knowledge/embedding-models' +import type { KnowledgeBundleManifest, KnowledgeBundleTag } from '@/lib/knowledge/transfer/bundle' +import { + type ExportableChunk, + type ExportableDocument, + iterateDocumentChunks, + listExportableDocuments, + listExportableTags, +} from '@/lib/knowledge/transfer/export-source' + +export interface ExportKnowledgeBaseInput { + knowledgeBaseId: string + assertedWorkspaceId?: string + /** Carry chunk vectors so a same-model import can reuse them instead of re-embedding. */ + vectors: boolean +} + +/** + * Everything an export archive is built from. Document metadata is loaded + * eagerly because it is small and the manifest needs all of it; chunk content + * is handed over as a generator so the archive can stream it document by + * document. + */ +export interface KnowledgeBaseExportBundle { + knowledgeBase: KnowledgeBundleManifest['knowledgeBase'] + embedding: KnowledgeBundleManifest['embedding'] + tags: KnowledgeBundleTag[] + documents: ExportableDocument[] + chunks(documentId: string): AsyncIterable +} + +export const exportKnowledgeBase = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.export, + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: ExportKnowledgeBaseInput + }) => resolveActiveKnowledgeBaseContext(input, principal), + async execute({ input, context }): Promise { + const { knowledgeBase } = context + const dimension = toKbEmbeddingDimensions(knowledgeBase.embeddingDimension) + const [tags, documents] = await Promise.all([ + listExportableTags(knowledgeBase.id), + listExportableDocuments(knowledgeBase.id), + ]) + return { + knowledgeBase: { + name: knowledgeBase.name, + description: knowledgeBase.description, + chunkingConfig: knowledgeBase.chunkingConfig, + }, + embedding: { + model: knowledgeBase.embeddingModel, + dimension, + vectorsIncluded: input.vectors, + }, + tags, + documents, + chunks: (documentId) => iterateDocumentChunks(documentId, input.vectors ? dimension : null), + } + }, + projectAudit: ({ context, input, result }) => ({ + action: AuditAction.KNOWLEDGE_BASE_EXPORTED, + resourceType: AuditResourceType.KNOWLEDGE_BASE, + resourceId: context.knowledgeBase.id, + resourceName: context.knowledgeBase.name, + description: `Exported knowledge base "${context.knowledgeBase.name}"`, + metadata: { + workspaceId: context.workspaceId, + vectors: input.vectors, + documentCount: result.documents.length, + }, + }), +}) diff --git a/apps/sim/lib/knowledge/application/operations.ts b/apps/sim/lib/knowledge/application/operations.ts index 627c6c4b5b3..6887d5767dd 100644 --- a/apps/sim/lib/knowledge/application/operations.ts +++ b/apps/sim/lib/knowledge/application/operations.ts @@ -129,6 +129,21 @@ export const knowledgeOperations = { ...ALL_PRINCIPAL_POLICY, }) ), + /** + * Streams a whole knowledge base out as one archive. A read-role principal + * may export because nothing leaves that the reader could not already page + * through, but the bulk shape is what `knowledge.export` lets a group withhold. + */ + export: defineKnowledgeOperation( + defineWorkspaceOperation({ + id: 'knowledge.export', + oauthScope: 'api:read', + minimumRole: 'read', + workspaceApiKey: 'allow', + capability: 'knowledge.export', + principalKinds: HTTP_PRINCIPAL_KINDS, + }) + ), /** * The only operation that brings a knowledge base into existence, so it is the * only one `knowledge.create` governs — a group may be allowed to query, diff --git a/apps/sim/lib/knowledge/constants.ts b/apps/sim/lib/knowledge/constants.ts index 80c2f8b2da1..8d1933e4c1f 100644 --- a/apps/sim/lib/knowledge/constants.ts +++ b/apps/sim/lib/knowledge/constants.ts @@ -188,3 +188,19 @@ export function getPlaceholderForFieldType(fieldType: string): string { * same 45-minute floor prevents the default UI from racing a legitimate run. */ export const KNOWLEDGE_DOCUMENT_PROCESSING_STALE_THRESHOLD_MS = 45 * 60 * 1000 + +/** Bundle layout version written to `manifest.json`; an importer refuses any other. */ +export const KNOWLEDGE_BUNDLE_VERSION = 1 +/** Documents one export bundle may carry, so every produced bundle stays importable. */ +export const MAX_KNOWLEDGE_BUNDLE_DOCUMENTS = 2_000 +/** Upper bound for one uploaded bundle archive. */ +export const MAX_KNOWLEDGE_BUNDLE_BYTES = 2 * 1024 ** 3 +/** Upper bound for the manifest entry of one bundle. */ +export const MAX_KNOWLEDGE_BUNDLE_MANIFEST_BYTES = 8 * 1024 ** 2 +/** + * Characters one exported chunk may hold. Wider than the manual-chunk API cap + * because the processor's largest chunking config emits chunks past 10k. + */ +export const MAX_KNOWLEDGE_BUNDLE_CHUNK_CONTENT_LENGTH = 100_000 +/** Bytes one chunk line may span: the content above plus a base64 3072-wide vector. */ +export const MAX_KNOWLEDGE_BUNDLE_CHUNK_LINE_BYTES = 512 * 1024 diff --git a/apps/sim/lib/knowledge/transfer/bundle.test.ts b/apps/sim/lib/knowledge/transfer/bundle.test.ts new file mode 100644 index 00000000000..b1292b763d2 --- /dev/null +++ b/apps/sim/lib/knowledge/transfer/bundle.test.ts @@ -0,0 +1,250 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { MAX_KNOWLEDGE_BUNDLE_DOCUMENTS } from '@/lib/knowledge/constants' +import { + chunksEntryPath, + decodeVectorBase64, + type ExportableDocumentRecord, + encodeVectorBase64, + fileEntryPath, + KnowledgeBundleVectorError, + knowledgeBundleChunkLineSchema, + knowledgeBundleManifestSchema, + safeBundleLeafName, + toManifestDocument, +} from '@/lib/knowledge/transfer/bundle' + +const DOCUMENT_ID = 'a2f1c3d4-1111-4222-8333-444455556666' + +function manifestDocument(overrides: Record = {}) { + return { + id: DOCUMENT_ID, + filename: 'handbook.pdf', + mimeType: 'application/pdf', + fileSize: 1234, + enabled: true, + tags: { tag1: 'Billing' }, + file: `files/${DOCUMENT_ID}/handbook.pdf`, + chunks: `chunks/${DOCUMENT_ID}.ndjson`, + chunkCount: 3, + tokenCount: 900, + characterCount: 4000, + ...overrides, + } +} + +function manifest(overrides: Record = {}) { + return { + version: 1, + exportedAt: '2026-09-08T12:00:00.000Z', + embedding: { model: 'text-embedding-3-small', dimension: 1536, vectorsIncluded: true }, + knowledgeBase: { + name: 'Support docs', + description: null, + chunkingConfig: { maxSize: 1024, minSize: 100, overlap: 200 }, + }, + tags: [{ slot: 'tag1', displayName: 'Product', fieldType: 'text' }], + documents: [manifestDocument()], + ...overrides, + } +} + +describe('knowledgeBundleManifestSchema', () => { + it('accepts a well-formed manifest', () => { + expect(knowledgeBundleManifestSchema.safeParse(manifest()).success).toBe(true) + }) + + it('refuses any other layout version', () => { + expect(knowledgeBundleManifestSchema.safeParse(manifest({ version: 2 })).success).toBe(false) + }) + + /** Strictness is what keeps a tampered or future field from silently riding along. */ + it('refuses unknown fields at every level', () => { + expect(knowledgeBundleManifestSchema.safeParse({ ...manifest(), acl: ['ws'] }).success).toBe( + false + ) + expect( + knowledgeBundleManifestSchema.safeParse( + manifest({ documents: [manifestDocument({ connectorId: 'kc-1' })] }) + ).success + ).toBe(false) + expect( + knowledgeBundleManifestSchema.safeParse( + manifest({ documents: [manifestDocument({ uploadedBy: 'user-1' })] }) + ).success + ).toBe(false) + }) + + it('refuses a dimension no storage column holds', () => { + expect( + knowledgeBundleManifestSchema.safeParse( + manifest({ embedding: { model: 'x', dimension: 1000, vectorsIncluded: false } }) + ).success + ).toBe(false) + }) + + it('refuses a tag slot that does not belong to its field type', () => { + expect( + knowledgeBundleManifestSchema.safeParse( + manifest({ tags: [{ slot: 'number1', displayName: 'Product', fieldType: 'text' }] }) + ).success + ).toBe(false) + }) + + it('refuses duplicate tag slots and case-insensitively duplicate tag names', () => { + expect( + knowledgeBundleManifestSchema.safeParse( + manifest({ + tags: [ + { slot: 'tag1', displayName: 'Product', fieldType: 'text' }, + { slot: 'tag1', displayName: 'Region', fieldType: 'text' }, + ], + }) + ).success + ).toBe(false) + expect( + knowledgeBundleManifestSchema.safeParse( + manifest({ + tags: [ + { slot: 'tag1', displayName: 'Product', fieldType: 'text' }, + { slot: 'tag2', displayName: 'product', fieldType: 'text' }, + ], + }) + ).success + ).toBe(false) + }) + + it('refuses a document with neither a file nor chunks', () => { + expect( + knowledgeBundleManifestSchema.safeParse( + manifest({ documents: [manifestDocument({ file: null, chunks: null })] }) + ).success + ).toBe(false) + }) + + it('refuses entry paths that do not belong to the document', () => { + expect( + knowledgeBundleManifestSchema.safeParse( + manifest({ documents: [manifestDocument({ file: 'files/other/handbook.pdf' })] }) + ).success + ).toBe(false) + expect( + knowledgeBundleManifestSchema.safeParse( + manifest({ documents: [manifestDocument({ chunks: 'chunks/other.ndjson' })] }) + ).success + ).toBe(false) + }) + + it('caps the document list', () => { + const documents = Array.from({ length: MAX_KNOWLEDGE_BUNDLE_DOCUMENTS + 1 }, (_, index) => + manifestDocument({ + id: `doc-${index}`, + file: `files/doc-${index}/handbook.pdf`, + chunks: `chunks/doc-${index}.ndjson`, + }) + ) + expect(knowledgeBundleManifestSchema.safeParse(manifest({ documents })).success).toBe(false) + }) +}) + +describe('knowledgeBundleChunkLineSchema', () => { + it('accepts a line with and without a vector', () => { + const line = { + index: 0, + content: 'Refunds take five days.', + tokenCount: 6, + startOffset: 0, + endOffset: 23, + enabled: true, + } + expect(knowledgeBundleChunkLineSchema.safeParse(line).success).toBe(true) + expect( + knowledgeBundleChunkLineSchema.safeParse({ ...line, vector: encodeVectorBase64([0.5, 1]) }) + .success + ).toBe(true) + expect(knowledgeBundleChunkLineSchema.safeParse({ ...line, vector: '***' }).success).toBe(false) + }) +}) + +describe('toManifestDocument', () => { + const record: ExportableDocumentRecord = { + id: DOCUMENT_ID, + filename: 'handbook.pdf', + mimeType: 'application/pdf', + fileSize: 1234, + enabled: false, + tokenCount: 900, + characterCount: 4000, + tags: { + tag1: 'Billing', + number1: 42, + date1: new Date('2026-01-02T00:00:00.000Z'), + boolean1: false, + tag2: null, + }, + } + + it('projects exactly the manifest fields and renders tag values as strings', () => { + const entries = { + file: fileEntryPath(record.id, record.filename), + chunks: chunksEntryPath(record.id), + } + const projected = toManifestDocument(record, entries, 7) + expect(projected).toEqual({ + id: DOCUMENT_ID, + filename: 'handbook.pdf', + mimeType: 'application/pdf', + fileSize: 1234, + enabled: false, + tags: { + tag1: 'Billing', + number1: '42', + date1: '2026-01-02T00:00:00.000Z', + boolean1: 'false', + }, + file: `files/${DOCUMENT_ID}/handbook.pdf`, + chunks: `chunks/${DOCUMENT_ID}.ndjson`, + chunkCount: 7, + tokenCount: 900, + characterCount: 4000, + }) + expect( + knowledgeBundleManifestSchema.safeParse(manifest({ documents: [projected] })).success + ).toBe(true) + }) +}) + +describe('entry paths', () => { + it('drops directories and illegal characters from the file leaf', () => { + expect(fileEntryPath('doc-1', '../../etc/passwd')).toBe('files/doc-1/passwd') + expect(fileEntryPath('doc-1', 'a:c.txt')).toBe('files/doc-1/a_b__c.txt') + expect(safeBundleLeafName('..')).toBe('file') + expect(safeBundleLeafName('x'.repeat(300))).toHaveLength(200) + }) +}) + +describe('vector codec', () => { + it('round-trips float32 vectors of every stored width', () => { + for (const width of [384, 768, 1024, 1536, 3072]) { + const vector = Array.from({ length: width }, (_, index) => Math.fround(index / width - 0.5)) + expect(decodeVectorBase64(encodeVectorBase64(vector), width)).toEqual(vector) + } + }) + + it('refuses a payload whose width differs from the declared dimension', () => { + expect(() => decodeVectorBase64(encodeVectorBase64([1, 2, 3]), 4)).toThrow( + KnowledgeBundleVectorError + ) + }) + + it('refuses non-finite values', () => { + expect(() => decodeVectorBase64(encodeVectorBase64([1, Number.NaN]), 2)).toThrow( + KnowledgeBundleVectorError + ) + expect(() => decodeVectorBase64(encodeVectorBase64([Number.POSITIVE_INFINITY, 1]), 2)).toThrow( + KnowledgeBundleVectorError + ) + }) +}) diff --git a/apps/sim/lib/knowledge/transfer/bundle.ts b/apps/sim/lib/knowledge/transfer/bundle.ts new file mode 100644 index 00000000000..1503c85de4f --- /dev/null +++ b/apps/sim/lib/knowledge/transfer/bundle.ts @@ -0,0 +1,276 @@ +import { z } from 'zod' +import { chunkingConfigSchema } from '@/lib/api/contracts/knowledge/base' +import { KB_EMBEDDING_STORAGE_DIMENSIONS } from '@/lib/embeddings/catalog' +import { + ALL_TAG_SLOTS, + type AllTagSlot, + isValidSlotForFieldType, + KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH, + KNOWLEDGE_BUNDLE_VERSION, + KNOWLEDGE_TAG_DISPLAY_NAME_MAX_LENGTH, + MAX_KNOWLEDGE_BUNDLE_CHUNK_CONTENT_LENGTH, + MAX_KNOWLEDGE_BUNDLE_DOCUMENTS, + SUPPORTED_FIELD_TYPES, +} from '@/lib/knowledge/constants' +import { MAX_DOCUMENT_CHUNKS } from '@/lib/knowledge/documents/document-processing-error' +import { MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE } from '@/lib/uploads/shared/types' +import { safeZipLeafName } from '@/lib/uploads/zip-entry-path' + +/** + * The knowledge-base bundle: one zip holding a knowledge base's configuration, + * tag definitions, original files, chunk text, and optionally chunk vectors. + * + * ``` + * files// original blob, when the document has one + * chunks/.ndjson one {@link KnowledgeBundleChunkLine} per line + * manifest.json {@link KnowledgeBundleManifest}, written last + * ``` + * + * Nothing that binds a document to the workspace it came from travels: no + * access-control lists, connector links, credentials, uploader identity, + * storage keys, or secret-provenance sidecars. {@link toManifestDocument} is + * the only projection from a stored document onto the manifest, which is what + * keeps that list closed. + * + * Schemas here validate a file format, not an HTTP boundary, so they live with + * the transfer code rather than under `lib/api/contracts`. A manifest is + * untrusted input on import, so every free-text field carries a ceiling even + * where the stored column has none: the domain caps below are reused where one + * exists, and {@link MAX_BUNDLE_TEXT_LENGTH} bounds the rest. + */ + +export const KNOWLEDGE_BUNDLE_MANIFEST_ENTRY = 'manifest.json' + +const BUNDLE_DOCUMENT_ID_PATTERN = /^[A-Za-z0-9_-]{1,64}$/ + +/** Longest leaf name written under `files/`, keeping every entry path short. */ +const MAX_BUNDLE_LEAF_NAME_LENGTH = 200 + +/** Ceiling for names, MIME types, and the embedding model id, which no stored column bounds. */ +const MAX_BUNDLE_TEXT_LENGTH = 255 + +/** + * Ceiling for one tag value. Uploads cap values at 1,000 characters, but + * connector-written values have no cap, so the bundle allows the same width as + * a knowledge base description. + */ +const MAX_BUNDLE_TAG_VALUE_LENGTH = KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH + +const tagSlotSchema = z.enum(ALL_TAG_SLOTS) + +export const knowledgeBundleTagSchema = z + .object({ + slot: tagSlotSchema, + displayName: z.string().trim().min(1).max(KNOWLEDGE_TAG_DISPLAY_NAME_MAX_LENGTH), + fieldType: z.enum(SUPPORTED_FIELD_TYPES), + }) + .strict() + .refine((tag) => isValidSlotForFieldType(tag.slot, tag.fieldType), { + message: 'Tag slot does not belong to its field type', + }) + +const bundleDocumentSchema = z + .object({ + id: z.string().regex(BUNDLE_DOCUMENT_ID_PATTERN, 'Document id must be a short identifier'), + filename: z.string().min(1).max(MAX_BUNDLE_TEXT_LENGTH), + mimeType: z.string().min(1).max(MAX_BUNDLE_TEXT_LENGTH), + fileSize: z.number().int().min(0).max(MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE), + enabled: z.boolean(), + tags: z.partialRecord(tagSlotSchema, z.string().max(MAX_BUNDLE_TAG_VALUE_LENGTH)), + file: z.string().nullable(), + chunks: z.string().nullable(), + chunkCount: z.number().int().min(0).max(MAX_DOCUMENT_CHUNKS), + tokenCount: z.number().int().min(0), + characterCount: z.number().int().min(0), + }) + .strict() + .superRefine((document, ctx) => { + if (document.file === null && document.chunks === null) { + ctx.addIssue({ + code: 'custom', + path: ['file'], + message: 'Document carries neither a file nor chunks', + }) + } + if (document.file !== null && !document.file.startsWith(`files/${document.id}/`)) { + ctx.addIssue({ + code: 'custom', + path: ['file'], + message: 'File entry must live under files//', + }) + } + if (document.chunks !== null && document.chunks !== chunksEntryPath(document.id)) { + ctx.addIssue({ + code: 'custom', + path: ['chunks'], + message: 'Chunks entry must be chunks/.ndjson', + }) + } + }) + +export const knowledgeBundleManifestSchema = z + .object({ + version: z.literal(KNOWLEDGE_BUNDLE_VERSION), + exportedAt: z.iso.datetime(), + embedding: z + .object({ + model: z.string().min(1).max(MAX_BUNDLE_TEXT_LENGTH), + dimension: z.literal( + KB_EMBEDDING_STORAGE_DIMENSIONS, + 'Embedding dimension has no storage column' + ), + vectorsIncluded: z.boolean(), + }) + .strict(), + knowledgeBase: z + .object({ + name: z.string().trim().min(1).max(MAX_BUNDLE_TEXT_LENGTH), + description: z.string().max(KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH).nullable(), + chunkingConfig: chunkingConfigSchema, + }) + .strict(), + tags: z + .array(knowledgeBundleTagSchema) + .max(ALL_TAG_SLOTS.length) + .superRefine((tags, ctx) => { + const slots = new Set() + const names = new Set() + for (const [index, tag] of tags.entries()) { + if (slots.has(tag.slot)) { + ctx.addIssue({ code: 'custom', path: [index, 'slot'], message: 'Duplicate tag slot' }) + } + const name = tag.displayName.toLowerCase() + if (names.has(name)) { + ctx.addIssue({ + code: 'custom', + path: [index, 'displayName'], + message: 'Duplicate tag name', + }) + } + slots.add(tag.slot) + names.add(name) + } + }), + documents: z.array(bundleDocumentSchema).max(MAX_KNOWLEDGE_BUNDLE_DOCUMENTS), + }) + .strict() + +export const knowledgeBundleChunkLineSchema = z + .object({ + index: z + .number() + .int() + .min(0) + .max(MAX_DOCUMENT_CHUNKS - 1), + content: z.string().min(1).max(MAX_KNOWLEDGE_BUNDLE_CHUNK_CONTENT_LENGTH), + tokenCount: z.number().int().min(0), + startOffset: z.number().int().min(0), + endOffset: z.number().int().min(0), + enabled: z.boolean(), + vector: z.base64().optional(), + }) + .strict() + +export type KnowledgeBundleManifest = z.output +export type KnowledgeBundleDocument = KnowledgeBundleManifest['documents'][number] +export type KnowledgeBundleTag = z.output +export type KnowledgeBundleChunkLine = z.output + +/** Tag values a stored document row carries, keyed by slot. */ +export type KnowledgeBundleTagValues = Partial< + Record +> + +/** The stored fields the manifest is projected from. */ +export interface ExportableDocumentRecord { + id: string + filename: string + mimeType: string + fileSize: number + enabled: boolean + tokenCount: number + characterCount: number + tags: KnowledgeBundleTagValues +} + +/** Where a document's entries sit inside the bundle. */ +export interface KnowledgeBundleEntryPaths { + file: string | null + chunks: string | null +} + +export function chunksEntryPath(documentId: string): string { + return `chunks/${documentId}.ndjson` +} + +export function fileEntryPath(documentId: string, filename: string): string { + return `files/${documentId}/${safeBundleLeafName(filename)}` +} + +/** A filesystem-safe leaf name for a bundle entry or the bundle download itself. */ +export function safeBundleLeafName(name: string): string { + return safeZipLeafName(name).slice(0, MAX_BUNDLE_LEAF_NAME_LENGTH) +} + +function tagValueToWire(value: string | number | boolean | Date): string { + if (value instanceof Date) return value.toISOString() + return String(value) +} + +/** + * The one projection from a stored document onto its manifest entry. Any field + * not named here does not leave in an export. + */ +export function toManifestDocument( + record: ExportableDocumentRecord, + entries: KnowledgeBundleEntryPaths, + chunkCount: number +): KnowledgeBundleDocument { + const tags: KnowledgeBundleDocument['tags'] = {} + for (const slot of ALL_TAG_SLOTS) { + const value = record.tags[slot] + if (value !== null && value !== undefined) tags[slot] = tagValueToWire(value) + } + return { + id: record.id, + filename: record.filename, + mimeType: record.mimeType, + fileSize: record.fileSize, + enabled: record.enabled, + tags, + file: entries.file, + chunks: entries.chunks, + chunkCount, + tokenCount: record.tokenCount, + characterCount: record.characterCount, + } +} + +/** Serializes a vector as little-endian float32, which is lossless for pgvector's `real` storage. */ +export function encodeVectorBase64(vector: readonly number[]): string { + return Buffer.from(Float32Array.from(vector).buffer).toString('base64') +} + +export class KnowledgeBundleVectorError extends Error { + constructor(message: string) { + super(message) + this.name = 'KnowledgeBundleVectorError' + } +} + +/** Reverses {@link encodeVectorBase64}, refusing any width or value pgvector could not store. */ +export function decodeVectorBase64(encoded: string, dimension: number): number[] { + const bytes = Buffer.from(encoded, 'base64') + if (bytes.byteLength !== dimension * Float32Array.BYTES_PER_ELEMENT) { + throw new KnowledgeBundleVectorError( + `Vector holds ${bytes.byteLength} bytes; expected ${dimension} float32 values` + ) + } + const vector = Array.from( + new Float32Array(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength)) + ) + if (!vector.every(Number.isFinite)) { + throw new KnowledgeBundleVectorError('Vector contains a non-finite value') + } + return vector +} diff --git a/apps/sim/lib/knowledge/transfer/export-archive.test.ts b/apps/sim/lib/knowledge/transfer/export-archive.test.ts new file mode 100644 index 00000000000..c2712b0332d --- /dev/null +++ b/apps/sim/lib/knowledge/transfer/export-archive.test.ts @@ -0,0 +1,200 @@ +/** + * @vitest-environment node + */ +import { Readable } from 'node:stream' +import JSZip from 'jszip' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + downloadFileStream: vi.fn(), +})) + +vi.mock('@/lib/uploads/core/storage-service', () => ({ + downloadFileStream: mocks.downloadFileStream, +})) + +import type { KnowledgeBaseExportBundle } from '@/lib/knowledge/application/exports' +import { decodeVectorBase64, knowledgeBundleManifestSchema } from '@/lib/knowledge/transfer/bundle' +import { + buildKnowledgeBundleArchive, + knowledgeBundleFileName, +} from '@/lib/knowledge/transfer/export-archive' +import type { ExportableChunk, ExportableDocument } from '@/lib/knowledge/transfer/export-source' + +const STORED_ID = 'doc-stored' +const INLINE_ID = 'doc-inline' +const TEXT_ONLY_ID = 'doc-text' + +function exportableDocument(overrides: Partial): ExportableDocument { + return { + id: STORED_ID, + filename: 'handbook.pdf', + mimeType: 'application/pdf', + fileSize: 3, + enabled: true, + tokenCount: 12, + characterCount: 40, + tags: { tag1: 'Billing' }, + file: { kind: 'storage', key: 'kb/handbook.pdf' }, + hasChunks: true, + ...overrides, + } +} + +function chunk(index: number, vector: number[] | null): ExportableChunk { + return { + index, + content: `chunk ${index}`, + tokenCount: 2, + startOffset: index * 10, + endOffset: index * 10 + 7, + enabled: index !== 1, + vector, + } +} + +async function* chunksOf(...chunks: ExportableChunk[]): AsyncGenerator { + for (const item of chunks) yield item +} + +function bundle(overrides: Partial = {}): KnowledgeBaseExportBundle { + return { + knowledgeBase: { + name: 'Support docs', + description: 'Everything support knows', + chunkingConfig: { maxSize: 1024, minSize: 100, overlap: 200 }, + }, + embedding: { model: 'text-embedding-3-small', dimension: 1536, vectorsIncluded: true }, + tags: [{ slot: 'tag1', displayName: 'Product', fieldType: 'text' }], + documents: [ + exportableDocument({}), + exportableDocument({ + id: INLINE_ID, + filename: 'note.txt', + mimeType: 'text/plain', + file: { + kind: 'data-uri', + fileUrl: `data:text/plain;base64,${Buffer.from('hi').toString('base64')}`, + }, + hasChunks: false, + }), + exportableDocument({ id: TEXT_ONLY_ID, filename: 'wiki page', file: null, hasChunks: true }), + ], + chunks: (documentId) => + documentId === STORED_ID + ? chunksOf(chunk(0, [0.25, 0.5]), chunk(1, [1, 2])) + : chunksOf(chunk(0, null)), + ...overrides, + } +} + +async function readArchive(source: Readable): Promise { + const parts: Buffer[] = [] + for await (const part of source) parts.push(Buffer.from(part)) + return JSZip.loadAsync(Buffer.concat(parts)) +} + +describe('buildKnowledgeBundleArchive', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.downloadFileStream.mockImplementation(async () => Readable.from([Buffer.from('pdf')])) + }) + + it('writes files, chunk lines, and a manifest that validates against the bundle schema', async () => { + const zip = await readArchive(buildKnowledgeBundleArchive(bundle())) + + expect(Object.keys(zip.files)).toEqual([ + `files/${STORED_ID}/handbook.pdf`, + `chunks/${STORED_ID}.ndjson`, + `files/${INLINE_ID}/note.txt`, + `chunks/${TEXT_ONLY_ID}.ndjson`, + 'manifest.json', + ]) + expect(await zip.file(`files/${STORED_ID}/handbook.pdf`)!.async('string')).toBe('pdf') + expect(await zip.file(`files/${INLINE_ID}/note.txt`)!.async('string')).toBe('hi') + + const manifest = knowledgeBundleManifestSchema.parse( + JSON.parse(await zip.file('manifest.json')!.async('string')) + ) + expect(manifest.embedding).toEqual({ + model: 'text-embedding-3-small', + dimension: 1536, + vectorsIncluded: true, + }) + expect(manifest.knowledgeBase.name).toBe('Support docs') + expect(manifest.tags).toEqual([{ slot: 'tag1', displayName: 'Product', fieldType: 'text' }]) + expect( + manifest.documents.map((document) => [document.id, document.file, document.chunks]) + ).toEqual([ + [STORED_ID, `files/${STORED_ID}/handbook.pdf`, `chunks/${STORED_ID}.ndjson`], + [INLINE_ID, `files/${INLINE_ID}/note.txt`, null], + [TEXT_ONLY_ID, null, `chunks/${TEXT_ONLY_ID}.ndjson`], + ]) + }) + + /** Counts come from what the chunk stream wrote, not from the stored counter. */ + it('records the chunk count actually written and carries vectors on every line', async () => { + const zip = await readArchive(buildKnowledgeBundleArchive(bundle())) + const lines = (await zip.file(`chunks/${STORED_ID}.ndjson`)!.async('string')) + .trimEnd() + .split('\n') + .map((line) => JSON.parse(line)) + + expect(lines).toHaveLength(2) + expect(lines[0]).toMatchObject({ index: 0, content: 'chunk 0', enabled: true }) + expect(lines[1]).toMatchObject({ index: 1, enabled: false }) + expect(decodeVectorBase64(lines[0].vector, 2)).toEqual([0.25, 0.5]) + + const manifest = JSON.parse(await zip.file('manifest.json')!.async('string')) + expect(manifest.documents[0].chunkCount).toBe(2) + expect(manifest.documents[2].chunkCount).toBe(1) + }) + + it('omits vectors when the bundle does not include them', async () => { + const zip = await readArchive( + buildKnowledgeBundleArchive( + bundle({ + embedding: { model: 'text-embedding-3-small', dimension: 1536, vectorsIncluded: false }, + }) + ) + ) + const [first] = (await zip.file(`chunks/${STORED_ID}.ndjson`)!.async('string')).split('\n') + expect(JSON.parse(first)).not.toHaveProperty('vector') + }) + + /** Blobs open only as the archiver reaches them, so a large base never fans out storage reads. */ + it('opens stored blobs one at a time, in entry order', async () => { + const events: string[] = [] + mocks.downloadFileStream.mockImplementation(async ({ key }: { key: string }) => { + events.push(`open:${key}`) + return Readable.from([Buffer.from('pdf')]) + }) + const archive = buildKnowledgeBundleArchive( + bundle({ + documents: [ + exportableDocument({ id: 'doc-a', file: { kind: 'storage', key: 'kb/a.pdf' } }), + exportableDocument({ id: 'doc-b', file: { kind: 'storage', key: 'kb/b.pdf' } }), + ], + }) + ) + archive.on('entry', (entry: { name: string }) => events.push(`entry:${entry.name}`)) + + await readArchive(archive) + expect(events).toEqual([ + 'open:kb/a.pdf', + 'entry:files/doc-a/handbook.pdf', + 'entry:chunks/doc-a.ndjson', + 'open:kb/b.pdf', + 'entry:files/doc-b/handbook.pdf', + 'entry:chunks/doc-b.ndjson', + 'entry:manifest.json', + ]) + }) +}) + +describe('knowledgeBundleFileName', () => { + it('sanitizes the base name and appends the bundle suffix', () => { + expect(knowledgeBundleFileName('Support docs')).toBe('Support docs.simkb.zip') + expect(knowledgeBundleFileName('a/b:c')).toBe('b_c.simkb.zip') + }) +}) diff --git a/apps/sim/lib/knowledge/transfer/export-archive.ts b/apps/sim/lib/knowledge/transfer/export-archive.ts new file mode 100644 index 00000000000..97c73b2ae3f --- /dev/null +++ b/apps/sim/lib/knowledge/transfer/export-archive.ts @@ -0,0 +1,146 @@ +import { once } from 'node:events' +import { Readable } from 'node:stream' +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { ZipArchive } from 'archiver' +import { decodeDataUriWithinLimit } from '@/lib/file-parsers/data-uri' +import type { KnowledgeBaseExportBundle } from '@/lib/knowledge/application/exports' +import { KNOWLEDGE_BUNDLE_VERSION } from '@/lib/knowledge/constants' +import { + chunksEntryPath, + encodeVectorBase64, + fileEntryPath, + KNOWLEDGE_BUNDLE_MANIFEST_ENTRY, + type KnowledgeBundleChunkLine, + type KnowledgeBundleDocument, + type KnowledgeBundleManifest, + safeBundleLeafName, + toManifestDocument, +} from '@/lib/knowledge/transfer/bundle' +import type { ExportableChunk, ExportableFileSource } from '@/lib/knowledge/transfer/export-source' +import { downloadFileStream } from '@/lib/uploads/core/storage-service' +import { MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE } from '@/lib/uploads/shared/types' + +const logger = createLogger('KnowledgeExportArchive') + +/** Chunk text compresses several times over; vectors do not, so a middle level pays off either way. */ +const ZIP_COMPRESSION_LEVEL = 6 + +/** The download name for a knowledge base's bundle. */ +export function knowledgeBundleFileName(knowledgeBaseName: string): string { + return `${safeBundleLeafName(knowledgeBaseName)}.simkb.zip` +} + +async function openFileSource(source: ExportableFileSource): Promise { + if (source.kind === 'storage') { + return downloadFileStream({ key: source.key, context: 'knowledge-base' }) + } + return decodeDataUriWithinLimit(source.fileUrl, MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE).buffer +} + +/** + * Projects a chunk onto its NDJSON line. `vectors` is the manifest's promise: + * a vector the source still carries is dropped when the manifest says none are + * included, so the lines never contradict `embedding.vectorsIncluded`. + */ +function toChunkLine(chunk: ExportableChunk, vectors: boolean): KnowledgeBundleChunkLine { + const { vector, ...line } = chunk + return vectors && vector ? { ...line, vector: encodeVectorBase64(vector) } : line +} + +/** + * Appends one entry and resolves once the archiver has consumed it. + * + * The archiver pipes a source into its own buffer the moment it is appended, + * so appending everything up front would open every blob at once and read the + * manifest before any chunk stream ended. Waiting on the archiver's `entry` + * event keeps exactly one source open and lets the manifest go last with the + * counts the chunk streams actually produced. Archiver drains its queue one + * entry at a time and emits `entry` exactly once per append, or `error` in its + * place, which `once` turns into a rejection. + */ +async function appendEntry( + archive: ZipArchive, + source: Readable | Buffer | string, + name: string +): Promise { + const consumed = once(archive, 'entry') + archive.append(source, { name }) + await consumed +} + +/** Appends a document's chunks as NDJSON and returns how many lines were written. */ +async function appendChunkEntry( + archive: ZipArchive, + chunks: AsyncIterable, + vectors: boolean, + name: string +): Promise { + let written = 0 + const lines = Readable.from( + (async function* () { + for await (const chunk of chunks) { + yield `${JSON.stringify(toChunkLine(chunk, vectors))}\n` + written += 1 + } + })(), + { objectMode: false } + ) + await appendEntry(archive, lines, name) + return written +} + +async function appendBundleEntries( + archive: ZipArchive, + bundle: KnowledgeBaseExportBundle +): Promise { + const documents: KnowledgeBundleDocument[] = [] + for (const document of bundle.documents) { + let file: string | null = null + if (document.file) { + file = fileEntryPath(document.id, document.filename) + await appendEntry(archive, await openFileSource(document.file), file) + } + const chunks = document.hasChunks ? chunksEntryPath(document.id) : null + const chunkCount = chunks + ? await appendChunkEntry( + archive, + bundle.chunks(document.id), + bundle.embedding.vectorsIncluded, + chunks + ) + : 0 + documents.push(toManifestDocument(document, { file, chunks }, chunkCount)) + } + + const manifest: KnowledgeBundleManifest = { + version: KNOWLEDGE_BUNDLE_VERSION, + exportedAt: new Date().toISOString(), + embedding: bundle.embedding, + knowledgeBase: bundle.knowledgeBase, + tags: bundle.tags, + documents, + } + await appendEntry(archive, JSON.stringify(manifest, null, 2), KNOWLEDGE_BUNDLE_MANIFEST_ENTRY) + await archive.finalize() +} + +/** + * Streams a knowledge base as its bundle archive. + * + * Entries are appended one at a time in document order, so peak memory is one + * blob stream or one page of chunks. The manifest goes last: every document's + * chunk count is whatever its chunk stream actually wrote, so a document edited + * while the export ran still describes itself truthfully. + */ +export function buildKnowledgeBundleArchive(bundle: KnowledgeBaseExportBundle): Readable { + const archive = new ZipArchive({ zlib: { level: ZIP_COMPRESSION_LEVEL } }) + archive.on('warning', (error: Error) => { + logger.warn('Archive warning while streaming knowledge base bundle', { error }) + }) + appendBundleEntries(archive, bundle).catch((error: unknown) => { + logger.error('Failed to build knowledge base bundle archive', { error }) + archive.destroy(toError(error)) + }) + return archive +} diff --git a/apps/sim/lib/knowledge/transfer/export-source.ts b/apps/sim/lib/knowledge/transfer/export-source.ts new file mode 100644 index 00000000000..f284d4316a1 --- /dev/null +++ b/apps/sim/lib/knowledge/transfer/export-source.ts @@ -0,0 +1,190 @@ +import { db } from '@sim/db' +import { document, embedding } from '@sim/db/schema' +import { and, asc, eq, gt, isNull, sql } from 'drizzle-orm' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import type { KbEmbeddingDimensions } from '@/lib/embeddings/catalog' +import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' +import { WORKSPACE_ACCESS_SCOPE } from '@/lib/knowledge/access/scope' +import { ALL_TAG_SLOTS, MAX_KNOWLEDGE_BUNDLE_DOCUMENTS } from '@/lib/knowledge/constants' +import { getTagDefinitions } from '@/lib/knowledge/tags/service' +import { + type ExportableDocumentRecord, + type KnowledgeBundleChunkLine, + type KnowledgeBundleTag, + knowledgeBundleTagSchema, +} from '@/lib/knowledge/transfer/bundle' +import { embeddingVectorColumn } from '@/lib/knowledge/vector-columns' + +/** + * Read side of a knowledge-base export. Chunk reads page by keyset so a large + * document never materializes at once, and every document read carries + * {@link knowledgeAccessCondition} for the plain workspace scope: a bundle + * drops access-control lists, so only what every workspace member can already + * read may leave. + */ + +/** Chunk rows per page. A 3072-wide vector page is ~15 MB on the wire, so vector reads page smaller. */ +const CHUNK_PAGE_SIZE = { text: 500, vectors: 100 } as const + +/** Where a document's original bytes come from, when it has any. */ +export type ExportableFileSource = + | { kind: 'storage'; key: string } + | { kind: 'data-uri'; fileUrl: string } + +export interface ExportableDocument extends ExportableDocumentRecord { + file: ExportableFileSource | null + /** True when the document finished processing and holds chunks worth exporting. */ + hasChunks: boolean +} + +/** A chunk as stored, before its vector is encoded for the wire. */ +export type ExportableChunk = Omit & { + vector: number[] | null +} + +function exportableDocumentCondition(knowledgeBaseId: string) { + return and( + eq(document.knowledgeBaseId, knowledgeBaseId), + isNull(document.deletedAt), + isNull(document.archivedAt), + eq(document.userExcluded, false), + knowledgeAccessCondition(WORKSPACE_ACCESS_SCOPE) + ) +} + +function fileSourceFor(row: { + storageKey: string | null + fileUrl: string +}): ExportableFileSource | null { + if (row.storageKey) return { kind: 'storage', key: row.storageKey } + if (row.fileUrl.startsWith('data:')) return { kind: 'data-uri', fileUrl: row.fileUrl } + return null +} + +/** + * The base's tag definitions in slot order, validated against the bundle's tag + * schema: the stored `tagSlot` column type only names the text slots and + * `fieldType` is free text, so validation is what proves the rows form an + * importable manifest. + */ +export async function listExportableTags(knowledgeBaseId: string): Promise { + const definitions = await getTagDefinitions(knowledgeBaseId) + return knowledgeBundleTagSchema.array().parse( + definitions.map(({ tagSlot, displayName, fieldType }) => ({ + slot: tagSlot, + displayName, + fieldType, + })) + ) +} + +/** + * Every document the bundle will carry, in id order. Reads one row past + * {@link MAX_KNOWLEDGE_BUNDLE_DOCUMENTS} and refuses the base when it is there, + * so no export ever produces a bundle an import would refuse. Documents with + * neither a file nor chunks are dropped, since the manifest cannot describe them. + */ +export async function listExportableDocuments( + knowledgeBaseId: string +): Promise { + const rows = await db + .select({ + id: document.id, + filename: document.filename, + mimeType: document.mimeType, + fileSize: document.fileSize, + enabled: document.enabled, + storageKey: document.storageKey, + fileUrl: document.fileUrl, + processingStatus: document.processingStatus, + chunkCount: document.chunkCount, + tokenCount: document.tokenCount, + characterCount: document.characterCount, + tag1: document.tag1, + tag2: document.tag2, + tag3: document.tag3, + tag4: document.tag4, + tag5: document.tag5, + tag6: document.tag6, + tag7: document.tag7, + number1: document.number1, + number2: document.number2, + number3: document.number3, + number4: document.number4, + number5: document.number5, + date1: document.date1, + date2: document.date2, + boolean1: document.boolean1, + boolean2: document.boolean2, + boolean3: document.boolean3, + }) + .from(document) + .where(exportableDocumentCondition(knowledgeBaseId)) + .orderBy(asc(document.id)) + .limit(MAX_KNOWLEDGE_BUNDLE_DOCUMENTS + 1) + if (rows.length > MAX_KNOWLEDGE_BUNDLE_DOCUMENTS) { + throw new OrchestrationError( + 'payload_too_large', + `Knowledge base has more than ${MAX_KNOWLEDGE_BUNDLE_DOCUMENTS} documents, the most an export carries` + ) + } + + const documents: ExportableDocument[] = [] + for (const row of rows) { + const file = fileSourceFor(row) + const hasChunks = row.processingStatus === 'completed' && row.chunkCount > 0 + if (!file && !hasChunks) continue + documents.push({ + id: row.id, + filename: row.filename, + mimeType: row.mimeType, + fileSize: row.fileSize, + enabled: row.enabled, + tokenCount: row.tokenCount, + characterCount: row.characterCount, + file, + hasChunks, + tags: Object.fromEntries(ALL_TAG_SLOTS.map((slot) => [slot, row[slot]])), + }) + } + return documents +} + +/** + * A document's chunks in chunk-index order, one page at a time. The vector column is + * read only when `dimensions` is given, so a text-only export never pulls the + * widest column off disk. `(documentId, chunkIndex)` is unique, so the index + * alone is the keyset and each page is one index range scan. + */ +export async function* iterateDocumentChunks( + documentId: string, + dimensions: KbEmbeddingDimensions | null +): AsyncGenerator { + const pageSize = dimensions ? CHUNK_PAGE_SIZE.vectors : CHUNK_PAGE_SIZE.text + let after: number | null = null + for (;;) { + const page = await db + .select({ + index: embedding.chunkIndex, + content: embedding.content, + tokenCount: embedding.tokenCount, + startOffset: embedding.startOffset, + endOffset: embedding.endOffset, + enabled: embedding.enabled, + vector: dimensions ? embeddingVectorColumn(dimensions) : sql`null`, + }) + .from(embedding) + .where( + and( + eq(embedding.documentId, documentId), + after === null ? undefined : gt(embedding.chunkIndex, after) + ) + ) + .orderBy(asc(embedding.chunkIndex)) + .limit(pageSize) + + yield* page + if (page.length < pageSize) break + after = page[page.length - 1].index + } +} diff --git a/apps/sim/lib/permission-groups/capabilities.test.ts b/apps/sim/lib/permission-groups/capabilities.test.ts index 30e19dc4cf0..cf97d69dfba 100644 --- a/apps/sim/lib/permission-groups/capabilities.test.ts +++ b/apps/sim/lib/permission-groups/capabilities.test.ts @@ -17,6 +17,7 @@ describe('knowledge capability rules', () => { const create = CAPABILITY_RULES['knowledge.create'] const upload = CAPABILITY_RULES['knowledge.upload'] const connectors = CAPABILITY_RULES['knowledge.connectors'] + const exportRule = CAPABILITY_RULES['knowledge.export'] it('permits creation and upload under the unrestricted config', () => { expect(create.deniedBy(DEFAULT_PERMISSION_GROUP_CONFIG)).toBe(false) @@ -46,4 +47,16 @@ describe('knowledge capability rules', () => { const emptied = configWith({ allowedKnowledgeConnectors: [] }) expect(connectors.deniedBy(emptied, 'google_drive')).toBe(true) }) + + it('permits export under the unrestricted config', () => { + expect(exportRule.deniedBy(DEFAULT_PERMISSION_GROUP_CONFIG)).toBe(false) + }) + + it('withholds export from its own key', () => { + expect(exportRule.deniedBy(configWith({ disableKnowledgeBaseExport: true }))).toBe(true) + }) + + it('withholds export when the module is hidden', () => { + expect(exportRule.deniedBy(configWith({ hideKnowledgeBaseTab: true }))).toBe(true) + }) }) diff --git a/apps/sim/lib/permission-groups/capabilities.ts b/apps/sim/lib/permission-groups/capabilities.ts index b11187c6583..e210e952f6c 100644 --- a/apps/sim/lib/permission-groups/capabilities.ts +++ b/apps/sim/lib/permission-groups/capabilities.ts @@ -57,6 +57,7 @@ export const CAPABILITY_IDS = [ 'triggers.webhook', 'copilot.tool_auto_approval', 'sandboxes.use', + 'knowledge.export', ] as const export type PermissionGroupCapability = (typeof CAPABILITY_IDS)[number] @@ -422,6 +423,14 @@ export const CAPABILITY_RULES = { describe: 'The Sandboxes module', deniedBy: (config) => config.hideSandboxesTab, }, + /** Subsumes `knowledge.use` for the same reason as `knowledge.create`. */ + 'knowledge.export': { + kind: 'static', + configKeys: ['disableKnowledgeBaseExport', 'hideKnowledgeBaseTab'], + detailCode: 'PERMISSION_GROUP_CAPABILITY_BLOCKED', + describe: 'Exporting a knowledge base', + deniedBy: (config) => config.disableKnowledgeBaseExport || config.hideKnowledgeBaseTab, + }, } satisfies { readonly [K in PermissionGroupCapability]: CapabilityRule } /** diff --git a/apps/sim/lib/permission-groups/fields.test.ts b/apps/sim/lib/permission-groups/fields.test.ts index 8470d27bacc..3273d17ea9a 100644 --- a/apps/sim/lib/permission-groups/fields.test.ts +++ b/apps/sim/lib/permission-groups/fields.test.ts @@ -165,6 +165,7 @@ const fixtures: readonly CoercionFixture[] = [ disableToolAutoApproval: true, hideSandboxesTab: true, disableOAuthAppAccess: true, + disableKnowledgeBaseExport: true, }, expected: { allowedIntegrations: ['slack_v2'], @@ -208,6 +209,7 @@ const fixtures: readonly CoercionFixture[] = [ disableToolAutoApproval: true, hideSandboxesTab: true, disableOAuthAppAccess: true, + disableKnowledgeBaseExport: true, }, }, ] diff --git a/apps/sim/lib/permission-groups/fields.ts b/apps/sim/lib/permission-groups/fields.ts index c73a62a1e83..bbe5445f160 100644 --- a/apps/sim/lib/permission-groups/fields.ts +++ b/apps/sim/lib/permission-groups/fields.ts @@ -494,6 +494,13 @@ export const PERMISSION_GROUP_FIELDS = { category: 'Credentials & Access', hint: "Prevent OAuth apps from accessing this group's workspaces. The organization's default group also governs authorizing apps and refreshing their access.", }), + disableKnowledgeBaseExport: booleanRestriction('capability', { + scope: 'workspace', + id: 'disable-knowledge-base-export', + label: 'Knowledge Base Export', + category: 'Knowledge Base', + hint: 'Prevent downloading a whole knowledge base as an archive.', + }), } satisfies Record export type PermissionGroupFields = typeof PERMISSION_GROUP_FIELDS diff --git a/apps/sim/lib/uploads/zip-entry-path.ts b/apps/sim/lib/uploads/zip-entry-path.ts index d3f4e895abd..63c30b18c72 100644 --- a/apps/sim/lib/uploads/zip-entry-path.ts +++ b/apps/sim/lib/uploads/zip-entry-path.ts @@ -53,6 +53,11 @@ function safeEntryPath(segments: string[]): string { .join('/') } +/** One archive-safe leaf name for `name`, with its directory and illegal characters dropped. */ +export function safeZipLeafName(name: string): string { + return safeEntryPath([toLeafName(name)]) +} + /** * Append a numeric suffix to the file name of an entry path, leaving the directory * portion intact (`docs/report.pdf` -> `docs/report (1).pdf`). diff --git a/packages/audit/src/types.ts b/packages/audit/src/types.ts index 006a2bc7dff..0f0477c49c5 100644 --- a/packages/audit/src/types.ts +++ b/packages/audit/src/types.ts @@ -123,6 +123,7 @@ export const AuditAction = { KNOWLEDGE_BASE_UPDATED: 'knowledge_base.updated', KNOWLEDGE_BASE_DELETED: 'knowledge_base.deleted', KNOWLEDGE_BASE_RESTORED: 'knowledge_base.restored', + KNOWLEDGE_BASE_EXPORTED: 'knowledge_base.exported', // MCP Servers MCP_SERVER_ADDED: 'mcp_server.added', diff --git a/packages/sim-cli/README.md b/packages/sim-cli/README.md index 54b3a234b88..d12e65321b9 100644 --- a/packages/sim-cli/README.md +++ b/packages/sim-cli/README.md @@ -188,6 +188,7 @@ The commands you will use most often are: | Upload or download files | `sim files upload ./report.pdf`, `sim files get ` | | Search knowledge bases | `sim knowledge search --query "refund policy" --kb ` | | Upload a knowledge document | `sim knowledge documents upload ./handbook.pdf` | +| Export a knowledge base | `sim knowledge export -o ./kb.simkb.zip` | | Manage integration credentials | `sim credentials --help` | | Manage workspace secrets | `sim secrets list`, `sim secrets set ` | diff --git a/packages/sim-cli/src/commands/protocol/index.ts b/packages/sim-cli/src/commands/protocol/index.ts index ea85ee7d69f..420739760b3 100644 --- a/packages/sim-cli/src/commands/protocol/index.ts +++ b/packages/sim-cli/src/commands/protocol/index.ts @@ -3,6 +3,7 @@ import { attachChat } from './chat' import { attachFileGet } from './files-get' import { attachFileUpload } from './files-upload' import { attachKnowledgeDocumentUpload } from './knowledge-document-upload' +import { attachKnowledgeExport } from './knowledge-export' import { attachLogsFollow } from './logs-follow' import { attachResourceDirectoryCommands } from './resource-directory' import { attachTableImport } from './tables-import' @@ -31,6 +32,7 @@ export function attachProtocolCommands(program: Command): void { const knowledge = group(program, 'knowledge') attachKnowledgeDocumentUpload(group(knowledge, 'documents')) + attachKnowledgeExport(knowledge) attachResourceDirectoryCommands(knowledge, { kind: 'knowledge', resources: 'listKnowledgeBases', diff --git a/packages/sim-cli/src/commands/protocol/knowledge-export.test.ts b/packages/sim-cli/src/commands/protocol/knowledge-export.test.ts new file mode 100644 index 00000000000..3eb8387c3e9 --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/knowledge-export.test.ts @@ -0,0 +1,239 @@ +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Command } from 'commander' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { buildGeneratedCommands } from '../../runtime/build' +import { attachProtocolCommands } from './index' +import { attachmentFileName } from './knowledge-export' + +const { output, requestRaw } = vi.hoisted(() => ({ + output: { format: 'json' }, + requestRaw: vi.fn(), +})) + +vi.mock('../../context', () => ({ + clientFrom: () => ({ + client: { requestRaw, requireWorkspace: () => 'ws_local' }, + profile: { + workspaceId: 'ws_local', + output: output.format, + name: 'default', + apiKey: 'k', + endpoint: 'https://sim.example', + }, + }), +})) + +const KB_ID = '4c1b7f60-2d55-4a3e-9c18-70b6ea2f9d31' + +let dir: string +let originalCwd: string + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'sim-kb-export-')) + originalCwd = process.cwd() + output.format = 'json' + requestRaw.mockReset() +}) + +afterEach(() => { + process.chdir(originalCwd) + vi.restoreAllMocks() + rmSync(dir, { recursive: true, force: true }) +}) + +function zipResponse(fileName?: string): Response { + return new Response(new Uint8Array([0x50, 0x4b, 0x03, 0x04]), { + status: 200, + headers: { + 'content-type': 'application/zip', + ...(fileName ? { 'content-disposition': `attachment; filename="${fileName}"` } : {}), + }, + }) +} + +function program(): Command { + const root = new Command('sim').exitOverride() + for (const group of buildGeneratedCommands()) root.addCommand(group) + attachProtocolCommands(root) + const override = (command: Command) => { + command.exitOverride() + command.commands.forEach(override) + } + override(root) + return root +} + +function captureLog(): string[] { + const logged: string[] = [] + vi.spyOn(console, 'log').mockImplementation((line: string) => logged.push(line)) + return logged +} + +async function withStdoutTTY(isTTY: boolean, run: () => Promise): Promise { + const original = Object.getOwnPropertyDescriptor(process.stdout, 'isTTY') + Object.defineProperty(process.stdout, 'isTTY', { configurable: true, value: isTTY }) + try { + return await run() + } finally { + if (original) Object.defineProperty(process.stdout, 'isTTY', original) + else Reflect.deleteProperty(process.stdout, 'isTTY') + } +} + +describe('knowledge export', () => { + it('saves the bundle to --output-file and prints a machine-readable result', async () => { + const target = join(dir, 'handbook.simkb.zip') + requestRaw.mockResolvedValue(zipResponse('Handbook.simkb.zip')) + const logged = captureLog() + + await program().parseAsync([ + 'node', + 'sim', + 'knowledge', + 'export', + KB_ID, + '--output-file', + target, + ]) + + expect(readFileSync(target)).toEqual(Buffer.from([0x50, 0x4b, 0x03, 0x04])) + expect(JSON.parse(logged[0])).toEqual({ + id: KB_ID, + path: target, + status: 'saved', + vectors: true, + }) + expect(requestRaw).toHaveBeenCalledWith(`/api/v2/knowledge/${KB_ID}/export`, { + method: 'GET', + query: { workspaceId: 'ws_local', vectors: true }, + }) + }) + + it('refuses to overwrite an existing file without --force', async () => { + const target = join(dir, 'existing.simkb.zip') + writeFileSync(target, 'precious') + requestRaw.mockResolvedValue(zipResponse()) + + await expect( + program().parseAsync(['node', 'sim', 'knowledge', 'export', KB_ID, '-o', target]) + ).rejects.toThrow(/already exists.*--force/s) + + expect(readFileSync(target, 'utf8')).toBe('precious') + }) + + it('overwrites an existing file with --force', async () => { + const target = join(dir, 'existing.simkb.zip') + writeFileSync(target, 'old') + requestRaw.mockResolvedValue(zipResponse()) + captureLog() + + await program().parseAsync([ + 'node', + 'sim', + 'knowledge', + 'export', + KB_ID, + '-o', + target, + '--force', + ]) + + expect(readFileSync(target)).toEqual(Buffer.from([0x50, 0x4b, 0x03, 0x04])) + }) + + it('sends vectors=false for --no-vectors and reports it', async () => { + const target = join(dir, 'lean.simkb.zip') + requestRaw.mockResolvedValue(zipResponse()) + const logged = captureLog() + + await program().parseAsync([ + 'node', + 'sim', + 'knowledge', + 'export', + KB_ID, + '-o', + target, + '--no-vectors', + ]) + + expect(requestRaw).toHaveBeenCalledWith(`/api/v2/knowledge/${KB_ID}/export`, { + method: 'GET', + query: { workspaceId: 'ws_local', vectors: false }, + }) + expect(JSON.parse(logged[0]).vectors).toBe(false) + }) + + it('names the file after Content-Disposition in the current directory by default', async () => { + process.chdir(dir) + requestRaw.mockResolvedValue(zipResponse('Refund Policy.simkb.zip')) + const logged = captureLog() + + await program().parseAsync(['node', 'sim', 'knowledge', 'export', KB_ID]) + + const expected = join(process.cwd(), 'Refund Policy.simkb.zip') + expect(existsSync(expected)).toBe(true) + expect(JSON.parse(logged[0]).path).toBe(expected) + }) + + it('falls back to the knowledge base id when the server names no file', async () => { + process.chdir(dir) + requestRaw.mockResolvedValue(zipResponse()) + captureLog() + + await program().parseAsync(['node', 'sim', 'knowledge', 'export', KB_ID]) + + expect(existsSync(join(process.cwd(), `${KB_ID}.simkb.zip`))).toBe(true) + }) + + it('streams raw bytes to stdout for --output-file -', async () => { + requestRaw.mockResolvedValue(zipResponse('x.simkb.zip')) + const chunks: Uint8Array[] = [] + vi.spyOn(process.stdout, 'write').mockImplementation((chunk: string | Uint8Array) => { + chunks.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk) + return true + }) + const logged = vi.spyOn(console, 'log').mockImplementation(() => {}) + + await withStdoutTTY(false, () => + program().parseAsync(['node', 'sim', 'knowledge', 'export', KB_ID, '-o', '-']) + ) + + expect(Buffer.concat(chunks)).toEqual(Buffer.from([0x50, 0x4b, 0x03, 0x04])) + expect(logged).not.toHaveBeenCalled() + }) + + it('refuses to write the zip to an interactive terminal', async () => { + requestRaw.mockResolvedValue(zipResponse()) + + await withStdoutTTY(true, () => + expect( + program().parseAsync(['node', 'sim', 'knowledge', 'export', KB_ID, '-o', '-']) + ).rejects.toThrow(/Refusing to write application\/zip.*--output-file/s) + ) + }) + + it('rejects --force with the stdout alias before any request', async () => { + await expect( + program().parseAsync(['node', 'sim', 'knowledge', 'export', KB_ID, '-o', '-', '--force']) + ).rejects.toThrow(/--force requires --output-file /) + expect(requestRaw).not.toHaveBeenCalled() + }) +}) + +describe('attachmentFileName', () => { + it('reads the quoted file name', () => { + expect(attachmentFileName('attachment; filename="Handbook.simkb.zip"')).toBe( + 'Handbook.simkb.zip' + ) + }) + + it('keeps only the base name and ignores a missing or empty header', () => { + expect(attachmentFileName('attachment; filename="../../etc/passwd"')).toBe('passwd') + expect(attachmentFileName('attachment; filename=".."')).toBeNull() + expect(attachmentFileName('attachment')).toBeNull() + expect(attachmentFileName(null)).toBeNull() + }) +}) diff --git a/packages/sim-cli/src/commands/protocol/knowledge-export.ts b/packages/sim-cli/src/commands/protocol/knowledge-export.ts new file mode 100644 index 00000000000..9418429e743 --- /dev/null +++ b/packages/sim-cli/src/commands/protocol/knowledge-export.ts @@ -0,0 +1,93 @@ +import { basename, join } from 'node:path' +import type { Command } from 'commander' +import { clientFrom } from '../../context' +import { V2_OPERATIONS } from '../../generated/v2-api' +import { resolvePath, SimApiError } from '../../http/client' +import { isTerminalSafeContentType, saveToFile, streamToStdout } from './files-get' +import { printProtocolResult } from './result' + +interface KnowledgeExportOptions { + outputFile?: string + force?: boolean + vectors: boolean +} + +/** + * The file name a `Content-Disposition: attachment; filename="..."` header + * carries, or `null` when the header names none. + * + * Only the quoted form is read: the export route always emits it, with a name + * the server has already stripped of quotes, slashes, and control characters. + * Only the base name is kept so a directory in the header can never decide + * where the archive lands on the caller's disk. + */ +export function attachmentFileName(contentDisposition: string | null): string | null { + const match = contentDisposition ? /filename="([^"]*)"/.exec(contentDisposition) : null + if (!match) return null + const base = basename(match[1].trim()) + return base && base !== '.' && base !== '..' ? base : null +} + +export function attachKnowledgeExport(knowledge: Command): void { + knowledge + .command('export') + .argument('', 'Knowledge base to export') + .allowExcessArguments(false) + .description('Export a knowledge base as a .simkb.zip bundle') + .option( + '-o, --output-file ', + 'Write the bundle to this path instead of the name the server suggests; pass - to stream it to stdout' + ) + .option('--force', 'Overwrite --output-file if it already exists') + .option( + '--no-vectors', + 'Leave chunk vectors out of the bundle, so an import re-embeds every chunk' + ) + .action(async (knowledgeBaseId: string, options: KnowledgeExportOptions, command: Command) => { + const writesToStdout = options.outputFile === '-' + if (writesToStdout && options.force) { + throw new SimApiError('--force requires --output-file ', 0) + } + + const { client, profile } = clientFrom(command) + const workspaceId = client.requireWorkspace() + const operation = V2_OPERATIONS.exportKnowledgeBase + const response = await client.requestRaw(resolvePath(operation.path, { knowledgeBaseId }), { + method: operation.method, + query: { workspaceId, vectors: options.vectors }, + }) + if (!response.body) { + throw new SimApiError('Knowledge base export response was empty.', response.status) + } + + if (writesToStdout) { + const contentType = response.headers.get('content-type') + if (process.stdout.isTTY && !isTerminalSafeContentType(contentType)) { + await response.body.cancel() + throw new SimApiError( + `Refusing to write ${contentType ?? 'unknown content'} to an interactive terminal. Use --output-file or pipe stdout.`, + 0 + ) + } + + await streamToStdout(response.body) + return + } + + const target = + options.outputFile ?? + join( + process.cwd(), + attachmentFileName(response.headers.get('content-disposition')) ?? + `${knowledgeBaseId}.simkb.zip` + ) + + await saveToFile(response.body, target, Boolean(options.force)) + printProtocolResult(profile.output, { + id: knowledgeBaseId, + path: target, + status: 'saved', + vectors: options.vectors, + }) + }) +} diff --git a/packages/sim-cli/src/generated/v2-api.ts b/packages/sim-cli/src/generated/v2-api.ts index 38b40e4ed74..51a4feeb3c0 100644 --- a/packages/sim-cli/src/generated/v2-api.ts +++ b/packages/sim-cli/src/generated/v2-api.ts @@ -3655,6 +3655,19 @@ export type ExecuteWorkflowResponse = data: ExecuteWorkflowResponseRef2 } +/** `GET /api/v2/knowledge/[knowledgeBaseId]/export` */ +export type ExportKnowledgeBaseParams = { + knowledgeBaseId: string +} + +export type ExportKnowledgeBaseQuery = { + workspaceId: string + vectors?: boolean +} + +/** Non-JSON response (`binary`). */ +export type ExportKnowledgeBaseResponse = never + /** `GET /api/v2/workflows/[workflowId]/export` */ export type ExportWorkflowParams = { workflowId: string @@ -11429,6 +11442,26 @@ export const V2_OPERATIONS = { }, }, }, + exportKnowledgeBase: { + method: 'GET', + path: '/api/v2/knowledge/[knowledgeBaseId]/export', + pathParams: ['knowledgeBaseId'] as const, + pathParamDocs: { knowledgeBaseId: 'Unique knowledge base identifier.' }, + responseMode: 'binary', + summary: 'Export Knowledge Base', + query: { + workspaceId: { + kind: 'string', + required: true, + describe: 'Workspace that owns the knowledge base.', + }, + vectors: { + kind: 'boolean', + describe: + 'Include chunk vectors so an import into a deployment with the same embedding model reuses them instead of re-embedding.', + }, + }, + }, exportWorkflow: { method: 'GET', path: '/api/v2/workflows/[workflowId]/export', diff --git a/packages/sim-cli/src/program.ts b/packages/sim-cli/src/program.ts index d83cd6afe79..28eeb00a8be 100644 --- a/packages/sim-cli/src/program.ts +++ b/packages/sim-cli/src/program.ts @@ -37,6 +37,7 @@ Examples: $ sim knowledge search --query "refund policy" --kb 4c1b7f60-2d55-4a3e-9c18-70b6ea2f9d31 $ sim workflows export 3a9e21d8-5f47-4c0b-b2ea-91d7c6034ef8 > wf.json $ sim workflows import --workflow @wf.json + $ sim knowledge export 4c1b7f60-2d55-4a3e-9c18-70b6ea2f9d31 -o ./kb.simkb.zip $ sim whoami --profile dev ` diff --git a/packages/testing/src/mocks/audit.mock.ts b/packages/testing/src/mocks/audit.mock.ts index 5c9083e0228..039f90d63e1 100644 --- a/packages/testing/src/mocks/audit.mock.ts +++ b/packages/testing/src/mocks/audit.mock.ts @@ -102,6 +102,7 @@ export const auditMock = { KNOWLEDGE_BASE_UPDATED: 'knowledge_base.updated', KNOWLEDGE_BASE_DELETED: 'knowledge_base.deleted', KNOWLEDGE_BASE_RESTORED: 'knowledge_base.restored', + KNOWLEDGE_BASE_EXPORTED: 'knowledge_base.exported', MCP_SERVER_ADDED: 'mcp_server.added', MCP_SERVER_UPDATED: 'mcp_server.updated', MCP_SERVER_REMOVED: 'mcp_server.removed', diff --git a/scripts/openapi/documents.test.ts b/scripts/openapi/documents.test.ts index 951dfc55668..6f6a7f382fb 100644 --- a/scripts/openapi/documents.test.ts +++ b/scripts/openapi/documents.test.ts @@ -112,7 +112,7 @@ const EXPECTED_OPERATION_COUNTS = new Map([ ['apps/docs/openapi-v2-logs.json', 3], ['apps/docs/openapi-v2-files-audit.json', 29], ['apps/docs/openapi-v2-tables.json', 53], - ['apps/docs/openapi-v2-knowledge.json', 44], + ['apps/docs/openapi-v2-knowledge.json', 45], ['apps/docs/openapi-v2-billing.json', 2], ['apps/docs/openapi-v2-resources.json', 51], ]) @@ -310,7 +310,7 @@ describe('generated OpenAPI documents', () => { }) } } - expect(totalOperations).toBe(220) + expect(totalOperations).toBe(221) }) it('documents mixed workflow execution and resume responses', () => { From cbf3fb96bd9c1a77ebe8e064a66c7f6ec787b555 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 8 Sep 2026 16:25:07 -0700 Subject: [PATCH 2/6] fix(knowledge): expose authorize on knowledge use cases and gate export on describable bundles - defineAuthorizedKnowledgeUseCase now answers authorize() through the same resolution execute() uses, which headSafe: false routes require at module init; defineAuthorizedWorkspaceUseCase's return type states that authorize is always present - the export use case validates the stored base against the bundle manifest schema before any byte streams, so a value the import side would refuse surfaces as a 409 rather than a truncated archive - Export is hidden in the base menu and header when a permission group withholds knowledge.export - operations id snapshot includes knowledge.export --- .../[workspaceId]/knowledge/[id]/base.tsx | 11 +- .../[workspaceId]/knowledge/knowledge.tsx | 2 +- .../authorized-workspace-use-case.ts | 16 ++- .../authorized-knowledge-use-case.ts | 108 +++++++++++++----- .../lib/knowledge/application/exports.test.ts | 15 +++ apps/sim/lib/knowledge/application/exports.ts | 22 +++- .../knowledge/application/operations.test.ts | 1 + apps/sim/lib/knowledge/transfer/bundle.ts | 31 ++++- .../lib/knowledge/transfer/export-archive.ts | 17 ++- 9 files changed, 172 insertions(+), 51 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx index 9cd9a351589..123316bd61c 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx @@ -131,6 +131,7 @@ import { useDebounce } from '@/hooks/use-debounce' import { useDebouncedSearchSetter } from '@/hooks/use-debounced-search-setter' import { useInlineRename } from '@/hooks/use-inline-rename' import { useOAuthReturnForKBConnectors } from '@/hooks/use-oauth-return' +import { usePermissionConfig } from '@/hooks/use-permission-config' import { useUrlSort } from '@/hooks/use-url-sort' const logger = createLogger('KnowledgeBase') @@ -315,6 +316,7 @@ export function KnowledgeBase({ useOAuthReturnForKBConnectors(id) const userPermissions = useUserPermissionsContext() + const { config: permissionConfig } = usePermissionConfig() const { mutate: updateDocumentMutation, mutateAsync: updateDocumentAsync } = useUpdateDocument() const { mutate: deleteDocumentMutation } = useDeleteDocument() @@ -1011,11 +1013,9 @@ export function KnowledgeBase({ const headerActions: ResourceAction[] = useMemo( () => [ - { - text: 'Export', - icon: Download, - onSelect: () => downloadKnowledgeBaseExport(id), - }, + ...(permissionConfig.disableKnowledgeBaseExport + ? [] + : [{ text: 'Export', icon: Download, onSelect: () => downloadKnowledgeBaseExport(id) }]), ...(userPermissions.canEdit || userPermissions.isLoading ? [ { @@ -1036,6 +1036,7 @@ export function KnowledgeBase({ ], [ id, + permissionConfig.disableKnowledgeBaseExport, userPermissions.canEdit, userPermissions.isLoading, setShowAddConnectorModal, diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx index 6d5c1f8e390..530208ed9ce 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/knowledge.tsx @@ -1545,7 +1545,7 @@ export function Knowledge() { onOpenInNewTab={handleOpenInNewTab} onViewTags={handleViewTags} onCopyId={handleCopyId} - onExport={handleExport} + onExport={permissionConfig.disableKnowledgeBaseExport ? undefined : handleExport} onTogglePin={handleToggleBasePin} pinned={pinnedBaseIds.has(activeKnowledgeBase.id)} onEdit={handleEdit} diff --git a/apps/sim/lib/core/application/authorized-workspace-use-case.ts b/apps/sim/lib/core/application/authorized-workspace-use-case.ts index 1166d082257..f27800f8b48 100644 --- a/apps/sim/lib/core/application/authorized-workspace-use-case.ts +++ b/apps/sim/lib/core/application/authorized-workspace-use-case.ts @@ -122,12 +122,26 @@ export function recordProjectedUseCaseAuditEntries( } } +/** + * A workspace use case always answers `authorize`, so a caller that must run + * the funnel without executing (a `HEAD`, or a wrapping domain builder) can + * rely on it without a runtime guard. + */ +export type AuthorizedWorkspaceUseCase = OperationUseCase< + O, + I, + R +> & + Required, 'authorize'>> + export function defineAuthorizedWorkspaceUseCase< const O extends WorkspaceOperation, I, C extends WorkspaceAuthorizationContext, R, ->(definition: AuthorizedWorkspaceUseCaseDefinition): OperationUseCase { +>( + definition: AuthorizedWorkspaceUseCaseDefinition +): AuthorizedWorkspaceUseCase { const resourceAuthorization = (() => { const { authorizeResource, operation } = definition const resourcePolicy = ('resourcePolicy' in operation ? operation.resourcePolicy : undefined) as diff --git a/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.ts b/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.ts index 4bfb827c6d9..a3947c7f5e0 100644 --- a/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.ts +++ b/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.ts @@ -134,41 +134,89 @@ export function defineAuthorizedKnowledgeUseCase< } ) + /** + * Resolves and authorizes the context every branch below shares, so `authorize` + * and `execute` cannot drift apart. An organization base is fully authorized + * here; a workspace base still owes the workspace funnel, which the returned + * scope tells the caller to run. + */ + async function resolveAuthorizedContext({ + principal, + input, + }: { + principal: Principal + input: I + }): Promise< + | { scope: 'organization'; principal: KnowledgePrincipalForOperation; context: C } + | { + scope: 'workspace' + principal: KnowledgePrincipalForOperation + context: WorkspaceContext + } + > { + requireKnowledgePrincipal(principal, definition.operation) + const context = await definition.resolveContext({ principal, input }) + if (context.organizationId) { + await authorizeOrganizationOperation( + principal, + definition.operation.organizationOperation, + context + ) + return { scope: 'organization', principal, context } + } + if (principal.kind === 'organization_delegated') + throw new OrchestrationError('not_found', 'Knowledge base not found') + assertWorkspaceKnowledgeContext(context) + return { scope: 'workspace', principal, context } + } + + function recordAudit(resultContext: AuthorizedKnowledgeUseCaseResultContext): void { + const projectedAudit = definition.projectAudit?.(resultContext) + if (projectedAudit === undefined) return + const auditEntries = Array.isArray(projectedAudit) ? projectedAudit : [projectedAudit] + if (auditEntries.length === 0) return + const organizationId = resultContext.context.organizationId ?? undefined + recordProjectedUseCaseAuditEntries( + definition.operation, + organizationId ? undefined : resultContext.context.workspaceId, + resultContext.principal, + resultContext.request, + auditEntries, + organizationId + ) + } + return { operation: definition.operation, + async authorize({ principal, input, request }) { + const resolved = await resolveAuthorizedContext({ principal, input }) + if (resolved.scope !== 'workspace') return + await workspaceUseCase.authorize({ + principal: resolved.principal, + input: { originalInput: input, context: resolved.context }, + request, + }) + }, async execute({ principal, input, request }) { - requireKnowledgePrincipal(principal, definition.operation) - const context = await definition.resolveContext({ principal, input }) - if (context.organizationId) { - await authorizeOrganizationOperation( - principal, - definition.operation.organizationOperation, - context - ) - const result = await definition.execute({ principal, input, context, request }) - const resultContext = { principal, input, context, request, result } - const projectedAudit = definition.projectAudit?.(resultContext) - if (projectedAudit !== undefined) { - recordProjectedUseCaseAuditEntries( - definition.operation, - undefined, - principal, - request, - Array.isArray(projectedAudit) ? projectedAudit : [projectedAudit], - context.organizationId - ) - } - await definition.afterSuccess?.(resultContext) - return result + const resolved = await resolveAuthorizedContext({ principal, input }) + if (resolved.scope === 'workspace') { + return workspaceUseCase.execute({ + principal: resolved.principal, + input: { originalInput: input, context: resolved.context }, + request, + }) } - if (principal.kind === 'organization_delegated') - throw new OrchestrationError('not_found', 'Knowledge base not found') - assertWorkspaceKnowledgeContext(context) - return workspaceUseCase.execute({ - principal, - input: { originalInput: input, context }, + const executionContext = { + principal: resolved.principal, + input, + context: resolved.context, request, - }) + } + const result = await definition.execute(executionContext) + const resultContext = { ...executionContext, result } + recordAudit(resultContext) + await definition.afterSuccess?.(resultContext) + return result }, } } diff --git a/apps/sim/lib/knowledge/application/exports.test.ts b/apps/sim/lib/knowledge/application/exports.test.ts index 85a153ebc68..310c71b54d9 100644 --- a/apps/sim/lib/knowledge/application/exports.test.ts +++ b/apps/sim/lib/knowledge/application/exports.test.ts @@ -181,6 +181,21 @@ describe('exportKnowledgeBase', () => { expect(mocks.recordAudit).not.toHaveBeenCalled() }) + /** The manifest is written last, so a value the format rejects must fail before any byte streams. */ + it('refuses a base whose stored values the bundle format cannot describe', async () => { + mocks.listDocuments.mockResolvedValueOnce([ + { ...documents[0], tags: { tag1: 'x'.repeat(10_001) } }, + ]) + + await expect( + exportKnowledgeBase.execute({ + principal, + input: { knowledgeBaseId: 'knowledge-1', vectors: true }, + }) + ).rejects.toMatchObject({ code: 'conflict' }) + expect(mocks.recordAudit).not.toHaveBeenCalled() + }) + it('propagates an oversized base without recording audit', async () => { mocks.listDocuments.mockRejectedValueOnce( new OrchestrationError('payload_too_large', 'Knowledge base has 2001 documents') diff --git a/apps/sim/lib/knowledge/application/exports.ts b/apps/sim/lib/knowledge/application/exports.ts index 57a4bcfee20..b322e5b960b 100644 --- a/apps/sim/lib/knowledge/application/exports.ts +++ b/apps/sim/lib/knowledge/application/exports.ts @@ -3,8 +3,15 @@ import type { Principal } from '@sim/auth/principal' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { resolveActiveKnowledgeBaseContext } from '@/lib/knowledge/application/contexts' import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { KNOWLEDGE_BUNDLE_VERSION } from '@/lib/knowledge/constants' import { toKbEmbeddingDimensions } from '@/lib/knowledge/embedding-models' -import type { KnowledgeBundleManifest, KnowledgeBundleTag } from '@/lib/knowledge/transfer/bundle' +import { + assertDescribableByBundle, + bundleEntryPaths, + type KnowledgeBundleManifest, + type KnowledgeBundleTag, + toManifestDocument, +} from '@/lib/knowledge/transfer/bundle' import { type ExportableChunk, type ExportableDocument, @@ -50,7 +57,7 @@ export const exportKnowledgeBase = defineAuthorizedKnowledgeUseCase({ listExportableTags(knowledgeBase.id), listExportableDocuments(knowledgeBase.id), ]) - return { + const bundle: KnowledgeBaseExportBundle = { knowledgeBase: { name: knowledgeBase.name, description: knowledgeBase.description, @@ -65,6 +72,17 @@ export const exportKnowledgeBase = defineAuthorizedKnowledgeUseCase({ documents, chunks: (documentId) => iterateDocumentChunks(documentId, input.vectors ? dimension : null), } + assertDescribableByBundle({ + version: KNOWLEDGE_BUNDLE_VERSION, + exportedAt: new Date().toISOString(), + embedding: bundle.embedding, + knowledgeBase: bundle.knowledgeBase, + tags, + documents: documents.map((document) => + toManifestDocument(document, bundleEntryPaths(document), 0) + ), + }) + return bundle }, projectAudit: ({ context, input, result }) => ({ action: AuditAction.KNOWLEDGE_BASE_EXPORTED, diff --git a/apps/sim/lib/knowledge/application/operations.test.ts b/apps/sim/lib/knowledge/application/operations.test.ts index fe22040c014..11c189ff10b 100644 --- a/apps/sim/lib/knowledge/application/operations.test.ts +++ b/apps/sim/lib/knowledge/application/operations.test.ts @@ -12,6 +12,7 @@ describe('knowledge operation registry', () => { expect(ids).toEqual([ 'knowledge.list', 'knowledge.read', + 'knowledge.export', 'knowledge.create', 'knowledge.update', 'knowledge.delete', diff --git a/apps/sim/lib/knowledge/transfer/bundle.ts b/apps/sim/lib/knowledge/transfer/bundle.ts index 1503c85de4f..9385896ed25 100644 --- a/apps/sim/lib/knowledge/transfer/bundle.ts +++ b/apps/sim/lib/knowledge/transfer/bundle.ts @@ -1,5 +1,6 @@ import { z } from 'zod' import { chunkingConfigSchema } from '@/lib/api/contracts/knowledge/base' +import { OrchestrationError } from '@/lib/core/orchestration/types' import { KB_EMBEDDING_STORAGE_DIMENSIONS } from '@/lib/embeddings/catalog' import { ALL_TAG_SLOTS, @@ -13,6 +14,7 @@ import { SUPPORTED_FIELD_TYPES, } from '@/lib/knowledge/constants' import { MAX_DOCUMENT_CHUNKS } from '@/lib/knowledge/documents/document-processing-error' +import type { ExportableDocument } from '@/lib/knowledge/transfer/export-source' import { MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE } from '@/lib/uploads/shared/types' import { safeZipLeafName } from '@/lib/uploads/zip-entry-path' @@ -69,7 +71,7 @@ export const knowledgeBundleTagSchema = z message: 'Tag slot does not belong to its field type', }) -const bundleDocumentSchema = z +export const knowledgeBundleDocumentSchema = z .object({ id: z.string().regex(BUNDLE_DOCUMENT_ID_PATTERN, 'Document id must be a short identifier'), filename: z.string().min(1).max(MAX_BUNDLE_TEXT_LENGTH), @@ -151,7 +153,7 @@ export const knowledgeBundleManifestSchema = z names.add(name) } }), - documents: z.array(bundleDocumentSchema).max(MAX_KNOWLEDGE_BUNDLE_DOCUMENTS), + documents: z.array(knowledgeBundleDocumentSchema).max(MAX_KNOWLEDGE_BUNDLE_DOCUMENTS), }) .strict() @@ -207,6 +209,31 @@ export function fileEntryPath(documentId: string, filename: string): string { return `files/${documentId}/${safeBundleLeafName(filename)}` } +/** Where an exportable document's entries sit inside the bundle, or `null` for entries it does not carry. */ +export function bundleEntryPaths( + document: Pick +): KnowledgeBundleEntryPaths { + return { + file: document.file ? fileEntryPath(document.id, document.filename) : null, + chunks: document.hasChunks ? chunksEntryPath(document.id) : null, + } +} + +/** + * Refuses an export whose stored values the bundle format cannot describe, + * before any byte streams: the manifest is written last, so a value the import + * side would reject must surface as a clear error rather than a truncated archive. + */ +export function assertDescribableByBundle(manifest: unknown): void { + const result = knowledgeBundleManifestSchema.safeParse(manifest) + if (result.success) return + const [issue] = result.error.issues + throw new OrchestrationError( + 'conflict', + `Knowledge base cannot be exported: ${issue.path.join('.')} ${issue.message}` + ) +} + /** A filesystem-safe leaf name for a bundle entry or the bundle download itself. */ export function safeBundleLeafName(name: string): string { return safeZipLeafName(name).slice(0, MAX_BUNDLE_LEAF_NAME_LENGTH) diff --git a/apps/sim/lib/knowledge/transfer/export-archive.ts b/apps/sim/lib/knowledge/transfer/export-archive.ts index 97c73b2ae3f..58e15071a3d 100644 --- a/apps/sim/lib/knowledge/transfer/export-archive.ts +++ b/apps/sim/lib/knowledge/transfer/export-archive.ts @@ -7,9 +7,8 @@ import { decodeDataUriWithinLimit } from '@/lib/file-parsers/data-uri' import type { KnowledgeBaseExportBundle } from '@/lib/knowledge/application/exports' import { KNOWLEDGE_BUNDLE_VERSION } from '@/lib/knowledge/constants' import { - chunksEntryPath, + bundleEntryPaths, encodeVectorBase64, - fileEntryPath, KNOWLEDGE_BUNDLE_MANIFEST_ENTRY, type KnowledgeBundleChunkLine, type KnowledgeBundleDocument, @@ -96,21 +95,19 @@ async function appendBundleEntries( ): Promise { const documents: KnowledgeBundleDocument[] = [] for (const document of bundle.documents) { - let file: string | null = null - if (document.file) { - file = fileEntryPath(document.id, document.filename) - await appendEntry(archive, await openFileSource(document.file), file) + const entries = bundleEntryPaths(document) + if (document.file && entries.file) { + await appendEntry(archive, await openFileSource(document.file), entries.file) } - const chunks = document.hasChunks ? chunksEntryPath(document.id) : null - const chunkCount = chunks + const chunkCount = entries.chunks ? await appendChunkEntry( archive, bundle.chunks(document.id), bundle.embedding.vectorsIncluded, - chunks + entries.chunks ) : 0 - documents.push(toManifestDocument(document, { file, chunks }, chunkCount)) + documents.push(toManifestDocument(document, entries, chunkCount)) } const manifest: KnowledgeBundleManifest = { From ccaaafb7c3682c04ee49eba62893c20518de86f6 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 8 Sep 2026 16:31:02 -0700 Subject: [PATCH 3/6] chore(docs): count the knowledge export path in the OpenAPI download test --- apps/docs/lib/openapi-download.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/docs/lib/openapi-download.test.ts b/apps/docs/lib/openapi-download.test.ts index 48b81490491..2bdfb47bf3c 100644 --- a/apps/docs/lib/openapi-download.test.ts +++ b/apps/docs/lib/openapi-download.test.ts @@ -33,7 +33,7 @@ describe('OpenAPI download', () => { const tags = document.tags as Array<{ name: string }> expect(document.openapi).toBe('3.1.0') - expect(Object.keys(paths)).toHaveLength(132) + expect(Object.keys(paths)).toHaveLength(133) expect(tags.map((tag) => tag.name)).toEqual([ 'Workflows', 'Workflow Runs', From 8d5b0ee281ec98a32910cedc332d2c1fc74c2e01 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 8 Sep 2026 17:27:27 -0700 Subject: [PATCH 4/6] improvement(knowledge): read inline document payloads lazily and scope chunk reads to the base Follow-ups from an independent review pass over the export path. - the document listing no longer selects fileUrl: a data: document can hold megabytes in that column, so the listing carries a flag and the archive reads one payload at a time when it reaches that entry - iterateDocumentChunks filters on knowledgeBaseId as well as documentId, so a future caller cannot reach another base's chunks through the bundle - tag definitions are validated by the single bundle gate, which answers 409 like every other undescribable value rather than throwing a bare ZodError - a consumer that abandons the download aborts the append loop and destroys the in-flight source instead of leaving it pending - direct tests for the knowledge use-case builder's authorize() on the workspace, organization, and legacy personal branches --- .../authorized-knowledge-use-case.test.ts | 144 ++++++++++++++++++ .../lib/knowledge/application/exports.test.ts | 17 ++- apps/sim/lib/knowledge/application/exports.ts | 34 ++--- apps/sim/lib/knowledge/transfer/bundle.ts | 11 +- .../knowledge/transfer/export-archive.test.ts | 28 +++- .../lib/knowledge/transfer/export-archive.ts | 52 +++++-- .../lib/knowledge/transfer/export-source.ts | 55 ++++--- 7 files changed, 278 insertions(+), 63 deletions(-) create mode 100644 apps/sim/lib/knowledge/application/authorized-knowledge-use-case.test.ts diff --git a/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.test.ts b/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.test.ts new file mode 100644 index 00000000000..7a523d0623c --- /dev/null +++ b/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.test.ts @@ -0,0 +1,144 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + resolvePermission: vi.fn(), + authorizeOrganization: vi.fn(), + recordAudit: vi.fn(), +})) + +vi.mock('@sim/audit', () => ({ + AuditAction: { KNOWLEDGE_BASE_UPDATED: 'knowledge_base.updated' }, + AuditResourceType: { KNOWLEDGE_BASE: 'knowledge_base' }, + recordAudit: mocks.recordAudit, +})) + +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: (actual: string | null, required: string) => { + const rank = { read: 1, write: 2, admin: 3 } as const + return ( + actual !== null && rank[actual as keyof typeof rank] >= rank[required as keyof typeof rank] + ) + }, + resolveEffectiveWorkspacePermission: mocks.resolvePermission, +})) + +vi.mock('@/lib/core/application/organization-authorization', () => ({ + authorizeOrganizationOperation: mocks.authorizeOrganization, +})) + +import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' + +const workspaceContext = { + workspaceId: 'workspace-1', + workspaceOrganizationId: 'organization-1', + allowPersonalApiKeys: true, + billedAccountUserId: 'owner-1', + knowledgeBaseId: 'knowledge-1', +} +const organizationContext = { + workspaceId: undefined, + organizationId: 'organization-1', + knowledgeBaseId: 'knowledge-1', +} +const legacyContext = { + workspaceId: undefined, + legacyPersonalOwnerUserId: 'user-1', + knowledgeBaseId: 'knowledge-1', +} + +const session = { kind: 'session', userId: 'user-1', sessionId: 'session-1' } as const + +function useCaseFor(context: object) { + const execute = vi.fn(async () => 'done') + const useCase = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.read, + resolveContext: () => context as never, + execute, + projectAudit: () => ({ + action: 'knowledge_base.updated', + resourceType: 'knowledge_base', + resourceId: 'knowledge-1', + resourceName: 'Docs', + description: 'audited', + metadata: {}, + }), + }) + return { useCase, execute } +} + +describe('defineAuthorizedKnowledgeUseCase', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.resolvePermission.mockResolvedValue('read') + mocks.authorizeOrganization.mockResolvedValue(undefined) + }) + + /** `authorize` must run the same funnel `execute` does, and nothing else. */ + it('authorizes a workspace base through the workspace funnel without executing', async () => { + const { useCase, execute } = useCaseFor(workspaceContext) + + await useCase.authorize({ principal: session, input: {} }) + expect(mocks.resolvePermission).toHaveBeenCalledTimes(1) + expect(execute).not.toHaveBeenCalled() + expect(mocks.recordAudit).not.toHaveBeenCalled() + + mocks.resolvePermission.mockResolvedValue(null) + await expect(useCase.authorize({ principal: session, input: {} })).rejects.toMatchObject({ + name: 'NoWorkspaceAccessError', + }) + }) + + it('executes a workspace base and records its audit under the workspace', async () => { + const { useCase } = useCaseFor(workspaceContext) + + await expect(useCase.execute({ principal: session, input: {} })).resolves.toBe('done') + expect(mocks.recordAudit).toHaveBeenCalledWith( + expect.objectContaining({ workspaceId: 'workspace-1', resourceId: 'knowledge-1' }) + ) + }) + + it('authorizes and executes an organization base through the organization operation', async () => { + const { useCase, execute } = useCaseFor(organizationContext) + + await useCase.authorize({ principal: session, input: {} }) + expect(mocks.authorizeOrganization).toHaveBeenCalledWith( + session, + knowledgeOperations.read.organizationOperation, + organizationContext + ) + expect(execute).not.toHaveBeenCalled() + expect(mocks.resolvePermission).not.toHaveBeenCalled() + + await expect(useCase.execute({ principal: session, input: {} })).resolves.toBe('done') + expect(mocks.recordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + workspaceId: undefined, + resourceId: 'knowledge-1', + metadata: expect.objectContaining({ organizationId: 'organization-1' }), + }) + ) + }) + + it('admits only the owner of a legacy personal base', async () => { + const { useCase, execute } = useCaseFor(legacyContext) + + await useCase.authorize({ principal: session, input: {} }) + expect(execute).not.toHaveBeenCalled() + await expect(useCase.execute({ principal: session, input: {} })).resolves.toBe('done') + + const stranger = { kind: 'session', userId: 'user-2', sessionId: 'session-2' } as const + await expect(useCase.authorize({ principal: stranger, input: {} })).rejects.toMatchObject({ + code: 'not_found', + }) + await expect( + useCase.authorize({ + principal: { kind: 'workspace_api_key', workspaceId: 'workspace-1', keyId: 'key-1' }, + input: {}, + }) + ).rejects.toMatchObject({ code: 'not_found' }) + }) +}) diff --git a/apps/sim/lib/knowledge/application/exports.test.ts b/apps/sim/lib/knowledge/application/exports.test.ts index 310c71b54d9..23d39c0b4fd 100644 --- a/apps/sim/lib/knowledge/application/exports.test.ts +++ b/apps/sim/lib/knowledge/application/exports.test.ts @@ -135,14 +135,14 @@ describe('exportKnowledgeBase', () => { input: { knowledgeBaseId: 'knowledge-1', vectors: true }, }) withVectors.chunks('doc-1') - expect(mocks.iterateChunks).toHaveBeenLastCalledWith('doc-1', 1536) + expect(mocks.iterateChunks).toHaveBeenLastCalledWith('knowledge-1', 'doc-1', 1536) const textOnly = await exportKnowledgeBase.execute({ principal, input: { knowledgeBaseId: 'knowledge-1', vectors: false }, }) textOnly.chunks('doc-1') - expect(mocks.iterateChunks).toHaveBeenLastCalledWith('doc-1', null) + expect(mocks.iterateChunks).toHaveBeenLastCalledWith('knowledge-1', 'doc-1', null) expect(textOnly.embedding.vectorsIncluded).toBe(false) }) @@ -196,6 +196,19 @@ describe('exportKnowledgeBase', () => { expect(mocks.recordAudit).not.toHaveBeenCalled() }) + it('refuses a stored tag definition the bundle format cannot describe', async () => { + mocks.listTags.mockResolvedValueOnce([ + { slot: 'tag1', displayName: 'Product', fieldType: 'mystery' }, + ]) + + await expect( + exportKnowledgeBase.execute({ + principal, + input: { knowledgeBaseId: 'knowledge-1', vectors: true }, + }) + ).rejects.toMatchObject({ code: 'conflict' }) + }) + it('propagates an oversized base without recording audit', async () => { mocks.listDocuments.mockRejectedValueOnce( new OrchestrationError('payload_too_large', 'Knowledge base has 2001 documents') diff --git a/apps/sim/lib/knowledge/application/exports.ts b/apps/sim/lib/knowledge/application/exports.ts index b322e5b960b..4f9857c78a7 100644 --- a/apps/sim/lib/knowledge/application/exports.ts +++ b/apps/sim/lib/knowledge/application/exports.ts @@ -6,10 +6,10 @@ import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { KNOWLEDGE_BUNDLE_VERSION } from '@/lib/knowledge/constants' import { toKbEmbeddingDimensions } from '@/lib/knowledge/embedding-models' import { - assertDescribableByBundle, bundleEntryPaths, type KnowledgeBundleManifest, type KnowledgeBundleTag, + parseDescribableBundle, toManifestDocument, } from '@/lib/knowledge/transfer/bundle' import { @@ -57,32 +57,32 @@ export const exportKnowledgeBase = defineAuthorizedKnowledgeUseCase({ listExportableTags(knowledgeBase.id), listExportableDocuments(knowledgeBase.id), ]) - const bundle: KnowledgeBaseExportBundle = { - knowledgeBase: { - name: knowledgeBase.name, - description: knowledgeBase.description, - chunkingConfig: knowledgeBase.chunkingConfig, - }, + const manifest = parseDescribableBundle({ + version: KNOWLEDGE_BUNDLE_VERSION, + exportedAt: new Date().toISOString(), embedding: { model: knowledgeBase.embeddingModel, dimension, vectorsIncluded: input.vectors, }, - tags, - documents, - chunks: (documentId) => iterateDocumentChunks(documentId, input.vectors ? dimension : null), - } - assertDescribableByBundle({ - version: KNOWLEDGE_BUNDLE_VERSION, - exportedAt: new Date().toISOString(), - embedding: bundle.embedding, - knowledgeBase: bundle.knowledgeBase, + knowledgeBase: { + name: knowledgeBase.name, + description: knowledgeBase.description, + chunkingConfig: knowledgeBase.chunkingConfig, + }, tags, documents: documents.map((document) => toManifestDocument(document, bundleEntryPaths(document), 0) ), }) - return bundle + return { + knowledgeBase: manifest.knowledgeBase, + embedding: manifest.embedding, + tags: manifest.tags, + documents, + chunks: (documentId) => + iterateDocumentChunks(knowledgeBase.id, documentId, input.vectors ? dimension : null), + } }, projectAudit: ({ context, input, result }) => ({ action: AuditAction.KNOWLEDGE_BASE_EXPORTED, diff --git a/apps/sim/lib/knowledge/transfer/bundle.ts b/apps/sim/lib/knowledge/transfer/bundle.ts index 9385896ed25..38e9c7b1098 100644 --- a/apps/sim/lib/knowledge/transfer/bundle.ts +++ b/apps/sim/lib/knowledge/transfer/bundle.ts @@ -220,13 +220,14 @@ export function bundleEntryPaths( } /** - * Refuses an export whose stored values the bundle format cannot describe, - * before any byte streams: the manifest is written last, so a value the import - * side would reject must surface as a clear error rather than a truncated archive. + * Validates an export's stored values against the bundle format before any byte + * streams, and returns them in their wire shape. The manifest is written last, + * so a value the import side would reject must surface as a clear error rather + * than a truncated archive. */ -export function assertDescribableByBundle(manifest: unknown): void { +export function parseDescribableBundle(manifest: unknown): KnowledgeBundleManifest { const result = knowledgeBundleManifestSchema.safeParse(manifest) - if (result.success) return + if (result.success) return result.data const [issue] = result.error.issues throw new OrchestrationError( 'conflict', diff --git a/apps/sim/lib/knowledge/transfer/export-archive.test.ts b/apps/sim/lib/knowledge/transfer/export-archive.test.ts index c2712b0332d..d9a90e6fd64 100644 --- a/apps/sim/lib/knowledge/transfer/export-archive.test.ts +++ b/apps/sim/lib/knowledge/transfer/export-archive.test.ts @@ -2,17 +2,23 @@ * @vitest-environment node */ import { Readable } from 'node:stream' +import { sleep } from '@sim/utils/helpers' import JSZip from 'jszip' import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ downloadFileStream: vi.fn(), + readInlineFileUrl: vi.fn(), })) vi.mock('@/lib/uploads/core/storage-service', () => ({ downloadFileStream: mocks.downloadFileStream, })) +vi.mock('@/lib/knowledge/transfer/export-source', () => ({ + readInlineFileUrl: mocks.readInlineFileUrl, +})) + import type { KnowledgeBaseExportBundle } from '@/lib/knowledge/application/exports' import { decodeVectorBase64, knowledgeBundleManifestSchema } from '@/lib/knowledge/transfer/bundle' import { @@ -72,10 +78,7 @@ function bundle(overrides: Partial = {}): KnowledgeBa id: INLINE_ID, filename: 'note.txt', mimeType: 'text/plain', - file: { - kind: 'data-uri', - fileUrl: `data:text/plain;base64,${Buffer.from('hi').toString('base64')}`, - }, + file: { kind: 'data-uri', documentId: INLINE_ID }, hasChunks: false, }), exportableDocument({ id: TEXT_ONLY_ID, filename: 'wiki page', file: null, hasChunks: true }), @@ -98,6 +101,9 @@ describe('buildKnowledgeBundleArchive', () => { beforeEach(() => { vi.clearAllMocks() mocks.downloadFileStream.mockImplementation(async () => Readable.from([Buffer.from('pdf')])) + mocks.readInlineFileUrl.mockResolvedValue( + `data:text/plain;base64,${Buffer.from('hi').toString('base64')}` + ) }) it('writes files, chunk lines, and a manifest that validates against the bundle schema', async () => { @@ -190,6 +196,20 @@ describe('buildKnowledgeBundleArchive', () => { 'entry:manifest.json', ]) }) + + /** A browser that abandons the download must not leave the append loop or its blob stream hanging. */ + it('releases the in-flight source and stops appending when the consumer goes away', async () => { + const blob = new Readable({ read() {} }) + mocks.downloadFileStream.mockResolvedValue(blob) + const archive = buildKnowledgeBundleArchive(bundle()) + await sleep(1) + expect(mocks.downloadFileStream).toHaveBeenCalledTimes(1) + + archive.destroy() + await sleep(1) + expect(blob.destroyed).toBe(true) + expect(mocks.readInlineFileUrl).not.toHaveBeenCalled() + }) }) describe('knowledgeBundleFileName', () => { diff --git a/apps/sim/lib/knowledge/transfer/export-archive.ts b/apps/sim/lib/knowledge/transfer/export-archive.ts index 58e15071a3d..0743d29aee0 100644 --- a/apps/sim/lib/knowledge/transfer/export-archive.ts +++ b/apps/sim/lib/knowledge/transfer/export-archive.ts @@ -16,7 +16,11 @@ import { safeBundleLeafName, toManifestDocument, } from '@/lib/knowledge/transfer/bundle' -import type { ExportableChunk, ExportableFileSource } from '@/lib/knowledge/transfer/export-source' +import { + type ExportableChunk, + type ExportableFileSource, + readInlineFileUrl, +} from '@/lib/knowledge/transfer/export-source' import { downloadFileStream } from '@/lib/uploads/core/storage-service' import { MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE } from '@/lib/uploads/shared/types' @@ -34,7 +38,10 @@ async function openFileSource(source: ExportableFileSource): Promise { - const consumed = once(archive, 'entry') + const consumed = once(archive, 'entry', { signal: closed }) archive.append(source, { name }) - await consumed + try { + await consumed + } catch (error) { + if (source instanceof Readable) source.destroy() + throw error + } } /** Appends a document's chunks as NDJSON and returns how many lines were written. */ @@ -73,7 +88,8 @@ async function appendChunkEntry( archive: ZipArchive, chunks: AsyncIterable, vectors: boolean, - name: string + name: string, + closed: AbortSignal ): Promise { let written = 0 const lines = Readable.from( @@ -85,26 +101,28 @@ async function appendChunkEntry( })(), { objectMode: false } ) - await appendEntry(archive, lines, name) + await appendEntry(archive, lines, name, closed) return written } async function appendBundleEntries( archive: ZipArchive, - bundle: KnowledgeBaseExportBundle + bundle: KnowledgeBaseExportBundle, + closed: AbortSignal ): Promise { const documents: KnowledgeBundleDocument[] = [] for (const document of bundle.documents) { const entries = bundleEntryPaths(document) if (document.file && entries.file) { - await appendEntry(archive, await openFileSource(document.file), entries.file) + await appendEntry(archive, await openFileSource(document.file), entries.file, closed) } const chunkCount = entries.chunks ? await appendChunkEntry( archive, bundle.chunks(document.id), bundle.embedding.vectorsIncluded, - entries.chunks + entries.chunks, + closed ) : 0 documents.push(toManifestDocument(document, entries, chunkCount)) @@ -118,7 +136,12 @@ async function appendBundleEntries( tags: bundle.tags, documents, } - await appendEntry(archive, JSON.stringify(manifest, null, 2), KNOWLEDGE_BUNDLE_MANIFEST_ENTRY) + await appendEntry( + archive, + JSON.stringify(manifest, null, 2), + KNOWLEDGE_BUNDLE_MANIFEST_ENTRY, + closed + ) await archive.finalize() } @@ -135,7 +158,10 @@ export function buildKnowledgeBundleArchive(bundle: KnowledgeBaseExportBundle): archive.on('warning', (error: Error) => { logger.warn('Archive warning while streaming knowledge base bundle', { error }) }) - appendBundleEntries(archive, bundle).catch((error: unknown) => { + const closed = new AbortController() + archive.once('close', () => closed.abort()) + appendBundleEntries(archive, bundle, closed.signal).catch((error: unknown) => { + if (closed.signal.aborted) return logger.error('Failed to build knowledge base bundle archive', { error }) archive.destroy(toError(error)) }) diff --git a/apps/sim/lib/knowledge/transfer/export-source.ts b/apps/sim/lib/knowledge/transfer/export-source.ts index f284d4316a1..7d8c325a677 100644 --- a/apps/sim/lib/knowledge/transfer/export-source.ts +++ b/apps/sim/lib/knowledge/transfer/export-source.ts @@ -7,11 +7,9 @@ import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' import { WORKSPACE_ACCESS_SCOPE } from '@/lib/knowledge/access/scope' import { ALL_TAG_SLOTS, MAX_KNOWLEDGE_BUNDLE_DOCUMENTS } from '@/lib/knowledge/constants' import { getTagDefinitions } from '@/lib/knowledge/tags/service' -import { - type ExportableDocumentRecord, - type KnowledgeBundleChunkLine, - type KnowledgeBundleTag, - knowledgeBundleTagSchema, +import type { + ExportableDocumentRecord, + KnowledgeBundleChunkLine, } from '@/lib/knowledge/transfer/bundle' import { embeddingVectorColumn } from '@/lib/knowledge/vector-columns' @@ -29,7 +27,7 @@ const CHUNK_PAGE_SIZE = { text: 500, vectors: 100 } as const /** Where a document's original bytes come from, when it has any. */ export type ExportableFileSource = | { kind: 'storage'; key: string } - | { kind: 'data-uri'; fileUrl: string } + | { kind: 'data-uri'; documentId: string } export interface ExportableDocument extends ExportableDocumentRecord { file: ExportableFileSource | null @@ -53,29 +51,40 @@ function exportableDocumentCondition(knowledgeBaseId: string) { } function fileSourceFor(row: { + id: string storageKey: string | null - fileUrl: string + hasInlineFile: boolean }): ExportableFileSource | null { if (row.storageKey) return { kind: 'storage', key: row.storageKey } - if (row.fileUrl.startsWith('data:')) return { kind: 'data-uri', fileUrl: row.fileUrl } + if (row.hasInlineFile) return { kind: 'data-uri', documentId: row.id } return null } +/** The base's tag definitions in slot order, as stored; the bundle gate validates them. */ +export async function listExportableTags( + knowledgeBaseId: string +): Promise> { + const definitions = await getTagDefinitions(knowledgeBaseId) + return definitions.map(({ tagSlot, displayName, fieldType }) => ({ + slot: tagSlot, + displayName, + fieldType, + })) +} + /** - * The base's tag definitions in slot order, validated against the bundle's tag - * schema: the stored `tagSlot` column type only names the text slots and - * `fieldType` is free text, so validation is what proves the rows form an - * importable manifest. + * A document's inline `data:` payload, read only when its archive entry is + * reached: the column can hold megabytes per row, so the listing carries a flag + * and the archive fetches one payload at a time. */ -export async function listExportableTags(knowledgeBaseId: string): Promise { - const definitions = await getTagDefinitions(knowledgeBaseId) - return knowledgeBundleTagSchema.array().parse( - definitions.map(({ tagSlot, displayName, fieldType }) => ({ - slot: tagSlot, - displayName, - fieldType, - })) - ) +export async function readInlineFileUrl(documentId: string): Promise { + const [row] = await db + .select({ fileUrl: document.fileUrl }) + .from(document) + .where(eq(document.id, documentId)) + .limit(1) + if (!row) throw new OrchestrationError('not_found', 'Document not found') + return row.fileUrl } /** @@ -95,7 +104,7 @@ export async function listExportableDocuments( fileSize: document.fileSize, enabled: document.enabled, storageKey: document.storageKey, - fileUrl: document.fileUrl, + hasInlineFile: sql`${document.fileUrl} LIKE 'data:%'`, processingStatus: document.processingStatus, chunkCount: document.chunkCount, tokenCount: document.tokenCount, @@ -157,6 +166,7 @@ export async function listExportableDocuments( * alone is the keyset and each page is one index range scan. */ export async function* iterateDocumentChunks( + knowledgeBaseId: string, documentId: string, dimensions: KbEmbeddingDimensions | null ): AsyncGenerator { @@ -176,6 +186,7 @@ export async function* iterateDocumentChunks( .from(embedding) .where( and( + eq(embedding.knowledgeBaseId, knowledgeBaseId), eq(embedding.documentId, documentId), after === null ? undefined : gt(embedding.chunkIndex, after) ) From c332986d66755cf3807c5366cc3cd623b4edbcc1 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 8 Sep 2026 17:31:00 -0700 Subject: [PATCH 5/6] improvement(knowledge): validate stored chunk counts and scope inline reads to the base - the bundle gate checks each document's stored chunk count against the format's per-document ceiling, so a base holding more chunks than a bundle can describe is refused up front rather than written into an invalid manifest - the inline payload read carries its knowledge base id, matching the chunk reads, so neither can reach a row outside the base being exported - an append is refused once the consumer has aborted, since a destroyed archive has no listener left to receive the error it would emit --- .../authorized-knowledge-use-case.test.ts | 24 --------------- .../lib/knowledge/application/exports.test.ts | 14 +++++++++ apps/sim/lib/knowledge/application/exports.ts | 2 +- .../knowledge/transfer/export-archive.test.ts | 3 +- .../lib/knowledge/transfer/export-archive.ts | 10 +++++-- .../lib/knowledge/transfer/export-source.ts | 29 ++++++++++++------- 6 files changed, 44 insertions(+), 38 deletions(-) diff --git a/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.test.ts b/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.test.ts index 7a523d0623c..fae3223f74f 100644 --- a/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.test.ts +++ b/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.test.ts @@ -44,11 +44,6 @@ const organizationContext = { organizationId: 'organization-1', knowledgeBaseId: 'knowledge-1', } -const legacyContext = { - workspaceId: undefined, - legacyPersonalOwnerUserId: 'user-1', - knowledgeBaseId: 'knowledge-1', -} const session = { kind: 'session', userId: 'user-1', sessionId: 'session-1' } as const @@ -122,23 +117,4 @@ describe('defineAuthorizedKnowledgeUseCase', () => { }) ) }) - - it('admits only the owner of a legacy personal base', async () => { - const { useCase, execute } = useCaseFor(legacyContext) - - await useCase.authorize({ principal: session, input: {} }) - expect(execute).not.toHaveBeenCalled() - await expect(useCase.execute({ principal: session, input: {} })).resolves.toBe('done') - - const stranger = { kind: 'session', userId: 'user-2', sessionId: 'session-2' } as const - await expect(useCase.authorize({ principal: stranger, input: {} })).rejects.toMatchObject({ - code: 'not_found', - }) - await expect( - useCase.authorize({ - principal: { kind: 'workspace_api_key', workspaceId: 'workspace-1', keyId: 'key-1' }, - input: {}, - }) - ).rejects.toMatchObject({ code: 'not_found' }) - }) }) diff --git a/apps/sim/lib/knowledge/application/exports.test.ts b/apps/sim/lib/knowledge/application/exports.test.ts index 23d39c0b4fd..d4dbd827338 100644 --- a/apps/sim/lib/knowledge/application/exports.test.ts +++ b/apps/sim/lib/knowledge/application/exports.test.ts @@ -85,6 +85,7 @@ const documents = [ characterCount: 40, tags: {}, file: { kind: 'storage', key: 'kb/handbook.pdf' }, + storedChunkCount: 2, hasChunks: true, }, ] @@ -196,6 +197,19 @@ describe('exportKnowledgeBase', () => { expect(mocks.recordAudit).not.toHaveBeenCalled() }) + /** The written manifest carries the streamed count, so the gate must check the stored one. */ + it('refuses a document holding more chunks than the bundle format describes', async () => { + mocks.listDocuments.mockResolvedValueOnce([{ ...documents[0], storedChunkCount: 5_001 }]) + + await expect( + exportKnowledgeBase.execute({ + principal, + input: { knowledgeBaseId: 'knowledge-1', vectors: true }, + }) + ).rejects.toMatchObject({ code: 'conflict' }) + expect(mocks.recordAudit).not.toHaveBeenCalled() + }) + it('refuses a stored tag definition the bundle format cannot describe', async () => { mocks.listTags.mockResolvedValueOnce([ { slot: 'tag1', displayName: 'Product', fieldType: 'mystery' }, diff --git a/apps/sim/lib/knowledge/application/exports.ts b/apps/sim/lib/knowledge/application/exports.ts index 4f9857c78a7..646374f806c 100644 --- a/apps/sim/lib/knowledge/application/exports.ts +++ b/apps/sim/lib/knowledge/application/exports.ts @@ -72,7 +72,7 @@ export const exportKnowledgeBase = defineAuthorizedKnowledgeUseCase({ }, tags, documents: documents.map((document) => - toManifestDocument(document, bundleEntryPaths(document), 0) + toManifestDocument(document, bundleEntryPaths(document), document.storedChunkCount) ), }) return { diff --git a/apps/sim/lib/knowledge/transfer/export-archive.test.ts b/apps/sim/lib/knowledge/transfer/export-archive.test.ts index d9a90e6fd64..1b32dc31dac 100644 --- a/apps/sim/lib/knowledge/transfer/export-archive.test.ts +++ b/apps/sim/lib/knowledge/transfer/export-archive.test.ts @@ -42,6 +42,7 @@ function exportableDocument(overrides: Partial): ExportableD characterCount: 40, tags: { tag1: 'Billing' }, file: { kind: 'storage', key: 'kb/handbook.pdf' }, + storedChunkCount: 2, hasChunks: true, ...overrides, } @@ -78,7 +79,7 @@ function bundle(overrides: Partial = {}): KnowledgeBa id: INLINE_ID, filename: 'note.txt', mimeType: 'text/plain', - file: { kind: 'data-uri', documentId: INLINE_ID }, + file: { kind: 'data-uri', knowledgeBaseId: 'kb-1', documentId: INLINE_ID }, hasChunks: false, }), exportableDocument({ id: TEXT_ONLY_ID, filename: 'wiki page', file: null, hasChunks: true }), diff --git a/apps/sim/lib/knowledge/transfer/export-archive.ts b/apps/sim/lib/knowledge/transfer/export-archive.ts index 0743d29aee0..10767347d82 100644 --- a/apps/sim/lib/knowledge/transfer/export-archive.ts +++ b/apps/sim/lib/knowledge/transfer/export-archive.ts @@ -39,7 +39,7 @@ async function openFileSource(source: ExportableFileSource): Promise { + if (closed.aborted) { + if (source instanceof Readable) source.destroy() + closed.throwIfAborted() + } const consumed = once(archive, 'entry', { signal: closed }) archive.append(source, { name }) try { diff --git a/apps/sim/lib/knowledge/transfer/export-source.ts b/apps/sim/lib/knowledge/transfer/export-source.ts index 7d8c325a677..925d33b9ec1 100644 --- a/apps/sim/lib/knowledge/transfer/export-source.ts +++ b/apps/sim/lib/knowledge/transfer/export-source.ts @@ -27,10 +27,16 @@ const CHUNK_PAGE_SIZE = { text: 500, vectors: 100 } as const /** Where a document's original bytes come from, when it has any. */ export type ExportableFileSource = | { kind: 'storage'; key: string } - | { kind: 'data-uri'; documentId: string } + | { kind: 'data-uri'; knowledgeBaseId: string; documentId: string } export interface ExportableDocument extends ExportableDocumentRecord { file: ExportableFileSource | null + /** + * Chunks the document reports holding. The archive writes what its chunk + * stream actually produced, which can only be lower; this is what the bundle + * gate checks against the format's per-document ceiling before any byte streams. + */ + storedChunkCount: number /** True when the document finished processing and holds chunks worth exporting. */ hasChunks: boolean } @@ -50,13 +56,12 @@ function exportableDocumentCondition(knowledgeBaseId: string) { ) } -function fileSourceFor(row: { - id: string - storageKey: string | null - hasInlineFile: boolean -}): ExportableFileSource | null { +function fileSourceFor( + knowledgeBaseId: string, + row: { id: string; storageKey: string | null; hasInlineFile: boolean } +): ExportableFileSource | null { if (row.storageKey) return { kind: 'storage', key: row.storageKey } - if (row.hasInlineFile) return { kind: 'data-uri', documentId: row.id } + if (row.hasInlineFile) return { kind: 'data-uri', knowledgeBaseId, documentId: row.id } return null } @@ -77,11 +82,14 @@ export async function listExportableTags( * reached: the column can hold megabytes per row, so the listing carries a flag * and the archive fetches one payload at a time. */ -export async function readInlineFileUrl(documentId: string): Promise { +export async function readInlineFileUrl( + knowledgeBaseId: string, + documentId: string +): Promise { const [row] = await db .select({ fileUrl: document.fileUrl }) .from(document) - .where(eq(document.id, documentId)) + .where(and(eq(document.knowledgeBaseId, knowledgeBaseId), eq(document.id, documentId))) .limit(1) if (!row) throw new OrchestrationError('not_found', 'Document not found') return row.fileUrl @@ -140,7 +148,7 @@ export async function listExportableDocuments( const documents: ExportableDocument[] = [] for (const row of rows) { - const file = fileSourceFor(row) + const file = fileSourceFor(knowledgeBaseId, row) const hasChunks = row.processingStatus === 'completed' && row.chunkCount > 0 if (!file && !hasChunks) continue documents.push({ @@ -152,6 +160,7 @@ export async function listExportableDocuments( tokenCount: row.tokenCount, characterCount: row.characterCount, file, + storedChunkCount: row.chunkCount, hasChunks, tags: Object.fromEntries(ALL_TAG_SLOTS.map((slot) => [slot, row[slot]])), }) From 60d7c7152668423f8f0e791a2210e6156b56903c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 8 Sep 2026 19:29:00 -0700 Subject: [PATCH 6/6] improvement(knowledge): bound the streamed chunks and validate the written manifest Findings from a reuse, simplification, efficiency, and altitude review. - the chunk stream stops at the per-document ceiling and raises afterwards, and the manifest that actually ships is the one validated: chunkCount is denormalized, so a document re-chunked mid-export could previously pass the pre-flight gate and land in a manifest no importer would accept - a real archive failure is no longer mistaken for the consumer walking away; archiver emits close for both, so only the abort's own error is swallowed - the inline-file probe reads a 5-byte prefix instead of matching the whole column, which was detoasting every document's fileUrl and defeating the optimization the listing exists for - storedChunkCount replaces the redundant hasChunks pair - the CLI reads the RFC 5987 filename first, so a knowledge base named "Suporte tecnico" no longer saves under a mangled ASCII name - drops three bundle ceilings the import half will introduce, breaks a circular type import, and states the authorize guarantee in the type so a headSafe route is proven at compile time rather than at module load --- .../knowledge-base-context-menu.tsx | 1 - .../authorized-workspace-use-case.ts | 16 ++---- apps/sim/lib/core/application/index.ts | 1 + .../authorized-knowledge-use-case.ts | 31 ++++++++--- .../lib/knowledge/application/exports.test.ts | 1 - apps/sim/lib/knowledge/constants.ts | 6 --- apps/sim/lib/knowledge/transfer/bundle.ts | 52 ++++++++++--------- .../knowledge/transfer/export-archive.test.ts | 25 +++++++-- .../lib/knowledge/transfer/export-archive.ts | 45 ++++++++++++++-- .../lib/knowledge/transfer/export-source.ts | 44 +++++++++------- .../protocol/knowledge-export.test.ts | 9 ++++ .../src/commands/protocol/knowledge-export.ts | 32 ++++++++---- 12 files changed, 176 insertions(+), 87 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/components/knowledge-base-context-menu/knowledge-base-context-menu.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/components/knowledge-base-context-menu/knowledge-base-context-menu.tsx index dfbf4e37995..b8ec8a6ba92 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/components/knowledge-base-context-menu/knowledge-base-context-menu.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/components/knowledge-base-context-menu/knowledge-base-context-menu.tsx @@ -52,7 +52,6 @@ interface KnowledgeBaseContextMenuProps { /** * Context menu component for knowledge base cards. - * Displays open in new tab, view tags, export, edit, and delete options. */ export const KnowledgeBaseContextMenu = memo(function KnowledgeBaseContextMenu({ isOpen, diff --git a/apps/sim/lib/core/application/authorized-workspace-use-case.ts b/apps/sim/lib/core/application/authorized-workspace-use-case.ts index f27800f8b48..14c4aba6102 100644 --- a/apps/sim/lib/core/application/authorized-workspace-use-case.ts +++ b/apps/sim/lib/core/application/authorized-workspace-use-case.ts @@ -123,15 +123,11 @@ export function recordProjectedUseCaseAuditEntries( } /** - * A workspace use case always answers `authorize`, so a caller that must run - * the funnel without executing (a `HEAD`, or a wrapping domain builder) can - * rely on it without a runtime guard. + * A use case that always answers `authorize`, so a caller that must run the + * funnel without executing — a `HEAD` on a route declaring `headSafe: false`, + * or a wrapping domain builder — can rely on it without a runtime guard. */ -export type AuthorizedWorkspaceUseCase = OperationUseCase< - O, - I, - R -> & +export type AuthorizingUseCase = OperationUseCase & Required, 'authorize'>> export function defineAuthorizedWorkspaceUseCase< @@ -139,9 +135,7 @@ export function defineAuthorizedWorkspaceUseCase< I, C extends WorkspaceAuthorizationContext, R, ->( - definition: AuthorizedWorkspaceUseCaseDefinition -): AuthorizedWorkspaceUseCase { +>(definition: AuthorizedWorkspaceUseCaseDefinition): AuthorizingUseCase { const resourceAuthorization = (() => { const { authorizeResource, operation } = definition const resourcePolicy = ('resourcePolicy' in operation ? operation.resourcePolicy : undefined) as diff --git a/apps/sim/lib/core/application/index.ts b/apps/sim/lib/core/application/index.ts index 74b577ee6f0..48215ea637a 100644 --- a/apps/sim/lib/core/application/index.ts +++ b/apps/sim/lib/core/application/index.ts @@ -4,6 +4,7 @@ export { type AuthorizedWorkspaceUseCaseContext, type AuthorizedWorkspaceUseCaseDefinition, type AuthorizedWorkspaceUseCaseResultContext, + type AuthorizingUseCase, defineAuthorizedWorkspaceUseCase, recordProjectedUseCaseAuditEntries, type WorkspaceUseCaseAuditEntry, diff --git a/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.ts b/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.ts index a3947c7f5e0..869b06f5156 100644 --- a/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.ts +++ b/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.ts @@ -1,7 +1,7 @@ import type { OrganizationDelegatedPrincipal, Principal } from '@sim/auth/principal' import { + type AuthorizingUseCase, defineAuthorizedWorkspaceUseCase, - type OperationUseCase, type PrincipalForOperation, recordProjectedUseCaseAuditEntries, requireAllowedWorkspacePrincipal, @@ -89,7 +89,7 @@ export function defineAuthorizedKnowledgeUseCase< I, C extends KnowledgeResourceAuthorizationContext, R, ->(definition: AuthorizedKnowledgeUseCaseDefinition): OperationUseCase { +>(definition: AuthorizedKnowledgeUseCaseDefinition): AuthorizingUseCase { type WorkspaceContext = C & KnowledgeAuthorizationContext type WorkspaceInput = { originalInput: I; context: WorkspaceContext } const projectAudit = definition.projectAudit @@ -147,7 +147,11 @@ export function defineAuthorizedKnowledgeUseCase< principal: Principal input: I }): Promise< - | { scope: 'organization'; principal: KnowledgePrincipalForOperation; context: C } + | { + scope: 'organization' + principal: KnowledgePrincipalForOperation + context: C & { organizationId: string } + } | { scope: 'workspace' principal: KnowledgePrincipalForOperation @@ -162,7 +166,11 @@ export function defineAuthorizedKnowledgeUseCase< definition.operation.organizationOperation, context ) - return { scope: 'organization', principal, context } + return { + scope: 'organization', + principal, + context: context as C & { organizationId: string }, + } } if (principal.kind === 'organization_delegated') throw new OrchestrationError('not_found', 'Knowledge base not found') @@ -170,15 +178,22 @@ export function defineAuthorizedKnowledgeUseCase< return { scope: 'workspace', principal, context } } - function recordAudit(resultContext: AuthorizedKnowledgeUseCaseResultContext): void { + /** + * Records an organization base's audit. A workspace base never reaches here — + * {@link defineAuthorizedWorkspaceUseCase} records its own — so the entries + * carry the organization rather than a workspace id. + */ + function recordOrganizationAudit( + resultContext: AuthorizedKnowledgeUseCaseResultContext, + organizationId: string + ): void { const projectedAudit = definition.projectAudit?.(resultContext) if (projectedAudit === undefined) return const auditEntries = Array.isArray(projectedAudit) ? projectedAudit : [projectedAudit] if (auditEntries.length === 0) return - const organizationId = resultContext.context.organizationId ?? undefined recordProjectedUseCaseAuditEntries( definition.operation, - organizationId ? undefined : resultContext.context.workspaceId, + undefined, resultContext.principal, resultContext.request, auditEntries, @@ -214,7 +229,7 @@ export function defineAuthorizedKnowledgeUseCase< } const result = await definition.execute(executionContext) const resultContext = { ...executionContext, result } - recordAudit(resultContext) + recordOrganizationAudit(resultContext, resolved.context.organizationId) await definition.afterSuccess?.(resultContext) return result }, diff --git a/apps/sim/lib/knowledge/application/exports.test.ts b/apps/sim/lib/knowledge/application/exports.test.ts index d4dbd827338..074aaa6e9a4 100644 --- a/apps/sim/lib/knowledge/application/exports.test.ts +++ b/apps/sim/lib/knowledge/application/exports.test.ts @@ -86,7 +86,6 @@ const documents = [ tags: {}, file: { kind: 'storage', key: 'kb/handbook.pdf' }, storedChunkCount: 2, - hasChunks: true, }, ] diff --git a/apps/sim/lib/knowledge/constants.ts b/apps/sim/lib/knowledge/constants.ts index 8d1933e4c1f..1703f00b74f 100644 --- a/apps/sim/lib/knowledge/constants.ts +++ b/apps/sim/lib/knowledge/constants.ts @@ -193,14 +193,8 @@ export const KNOWLEDGE_DOCUMENT_PROCESSING_STALE_THRESHOLD_MS = 45 * 60 * 1000 export const KNOWLEDGE_BUNDLE_VERSION = 1 /** Documents one export bundle may carry, so every produced bundle stays importable. */ export const MAX_KNOWLEDGE_BUNDLE_DOCUMENTS = 2_000 -/** Upper bound for one uploaded bundle archive. */ -export const MAX_KNOWLEDGE_BUNDLE_BYTES = 2 * 1024 ** 3 -/** Upper bound for the manifest entry of one bundle. */ -export const MAX_KNOWLEDGE_BUNDLE_MANIFEST_BYTES = 8 * 1024 ** 2 /** * Characters one exported chunk may hold. Wider than the manual-chunk API cap * because the processor's largest chunking config emits chunks past 10k. */ export const MAX_KNOWLEDGE_BUNDLE_CHUNK_CONTENT_LENGTH = 100_000 -/** Bytes one chunk line may span: the content above plus a base64 3072-wide vector. */ -export const MAX_KNOWLEDGE_BUNDLE_CHUNK_LINE_BYTES = 512 * 1024 diff --git a/apps/sim/lib/knowledge/transfer/bundle.ts b/apps/sim/lib/knowledge/transfer/bundle.ts index 38e9c7b1098..cf03fa43c4a 100644 --- a/apps/sim/lib/knowledge/transfer/bundle.ts +++ b/apps/sim/lib/knowledge/transfer/bundle.ts @@ -1,23 +1,3 @@ -import { z } from 'zod' -import { chunkingConfigSchema } from '@/lib/api/contracts/knowledge/base' -import { OrchestrationError } from '@/lib/core/orchestration/types' -import { KB_EMBEDDING_STORAGE_DIMENSIONS } from '@/lib/embeddings/catalog' -import { - ALL_TAG_SLOTS, - type AllTagSlot, - isValidSlotForFieldType, - KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH, - KNOWLEDGE_BUNDLE_VERSION, - KNOWLEDGE_TAG_DISPLAY_NAME_MAX_LENGTH, - MAX_KNOWLEDGE_BUNDLE_CHUNK_CONTENT_LENGTH, - MAX_KNOWLEDGE_BUNDLE_DOCUMENTS, - SUPPORTED_FIELD_TYPES, -} from '@/lib/knowledge/constants' -import { MAX_DOCUMENT_CHUNKS } from '@/lib/knowledge/documents/document-processing-error' -import type { ExportableDocument } from '@/lib/knowledge/transfer/export-source' -import { MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE } from '@/lib/uploads/shared/types' -import { safeZipLeafName } from '@/lib/uploads/zip-entry-path' - /** * The knowledge-base bundle: one zip holding a knowledge base's configuration, * tag definitions, original files, chunk text, and optionally chunk vectors. @@ -41,11 +21,30 @@ import { safeZipLeafName } from '@/lib/uploads/zip-entry-path' * exists, and {@link MAX_BUNDLE_TEXT_LENGTH} bounds the rest. */ +import { z } from 'zod' +import { chunkingConfigSchema } from '@/lib/api/contracts/knowledge/base' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { KB_EMBEDDING_STORAGE_DIMENSIONS } from '@/lib/embeddings/catalog' +import { + ALL_TAG_SLOTS, + type AllTagSlot, + isValidSlotForFieldType, + KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH, + KNOWLEDGE_BUNDLE_VERSION, + KNOWLEDGE_TAG_DISPLAY_NAME_MAX_LENGTH, + MAX_KNOWLEDGE_BUNDLE_CHUNK_CONTENT_LENGTH, + MAX_KNOWLEDGE_BUNDLE_DOCUMENTS, + SUPPORTED_FIELD_TYPES, +} from '@/lib/knowledge/constants' +import { MAX_DOCUMENT_CHUNKS } from '@/lib/knowledge/documents/document-processing-error' +import { MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE } from '@/lib/uploads/shared/types' +import { safeZipLeafName } from '@/lib/uploads/zip-entry-path' + export const KNOWLEDGE_BUNDLE_MANIFEST_ENTRY = 'manifest.json' const BUNDLE_DOCUMENT_ID_PATTERN = /^[A-Za-z0-9_-]{1,64}$/ -/** Longest leaf name written under `files/`, keeping every entry path short. */ +/** Longest leaf name a bundle uses, for an entry path or the download name. */ const MAX_BUNDLE_LEAF_NAME_LENGTH = 200 /** Ceiling for names, MIME types, and the embedding model id, which no stored column bounds. */ @@ -210,12 +209,15 @@ export function fileEntryPath(documentId: string, filename: string): string { } /** Where an exportable document's entries sit inside the bundle, or `null` for entries it does not carry. */ -export function bundleEntryPaths( - document: Pick -): KnowledgeBundleEntryPaths { +export function bundleEntryPaths(document: { + id: string + filename: string + file: unknown | null + storedChunkCount: number +}): KnowledgeBundleEntryPaths { return { file: document.file ? fileEntryPath(document.id, document.filename) : null, - chunks: document.hasChunks ? chunksEntryPath(document.id) : null, + chunks: document.storedChunkCount > 0 ? chunksEntryPath(document.id) : null, } } diff --git a/apps/sim/lib/knowledge/transfer/export-archive.test.ts b/apps/sim/lib/knowledge/transfer/export-archive.test.ts index 1b32dc31dac..002b2b68f72 100644 --- a/apps/sim/lib/knowledge/transfer/export-archive.test.ts +++ b/apps/sim/lib/knowledge/transfer/export-archive.test.ts @@ -20,6 +20,7 @@ vi.mock('@/lib/knowledge/transfer/export-source', () => ({ })) import type { KnowledgeBaseExportBundle } from '@/lib/knowledge/application/exports' +import { MAX_DOCUMENT_CHUNKS } from '@/lib/knowledge/documents/document-processing-error' import { decodeVectorBase64, knowledgeBundleManifestSchema } from '@/lib/knowledge/transfer/bundle' import { buildKnowledgeBundleArchive, @@ -43,7 +44,6 @@ function exportableDocument(overrides: Partial): ExportableD tags: { tag1: 'Billing' }, file: { kind: 'storage', key: 'kb/handbook.pdf' }, storedChunkCount: 2, - hasChunks: true, ...overrides, } } @@ -80,9 +80,9 @@ function bundle(overrides: Partial = {}): KnowledgeBa filename: 'note.txt', mimeType: 'text/plain', file: { kind: 'data-uri', knowledgeBaseId: 'kb-1', documentId: INLINE_ID }, - hasChunks: false, + storedChunkCount: 0, }), - exportableDocument({ id: TEXT_ONLY_ID, filename: 'wiki page', file: null, hasChunks: true }), + exportableDocument({ id: TEXT_ONLY_ID, filename: 'wiki page', file: null }), ], chunks: (documentId) => documentId === STORED_ID @@ -198,6 +198,25 @@ describe('buildKnowledgeBundleArchive', () => { ]) }) + /** + * `document.chunkCount` is denormalized, so the pre-flight gate can approve a + * document that has since grown past what a bundle describes. The archive is + * the last place that can refuse it. + */ + it('refuses a document whose chunk stream exceeds what the format describes', async () => { + const overLimit = (async function* () { + for (let index = 0; index <= MAX_DOCUMENT_CHUNKS; index += 1) yield chunk(index, null) + })() + const archive = buildKnowledgeBundleArchive( + bundle({ + documents: [exportableDocument({ id: TEXT_ONLY_ID, file: null })], + chunks: () => overLimit, + }) + ) + + await expect(readArchive(archive)).rejects.toThrow(`more than ${MAX_DOCUMENT_CHUNKS} chunks`) + }) + /** A browser that abandons the download must not leave the append loop or its blob stream hanging. */ it('releases the in-flight source and stops appending when the consumer goes away', async () => { const blob = new Readable({ read() {} }) diff --git a/apps/sim/lib/knowledge/transfer/export-archive.ts b/apps/sim/lib/knowledge/transfer/export-archive.ts index 10767347d82..d9756ea3c25 100644 --- a/apps/sim/lib/knowledge/transfer/export-archive.ts +++ b/apps/sim/lib/knowledge/transfer/export-archive.ts @@ -3,16 +3,18 @@ import { Readable } from 'node:stream' import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { ZipArchive } from 'archiver' +import { OrchestrationError } from '@/lib/core/orchestration/types' import { decodeDataUriWithinLimit } from '@/lib/file-parsers/data-uri' import type { KnowledgeBaseExportBundle } from '@/lib/knowledge/application/exports' import { KNOWLEDGE_BUNDLE_VERSION } from '@/lib/knowledge/constants' +import { MAX_DOCUMENT_CHUNKS } from '@/lib/knowledge/documents/document-processing-error' import { bundleEntryPaths, encodeVectorBase64, KNOWLEDGE_BUNDLE_MANIFEST_ENTRY, type KnowledgeBundleChunkLine, type KnowledgeBundleDocument, - type KnowledgeBundleManifest, + parseDescribableBundle, safeBundleLeafName, toManifestDocument, } from '@/lib/knowledge/transfer/bundle' @@ -89,7 +91,17 @@ async function appendEntry( } } -/** Appends a document's chunks as NDJSON and returns how many lines were written. */ +/** + * Appends a document's chunks as NDJSON and returns how many lines were written. + * + * The stream is bounded by the same per-document ceiling the bundle format + * declares: `document.chunkCount` is denormalized, so a document re-chunked + * while the export runs can hold more rows than its counter claimed and the + * pre-flight gate approved. The stream stops at the ceiling and the failure is + * raised afterwards rather than thrown into the generator, because a source + * that rejects mid-pipe surfaces as an unhandled stream error instead of + * reaching the caller. + */ async function appendChunkEntry( archive: ZipArchive, chunks: AsyncIterable, @@ -98,9 +110,14 @@ async function appendChunkEntry( closed: AbortSignal ): Promise { let written = 0 + let exceeded = false const lines = Readable.from( (async function* () { for await (const chunk of chunks) { + if (written >= MAX_DOCUMENT_CHUNKS) { + exceeded = true + return + } yield `${JSON.stringify(toChunkLine(chunk, vectors))}\n` written += 1 } @@ -108,6 +125,12 @@ async function appendChunkEntry( { objectMode: false } ) await appendEntry(archive, lines, name, closed) + if (exceeded) { + throw new OrchestrationError( + 'conflict', + `A document holds more than ${MAX_DOCUMENT_CHUNKS} chunks, the most a bundle describes` + ) + } return written } @@ -134,14 +157,20 @@ async function appendBundleEntries( documents.push(toManifestDocument(document, entries, chunkCount)) } - const manifest: KnowledgeBundleManifest = { + /** + * The written manifest, not a manifest shaped like it: the counts here come + * from what each chunk stream produced, so this is the only validation that + * covers the artifact a reader receives. A throw destroys the archive, and a + * truncated download is detectable where an invalid manifest is not. + */ + const manifest = parseDescribableBundle({ version: KNOWLEDGE_BUNDLE_VERSION, exportedAt: new Date().toISOString(), embedding: bundle.embedding, knowledgeBase: bundle.knowledgeBase, tags: bundle.tags, documents, - } + }) await appendEntry( archive, JSON.stringify(manifest, null, 2), @@ -167,7 +196,13 @@ export function buildKnowledgeBundleArchive(bundle: KnowledgeBaseExportBundle): const closed = new AbortController() archive.once('close', () => closed.abort()) appendBundleEntries(archive, bundle, closed.signal).catch((error: unknown) => { - if (closed.signal.aborted) return + /** + * Archiver emits `close` when it fails as well as when the consumer walks + * away, so the signal alone cannot tell the two apart. Only the abort's own + * error means nobody is listening; anything else is a failure the consumer + * must still see, and swallowing it would hand them a silently truncated archive. + */ + if (toError(error).name === 'AbortError') return logger.error('Failed to build knowledge base bundle archive', { error }) archive.destroy(toError(error)) }) diff --git a/apps/sim/lib/knowledge/transfer/export-source.ts b/apps/sim/lib/knowledge/transfer/export-source.ts index 925d33b9ec1..c2e5ca0844a 100644 --- a/apps/sim/lib/knowledge/transfer/export-source.ts +++ b/apps/sim/lib/knowledge/transfer/export-source.ts @@ -1,3 +1,11 @@ +/** + * Read side of a knowledge-base export. Chunk reads page by keyset so a large + * document never materializes at once, and the document listing carries + * {@link knowledgeAccessCondition} for the plain workspace scope: a bundle + * drops access-control lists, so only what every workspace member can already + * read may leave. Everything downstream reads by the ids that listing returned. + */ + import { db } from '@sim/db' import { document, embedding } from '@sim/db/schema' import { and, asc, eq, gt, isNull, sql } from 'drizzle-orm' @@ -5,7 +13,11 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' import type { KbEmbeddingDimensions } from '@/lib/embeddings/catalog' import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' import { WORKSPACE_ACCESS_SCOPE } from '@/lib/knowledge/access/scope' -import { ALL_TAG_SLOTS, MAX_KNOWLEDGE_BUNDLE_DOCUMENTS } from '@/lib/knowledge/constants' +import { + ALL_TAG_SLOTS, + MAX_KNOWLEDGE_BUNDLE_CHUNK_CONTENT_LENGTH, + MAX_KNOWLEDGE_BUNDLE_DOCUMENTS, +} from '@/lib/knowledge/constants' import { getTagDefinitions } from '@/lib/knowledge/tags/service' import type { ExportableDocumentRecord, @@ -14,14 +26,12 @@ import type { import { embeddingVectorColumn } from '@/lib/knowledge/vector-columns' /** - * Read side of a knowledge-base export. Chunk reads page by keyset so a large - * document never materializes at once, and every document read carries - * {@link knowledgeAccessCondition} for the plain workspace scope: a bundle - * drops access-control lists, so only what every workspace member can already - * read may leave. + * Chunk rows per page, sized for chunks of ordinary width. A vector page + * carries the widest column pgvector holds, so it pages far smaller than a + * text-only one. Content is bounded by + * {@link MAX_KNOWLEDGE_BUNDLE_CHUNK_CONTENT_LENGTH} rather than by these counts, + * so a base of unusually wide chunks pages heavier than the numbers suggest. */ - -/** Chunk rows per page. A 3072-wide vector page is ~15 MB on the wire, so vector reads page smaller. */ const CHUNK_PAGE_SIZE = { text: 500, vectors: 100 } as const /** Where a document's original bytes come from, when it has any. */ @@ -32,13 +42,12 @@ export type ExportableFileSource = export interface ExportableDocument extends ExportableDocumentRecord { file: ExportableFileSource | null /** - * Chunks the document reports holding. The archive writes what its chunk - * stream actually produced, which can only be lower; this is what the bundle - * gate checks against the format's per-document ceiling before any byte streams. + * Chunks this document contributes, from its denormalized counter, and zero + * unless processing finished. The bundle gate checks it against the format's + * per-document ceiling before any byte streams; the archive writes what its + * chunk stream actually produced, which the archive bounds again. */ storedChunkCount: number - /** True when the document finished processing and holds chunks worth exporting. */ - hasChunks: boolean } /** A chunk as stored, before its vector is encoded for the wire. */ @@ -112,7 +121,7 @@ export async function listExportableDocuments( fileSize: document.fileSize, enabled: document.enabled, storageKey: document.storageKey, - hasInlineFile: sql`${document.fileUrl} LIKE 'data:%'`, + hasInlineFile: sql`left(${document.fileUrl}, 5) = 'data:'`, processingStatus: document.processingStatus, chunkCount: document.chunkCount, tokenCount: document.tokenCount, @@ -149,8 +158,8 @@ export async function listExportableDocuments( const documents: ExportableDocument[] = [] for (const row of rows) { const file = fileSourceFor(knowledgeBaseId, row) - const hasChunks = row.processingStatus === 'completed' && row.chunkCount > 0 - if (!file && !hasChunks) continue + const storedChunkCount = row.processingStatus === 'completed' ? row.chunkCount : 0 + if (!file && storedChunkCount === 0) continue documents.push({ id: row.id, filename: row.filename, @@ -160,8 +169,7 @@ export async function listExportableDocuments( tokenCount: row.tokenCount, characterCount: row.characterCount, file, - storedChunkCount: row.chunkCount, - hasChunks, + storedChunkCount, tags: Object.fromEntries(ALL_TAG_SLOTS.map((slot) => [slot, row[slot]])), }) } diff --git a/packages/sim-cli/src/commands/protocol/knowledge-export.test.ts b/packages/sim-cli/src/commands/protocol/knowledge-export.test.ts index 3eb8387c3e9..6e92f7dfccc 100644 --- a/packages/sim-cli/src/commands/protocol/knowledge-export.test.ts +++ b/packages/sim-cli/src/commands/protocol/knowledge-export.test.ts @@ -232,6 +232,15 @@ describe('attachmentFileName', () => { it('keeps only the base name and ignores a missing or empty header', () => { expect(attachmentFileName('attachment; filename="../../etc/passwd"')).toBe('passwd') + /** The server mangles non-ASCII in the quoted form, so the encoded one wins. */ + expect( + attachmentFileName( + `attachment; filename="Suporte t_cnico.simkb.zip"; filename*=UTF-8''${encodeURIComponent('Suporte técnico.simkb.zip')}` + ) + ).toBe('Suporte técnico.simkb.zip') + expect(attachmentFileName('attachment; filename="ok.zip"; filename*=UTF-8\'\'%E0%A4%A')).toBe( + 'ok.zip' + ) expect(attachmentFileName('attachment; filename=".."')).toBeNull() expect(attachmentFileName('attachment')).toBeNull() expect(attachmentFileName(null)).toBeNull() diff --git a/packages/sim-cli/src/commands/protocol/knowledge-export.ts b/packages/sim-cli/src/commands/protocol/knowledge-export.ts index 9418429e743..4ffbcd77fca 100644 --- a/packages/sim-cli/src/commands/protocol/knowledge-export.ts +++ b/packages/sim-cli/src/commands/protocol/knowledge-export.ts @@ -13,18 +13,32 @@ interface KnowledgeExportOptions { } /** - * The file name a `Content-Disposition: attachment; filename="..."` header - * carries, or `null` when the header names none. + * The file name a `Content-Disposition: attachment` header carries, or `null` + * when it names none. * - * Only the quoted form is read: the export route always emits it, with a name - * the server has already stripped of quotes, slashes, and control characters. - * Only the base name is kept so a directory in the header can never decide - * where the archive lands on the caller's disk. + * The RFC 5987 `filename*` form is read first, because the server only emits it + * when the real name is not printable ASCII — and in exactly that case the + * quoted form beside it has had every such character replaced, so reading the + * quoted form alone would save a knowledge base named "Suporte técnico" as + * `Suporte t_cnico`. Only the base name is kept, so a directory in the header + * can never decide where the archive lands on the caller's disk. */ export function attachmentFileName(contentDisposition: string | null): string | null { - const match = contentDisposition ? /filename="([^"]*)"/.exec(contentDisposition) : null - if (!match) return null - const base = basename(match[1].trim()) + if (!contentDisposition) return null + const encoded = /filename\*=UTF-8''([^;]+)/i.exec(contentDisposition)?.[1] + if (encoded) { + try { + return safeBaseName(decodeURIComponent(encoded)) + } catch { + /** A malformed escape is not a name; fall through to the quoted form. */ + } + } + const quoted = /filename="([^"]*)"/.exec(contentDisposition)?.[1] + return quoted === undefined ? null : safeBaseName(quoted) +} + +function safeBaseName(name: string): string | null { + const base = basename(name.trim()) return base && base !== '.' && base !== '..' ? base : null }