diff --git a/apps/docs/content/docs/en/tables/index.mdx b/apps/docs/content/docs/en/tables/index.mdx index be5f75947b5..63d04146f51 100644 --- a/apps/docs/content/docs/en/tables/index.mdx +++ b/apps/docs/content/docs/en/tables/index.mdx @@ -24,6 +24,7 @@ Every column has a type, which decides how its values are stored and validated. | **Currency** | An amount in a currency you pick per column | `$1,234.56` | | **Boolean** | `true` or `false` | `true` | | **Date** | A date | `2026-03-16` | +| **TTL** | An absolute row expiration time, stored as Unix epoch seconds (seconds since January 1, 1970 UTC) | `1773671400` | | **JSON** | An object or array | `{ "tier": "pro" }` | | **Select** | One of a fixed set of options, or several | `Pro` | @@ -31,6 +32,8 @@ Types are enforced as you enter values, so a Number column only takes numbers. A Currency column stores a plain number and renders it in the currency you choose for that column, so filters, sorts, and exports all see the amount itself. Changing a column's currency relabels it — it does not convert the amounts. +A table can have one TTL column. Adding it enables row expiration; rows with a non-empty TTL value become eligible for deletion after that time passes. Cleanup runs periodically, so actual row removal may happen after the TTL timestamp rather than exactly at it. Deleting the TTL column disables expiration for the table. TTL cells use the date editor, while APIs and workflows read and write integer Unix epoch seconds. + ## Editing a table Open the **Tables** section in the sidebar and click **New table** to create one. Add columns from the column header, type into a cell to edit it, and paste rows from a spreadsheet to bulk-load. Filter and sort from the toolbar without changing the underlying data. The editor has full keyboard support; see [keyboard shortcuts](/keyboard-shortcuts). diff --git a/apps/docs/openapi-v2-tables.json b/apps/docs/openapi-v2-tables.json index 52a8d00198b..14ea7b5999e 100644 --- a/apps/docs/openapi-v2-tables.json +++ b/apps/docs/openapi-v2-tables.json @@ -4979,7 +4979,16 @@ }, "type": { "type": "string", - "enum": ["string", "number", "currency", "boolean", "date", "json", "select"], + "enum": [ + "string", + "number", + "currency", + "boolean", + "date", + "ttl", + "json", + "select" + ], "description": "Data type of values stored in the column." }, "required": { @@ -5257,7 +5266,16 @@ }, "type": { "type": "string", - "enum": ["string", "number", "currency", "boolean", "date", "json", "select"], + "enum": [ + "string", + "number", + "currency", + "boolean", + "date", + "ttl", + "json", + "select" + ], "description": "Column data type." }, "required": { @@ -5436,7 +5454,16 @@ }, "type": { "type": "string", - "enum": ["string", "number", "currency", "boolean", "date", "json", "select"], + "enum": [ + "string", + "number", + "currency", + "boolean", + "date", + "ttl", + "json", + "select" + ], "description": "Data type of values stored in the column." }, "required": { @@ -5536,7 +5563,16 @@ }, "type": { "type": "string", - "enum": ["string", "number", "currency", "boolean", "date", "json", "select"], + "enum": [ + "string", + "number", + "currency", + "boolean", + "date", + "ttl", + "json", + "select" + ], "description": "Column data type." }, "required": { @@ -5633,7 +5669,7 @@ "type": { "description": "Replacement column data type.", "type": "string", - "enum": ["string", "number", "currency", "boolean", "date", "json", "select"] + "enum": ["string", "number", "currency", "boolean", "date", "ttl", "json", "select"] }, "required": { "description": "Whether inserts must supply a value for this column.", @@ -7397,7 +7433,16 @@ }, "type": { "type": "string", - "enum": ["string", "number", "currency", "boolean", "date", "json", "select"], + "enum": [ + "string", + "number", + "currency", + "boolean", + "date", + "ttl", + "json", + "select" + ], "description": "Data type of values stored in the column." }, "required": { @@ -7597,7 +7642,16 @@ }, "type": { "type": "string", - "enum": ["string", "number", "currency", "boolean", "date", "json", "select"], + "enum": [ + "string", + "number", + "currency", + "boolean", + "date", + "ttl", + "json", + "select" + ], "description": "Output column data type." }, "required": { @@ -7738,7 +7792,16 @@ }, "type": { "type": "string", - "enum": ["string", "number", "currency", "boolean", "date", "json", "select"], + "enum": [ + "string", + "number", + "currency", + "boolean", + "date", + "ttl", + "json", + "select" + ], "description": "Output column data type." }, "required": { @@ -7856,7 +7919,16 @@ }, "type": { "type": "string", - "enum": ["string", "number", "currency", "boolean", "date", "json", "select"], + "enum": [ + "string", + "number", + "currency", + "boolean", + "date", + "ttl", + "json", + "select" + ], "description": "Data type of values stored in the column." }, "required": { diff --git a/apps/sim/app/api/cron/cleanup-table-row-ttl/route.test.ts b/apps/sim/app/api/cron/cleanup-table-row-ttl/route.test.ts new file mode 100644 index 00000000000..3bec9061001 --- /dev/null +++ b/apps/sim/app/api/cron/cleanup-table-row-ttl/route.test.ts @@ -0,0 +1,88 @@ +/** + * @vitest-environment node + */ +import { createMockRequest } from '@sim/testing' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockEnqueue, mockGetJobQueue, mockVerifyCronAuth } = vi.hoisted(() => ({ + mockEnqueue: vi.fn(), + mockGetJobQueue: vi.fn(), + mockVerifyCronAuth: vi.fn(), +})) + +vi.mock('@/lib/auth/internal', () => ({ verifyCronAuth: mockVerifyCronAuth })) +vi.mock('@/lib/core/async-jobs', () => ({ getJobQueue: mockGetJobQueue })) + +import { GET } from '@/app/api/cron/cleanup-table-row-ttl/route' + +describe('table row TTL cleanup route', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-08-22T17:12:00Z')) + mockVerifyCronAuth.mockReturnValue(null) + mockEnqueue.mockResolvedValue('job-ttl-1') + mockGetJobQueue.mockResolvedValue({ enqueue: mockEnqueue }) + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('enqueues one serialized cleanup job', async () => { + const response = await GET( + createMockRequest( + 'GET', + undefined, + {}, + 'http://localhost:3000/api/cron/cleanup-table-row-ttl' + ) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ triggered: true, jobId: 'job-ttl-1' }) + expect(mockEnqueue).toHaveBeenCalledWith( + 'cleanup-table-row-ttl', + {}, + expect.objectContaining({ + maxAttempts: 1, + jobId: 'cleanup-table-row-ttl:5958062', + concurrencyKey: 'cleanup:table-row-ttl', + concurrencyLimit: 1, + runner: expect.any(Function), + }) + ) + }) + + it('deduplicates retries within the same five-minute schedule window', async () => { + const request = () => + createMockRequest( + 'GET', + undefined, + {}, + 'http://localhost:3000/api/cron/cleanup-table-row-ttl' + ) + + await GET(request()) + vi.advanceTimersByTime(2 * 60 * 1000) + await GET(request()) + + expect(mockEnqueue.mock.calls[0]?.[2]?.jobId).toBe(mockEnqueue.mock.calls[1]?.[2]?.jobId) + }) + + it('returns the cron auth refusal without touching the queue', async () => { + mockVerifyCronAuth.mockReturnValue(new Response(null, { status: 401 })) + + const response = await GET( + createMockRequest( + 'GET', + undefined, + {}, + 'http://localhost:3000/api/cron/cleanup-table-row-ttl' + ) + ) + + expect(response.status).toBe(401) + expect(mockGetJobQueue).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/api/cron/cleanup-table-row-ttl/route.ts b/apps/sim/app/api/cron/cleanup-table-row-ttl/route.ts new file mode 100644 index 00000000000..a8a62dd53b2 --- /dev/null +++ b/apps/sim/app/api/cron/cleanup-table-row-ttl/route.ts @@ -0,0 +1,41 @@ +import { createLogger } from '@sim/logger' +import { type NextRequest, NextResponse } from 'next/server' +import { verifyCronAuth } from '@/lib/auth/internal' +import { getJobQueue } from '@/lib/core/async-jobs' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' + +export const dynamic = 'force-dynamic' + +const logger = createLogger('CleanupTableRowTtlApi') +const TTL_CLEANUP_INTERVAL_MS = 5 * 60 * 1000 + +export const GET = withRouteHandler(async (request: NextRequest) => { + try { + const authError = verifyCronAuth(request, 'table row TTL cleanup') + if (authError) return authError + + const queue = await getJobQueue() + const scheduleWindow = Math.floor(Date.now() / TTL_CLEANUP_INTERVAL_MS) + const jobId = await queue.enqueue( + 'cleanup-table-row-ttl', + {}, + { + maxAttempts: 1, + jobId: `cleanup-table-row-ttl:${scheduleWindow}`, + name: 'Table row TTL cleanup', + concurrencyKey: 'cleanup:table-row-ttl', + concurrencyLimit: 1, + runner: async (_payload, signal) => { + const { runCleanupTableRowTtl } = await import('@/background/cleanup-table-row-ttl') + return runCleanupTableRowTtl(signal) + }, + } + ) + + logger.info('Table row TTL cleanup dispatched', { jobId }) + return NextResponse.json({ triggered: true, jobId }) + } catch (error) { + logger.error('Failed to dispatch table row TTL cleanup', { error }) + return NextResponse.json({ error: 'Failed to dispatch table row TTL cleanup' }, { status: 500 }) + } +}) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/column-types.test.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/column-types.test.ts new file mode 100644 index 00000000000..4ee012d24ce --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/column-config-sidebar/column-types.test.ts @@ -0,0 +1,30 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import type { ColumnDefinition } from '@/lib/table' +import { columnTypeOptionsForTable } from './column-types' + +describe('columnTypeOptionsForTable', () => { + const ttlColumn: ColumnDefinition = { name: 'expires_at', type: 'ttl' } + + it('disables TTL with an explanation when the table already has one', () => { + const availableTtl = columnTypeOptionsForTable([{ name: 'name', type: 'string' }]).find( + (option) => option.type === 'ttl' + ) + const unavailableTtl = columnTypeOptionsForTable([ttlColumn]).find( + (option) => option.type === 'ttl' + ) + + expect(availableTtl?.disabledReason).toBeUndefined() + expect(unavailableTtl?.disabledReason).toBe('Only one TTL column allowed per table') + }) + + it('keeps TTL enabled while editing the existing TTL column', () => { + const ttlOption = columnTypeOptionsForTable([ttlColumn], ttlColumn).find( + (option) => option.type === 'ttl' + ) + + expect(ttlOption?.disabledReason).toBeUndefined() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx index d46e2193330..e9734bafba0 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/row-modal/row-modal.tsx @@ -251,7 +251,7 @@ function ColumnField({ column, value, onChange }: ColumnFieldProps) { required={column.required} hint={hint} mono - value={formatValueForInput(value, column.type)} + value={formatValueForInput(value, column.type, timeZone)} onChange={onChange} placeholder='{"key": "value"}' rows={4} @@ -260,7 +260,7 @@ function ColumnField({ column, value, onChange }: ColumnFieldProps) { } if (definition.editor === 'date') { - const parts = dateValueToLocalParts(formatValueForInput(value, 'date')) + const parts = dateValueToLocalParts(formatValueForInput(value, column.type, timeZone)) return (
@@ -306,7 +306,7 @@ function ColumnField({ column, value, onChange }: ColumnFieldProps) { inputType={ definition.inputMode === 'decimal' && !definition.acceptsFormattedInput ? 'number' : 'text' } - value={formatValueForInput(value, column.type)} + value={formatValueForInput(value, column.type, timeZone)} onChange={onChange} placeholder={`Enter ${column.name}`} /> diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-content.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-content.tsx index 54a2c7f2dea..1511332da6b 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-content.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-content.tsx @@ -13,6 +13,7 @@ interface CellContentProps { /** Current workspace id — lets string cells holding an in-workspace resource * URL render as a tagged-resource chip instead of a plain external link. */ workspaceId: string + timeZone: string isEditing: boolean initialCharacter?: string | null onSave: (value: unknown, reason: SaveReason) => void @@ -38,6 +39,7 @@ export function CellContent({ exec, column, workspaceId, + timeZone, isEditing, initialCharacter, onSave, @@ -52,6 +54,7 @@ export function CellContent({ waitingOnLabels, isEnrichmentOutput, currentWorkspaceId: workspaceId, + timeZone, }) return ( diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.test.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.test.ts new file mode 100644 index 00000000000..dee543bf4fb --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.test.ts @@ -0,0 +1,32 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { resolveCellRender } from '@/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render' +import type { DisplayColumn } from '@/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/types' + +function column(type: DisplayColumn['type']): DisplayColumn { + return { + key: 'expires_at', + name: 'expires_at', + type, + groupSize: 1, + groupStartColIndex: 0, + headerLabel: 'expires_at', + isGroupStart: true, + } +} + +describe('resolveCellRender', () => { + it('renders TTL epoch seconds through the date presentation', () => { + expect( + resolveCellRender({ + value: 1_700_000_000, + exec: undefined, + column: column('ttl'), + waitingOnLabels: undefined, + timeZone: 'America/New_York', + }) + ).toEqual({ kind: 'date', text: '2023-11-14T17:13:20-05:00' }) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx index 4e16d03912b..9ebaacf233f 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/cell-render.tsx @@ -53,6 +53,8 @@ interface ResolveCellRenderInput { /** Current workspace id — a URL pointing to a resource in this workspace * renders as a tagged-resource chip rather than a plain external link. */ currentWorkspaceId?: string + /** Effective viewer timezone for instant-like column presentations. */ + timeZone?: string } export function resolveCellRender({ @@ -62,6 +64,7 @@ export function resolveCellRender({ waitingOnLabels, isEnrichmentOutput, currentWorkspaceId, + timeZone, }: ResolveCellRenderInput): CellRenderKind { const isNull = value === null || value === undefined const isEmpty = isNull || value === '' @@ -137,7 +140,10 @@ export function resolveCellRender({ return { kind: 'text', text: columnTypeOf(column).formatForDisplay(value, column) } } if (column.type === 'json') return { kind: 'json', text: JSON.stringify(value) } - if (column.type === 'date') return { kind: 'date', text: String(value) } + const definition = columnTypeOf(column) + if (definition.editor === 'date') { + return { kind: 'date', text: definition.formatForInput(value, column, { timezone: timeZone }) } + } if (column.type === 'string') { const text = stringifyValue(value) return resolveLinkKind(text, currentWorkspaceId) ?? { kind: 'text', text } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx index b93cea863d5..f5c8526173b 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/cells/inline-editors.tsx @@ -70,7 +70,7 @@ function InlineDateEditor({ const popoverPointerAtRef = useRef(0) const timeZone = useTimezone() - const storedValue = formatValueForInput(value, column.type) + const storedValue = formatValueForInput(value, column.type, timeZone) const initialDraft = initialCharacter !== undefined ? initialCharacter @@ -115,7 +115,7 @@ function InlineDateEditor({ // silently shifting the instant of a value someone else wrote. if (storageVal === undefined && initialCharacter === undefined && current === initialDraft) { doneRef.current = true - onSave(storedValue || null, reason) + onSave(storedValue ? cleanCellValue(storedValue, column, timeZone) : null, reason) return } const raw = storageVal ?? displayToStorage(current, timeZone) ?? current @@ -132,9 +132,9 @@ function InlineDateEditor({ return } doneRef.current = true - onSave(raw || null, reason) + onSave(raw ? cleanCellValue(raw, column, timeZone) : null, reason) }, - [invalid, onSave, onCancel, timeZone, initialDraft, initialCharacter, storedValue] + [invalid, onSave, onCancel, timeZone, initialDraft, initialCharacter, storedValue, column] ) const handleKeyDown = useCallback( diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/data-row.tsx b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/data-row.tsx index 077f73fb2e5..abc6828ea77 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/data-row.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/data-row.tsx @@ -26,6 +26,8 @@ export interface DataRowProps { /** Current workspace id — forwarded to cells so in-workspace resource URLs * render as tagged-resource chips. */ workspaceId: string + /** Effective viewer timezone used to render TTL instants. */ + timeZone: string rowIndex: number isFirstRow: boolean editingColumnName: string | null @@ -114,6 +116,7 @@ function dataRowPropsAreEqual(prev: DataRowProps, next: DataRowProps): boolean { prev.row !== next.row || prev.columns !== next.columns || prev.workspaceId !== next.workspaceId || + prev.timeZone !== next.timeZone || prev.rowIndex !== next.rowIndex || prev.isFirstRow !== next.isFirstRow || prev.editingColumnName !== next.editingColumnName || @@ -161,6 +164,7 @@ export const DataRow = React.memo(function DataRow({ row, columns, workspaceId, + timeZone, rowIndex, isFirstRow, editingColumnName, @@ -396,6 +400,7 @@ export const DataRow = React.memo(function DataRow({
{ ) expect(formatValueForInput('2026-07-06', 'date')).toBe('2026-07-06') }) + + it('renders TTL instants in the editor timezone without changing the instant', () => { + expect(formatValueForInput(1_700_000_000, 'ttl', 'America/New_York')).toBe( + '2023-11-14T17:13:20-05:00' + ) + expect( + cleanCellValue('2023-11-14 17:13:20', { name: 'expires_at', type: 'ttl' }, 'America/New_York') + ).toBe(1_700_000_000) + expect( + cleanCellValue('2023-11-14', { name: 'expires_at', type: 'ttl' }, 'America/New_York') + ).toBe(1_699_938_000) + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.ts b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.ts index 69f7722d11d..b31b5f1ea48 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.ts +++ b/apps/sim/app/workspace/[workspaceId]/tables/[tableId]/utils.ts @@ -55,7 +55,7 @@ export function cleanCellValue( // Everything else runs the SAME coercion the server will run, so the // optimistic cache holds exactly the value that gets persisted. const columnType = columnTypeOf(column) - const coerced = columnType.coerce(value as JsonValue, column) + const coerced = columnType.coerce(value as JsonValue, column, { timezone: timeZone }) if (coerced.ok) return coerced.value const salvaged = columnType.salvage?.(value as JsonValue, column) return salvaged?.ok ? salvaged.value : null @@ -68,7 +68,7 @@ export function cleanCellValue( * row data already has the new mapping's value) would otherwise render * `[object Object]` via `String(value)`. */ -export function formatValueForInput(value: unknown, type: string): string { +export function formatValueForInput(value: unknown, type: string, timeZone?: string): string { if (value === null || value === undefined) return '' const definition = columnTypeById(type) // Shape-drift guard, kept ahead of the registry: a column whose declared type @@ -78,7 +78,11 @@ export function formatValueForInput(value: unknown, type: string): string { if (typeof value === 'object' && !definition.storesOpaqueIds && type !== 'json') { return JSON.stringify(value) } - return definition.formatForInput(value, { name: '', type: type as ColumnType }) + return definition.formatForInput( + value, + { name: '', type: type as ColumnType }, + { timezone: timeZone } + ) } /** A canonical date-cell value split into its wall-clock editing parts. */ diff --git a/apps/sim/background/cleanup-table-row-ttl.test.ts b/apps/sim/background/cleanup-table-row-ttl.test.ts new file mode 100644 index 00000000000..509d7432625 --- /dev/null +++ b/apps/sim/background/cleanup-table-row-ttl.test.ts @@ -0,0 +1,175 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockDeleteExecute, + mockListExecute, + mockSignalTableRowsChanged, + mockTask, + mockWithLockedTable, +} = vi.hoisted(() => ({ + mockDeleteExecute: vi.fn(), + mockListExecute: vi.fn(), + mockSignalTableRowsChanged: vi.fn(), + mockTask: vi.fn((config: unknown) => config), + mockWithLockedTable: vi.fn(), +})) + +vi.mock('@sim/db', () => ({ + dbFor: vi.fn(() => ({ execute: mockListExecute })), +})) + +vi.mock('@trigger.dev/sdk', () => ({ task: mockTask })) +vi.mock('@/lib/table/events', () => ({ signalTableRowsChanged: mockSignalTableRowsChanged })) +vi.mock('@/lib/table/service', () => ({ withLockedTable: mockWithLockedTable })) + +import { cleanupTableRowTtlTask, runCleanupTableRowTtl } from '@/background/cleanup-table-row-ttl' + +const table = { + id: 'table-1', + workspaceId: 'workspace-1', + schema: { columns: [{ id: 'col-ttl', name: 'expires_at', type: 'ttl' }] }, + locks: { insertLocked: false, updateLocked: false, deleteLocked: false, schemaLocked: false }, +} + +describe('table row TTL cleanup', () => { + beforeEach(() => { + vi.clearAllMocks() + mockListExecute.mockResolvedValue([{ id: table.id, workspaceId: table.workspaceId }]) + mockWithLockedTable.mockImplementation( + async ( + _tableId: string, + mutate: ( + fresh: typeof table, + trx: { execute: typeof mockDeleteExecute } + ) => Promise + ) => mutate(table, { execute: mockDeleteExecute }) + ) + }) + + it('deletes expired rows in locked, keyset batches and signals the table', async () => { + mockDeleteExecute + .mockResolvedValueOnce([{ count: 500, lastId: 'row-500' }]) + .mockResolvedValueOnce([{ count: 12, lastId: 'row-512' }]) + + await expect(runCleanupTableRowTtl()).resolves.toEqual({ + batches: 2, + deleted: 512, + limitReached: false, + }) + expect(mockWithLockedTable).toHaveBeenCalledTimes(2) + expect(mockDeleteExecute).toHaveBeenCalledTimes(2) + expect(mockSignalTableRowsChanged).toHaveBeenCalledWith(table.id) + }) + + it('compares TTL values with whole Date.now epoch seconds', async () => { + const nowEpochMilliseconds = 1_700_000_000_123 + const nowEpochSeconds = 1_700_000_000 + const nowSpy = vi.spyOn(Date, 'now').mockReturnValue(nowEpochMilliseconds) + mockDeleteExecute.mockResolvedValue([{ count: 0, lastId: null }]) + + try { + await runCleanupTableRowTtl() + } finally { + nowSpy.mockRestore() + } + + expect(mockListExecute.mock.calls[0][0]).toMatchObject({ + values: expect.arrayContaining([nowEpochSeconds]), + }) + expect(mockDeleteExecute.mock.calls[0][0]).toMatchObject({ + values: expect.arrayContaining([nowEpochSeconds]), + }) + }) + + it('does no work when already aborted', async () => { + const controller = new AbortController() + controller.abort() + + await expect(runCleanupTableRowTtl(controller.signal)).resolves.toEqual({ + batches: 0, + deleted: 0, + limitReached: false, + }) + expect(mockListExecute).not.toHaveBeenCalled() + }) + + it('honors a delete lock re-read inside the table advisory lock', async () => { + mockWithLockedTable.mockImplementationOnce(async (_tableId, mutate) => + mutate( + { ...table, locks: { ...table.locks, deleteLocked: true } }, + { execute: mockDeleteExecute } + ) + ) + + await expect(runCleanupTableRowTtl()).resolves.toEqual({ + batches: 0, + deleted: 0, + limitReached: false, + }) + expect(mockDeleteExecute).not.toHaveBeenCalled() + expect(mockSignalTableRowsChanged).not.toHaveBeenCalled() + }) + + it('stops after one hundred full batches', async () => { + mockDeleteExecute.mockResolvedValue([{ count: 500, lastId: 'row-cursor' }]) + + await expect(runCleanupTableRowTtl()).resolves.toEqual({ + batches: 100, + deleted: 50_000, + limitReached: true, + }) + expect(mockDeleteExecute).toHaveBeenCalledTimes(100) + expect(mockSignalTableRowsChanged).toHaveBeenCalledTimes(1) + }) + + it('gives each table one batch before returning to a backlogged table', async () => { + const secondTable = { + ...table, + id: 'table-2', + } + const attemptedTableIds: string[] = [] + const tableAttempts = new Map() + mockListExecute.mockResolvedValue([ + { id: table.id, workspaceId: table.workspaceId }, + { id: secondTable.id, workspaceId: secondTable.workspaceId }, + ]) + mockWithLockedTable.mockImplementation(async (tableId, mutate) => { + const freshTable = tableId === secondTable.id ? secondTable : table + return mutate(freshTable, { + execute: vi.fn(async () => { + attemptedTableIds.push(tableId) + const attempt = (tableAttempts.get(tableId) ?? 0) + 1 + tableAttempts.set(tableId, attempt) + if (tableId === table.id && attempt === 1) { + return [{ count: 500, lastId: 'row-500' }] + } + if (tableId === secondTable.id) { + return [{ count: 1, lastId: 'row-1' }] + } + return [{ count: 0, lastId: null }] + }), + }) + }) + + await expect(runCleanupTableRowTtl()).resolves.toEqual({ + batches: 3, + deleted: 501, + limitReached: false, + }) + expect(attemptedTableIds).toEqual([table.id, secondTable.id, table.id]) + expect(mockSignalTableRowsChanged).toHaveBeenCalledWith(table.id) + expect(mockSignalTableRowsChanged).toHaveBeenCalledWith(secondTable.id) + }) + + it('registers one serialized Trigger.dev task', () => { + expect(cleanupTableRowTtlTask).toEqual( + expect.objectContaining({ + id: 'cleanup-table-row-ttl', + queue: { concurrencyLimit: 1 }, + }) + ) + }) +}) diff --git a/apps/sim/background/cleanup-table-row-ttl.ts b/apps/sim/background/cleanup-table-row-ttl.ts new file mode 100644 index 00000000000..3243dbb2b99 --- /dev/null +++ b/apps/sim/background/cleanup-table-row-ttl.ts @@ -0,0 +1,227 @@ +import { dbFor } from '@sim/db' +import { userTableDefinitions, userTableRows } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { task } from '@trigger.dev/sdk' +import { sql } from 'drizzle-orm' +import { asOrchestrationError } from '@/lib/core/orchestration/types' +import { getColumnId } from '@/lib/table/column-keys' +import { signalTableRowsChanged } from '@/lib/table/events' +import { assertRowDelete, TableLockedError } from '@/lib/table/mutation-locks' +import type { DbTransaction } from '@/lib/table/planner' +import { withLockedTable } from '@/lib/table/service' + +const logger = createLogger('CleanupTableRowTtl') +const cleanupDb = dbFor('cleanup') + +const TTL_CLEANUP_BATCH_SIZE = 500 +const TTL_CLEANUP_MAX_BATCHES = 100 + +interface ExpiredTtlTableRef { + [key: string]: unknown + id: string + workspaceId: string +} + +interface DeletedTtlBatch { + attempted: boolean + deleted: number + lastId: string | null +} + +interface TtlTableCleanupState { + ref: ExpiredTtlTableRef + afterId?: string + deleted: number + complete: boolean +} + +export interface TableRowTtlCleanupResult { + batches: number + deleted: number + limitReached: boolean +} + +async function listExpiredTtlTables(nowEpochSeconds: number): Promise { + const rows = await cleanupDb.execute(sql` + SELECT + ${userTableDefinitions.id} AS id, + ${userTableDefinitions.workspaceId} AS "workspaceId" + FROM ${userTableDefinitions} + WHERE ${userTableDefinitions.archivedAt} IS NULL + AND ${userTableDefinitions.deleteLocked} = false + AND EXISTS ( + SELECT 1 + FROM jsonb_array_elements( + COALESCE(${userTableDefinitions.schema}->'columns', '[]'::jsonb) + ) AS ttl_column(column_definition) + JOIN ${userTableRows} AS table_row + ON table_row.table_id = ${userTableDefinitions.id} + AND table_row.workspace_id = ${userTableDefinitions.workspaceId} + WHERE ttl_column.column_definition->>'type' = 'ttl' + AND jsonb_typeof( + table_row.data->COALESCE( + ttl_column.column_definition->>'id', + ttl_column.column_definition->>'name' + ) + ) = 'number' + AND ( + table_row.data->>COALESCE( + ttl_column.column_definition->>'id', + ttl_column.column_definition->>'name' + ) + )::numeric <= ${nowEpochSeconds} + ) + ORDER BY + md5(${userTableDefinitions.id} || ${nowEpochSeconds}::text), + ${userTableDefinitions.id} + LIMIT ${TTL_CLEANUP_MAX_BATCHES} + `) + return Array.isArray(rows) ? rows : [] +} + +function parseDeletedBatch(rows: unknown): Omit { + const [row] = Array.isArray(rows) + ? (rows as Array<{ count?: number | string; lastId?: string | null }>) + : [] + if (!row) throw new Error('Table row TTL cleanup did not return a deleted count') + + const deleted = Number(row.count) + if (!Number.isSafeInteger(deleted) || deleted < 0 || deleted > TTL_CLEANUP_BATCH_SIZE) { + throw new Error('Table row TTL cleanup returned an invalid deleted count') + } + if (deleted > 0 && typeof row.lastId !== 'string') { + throw new Error('Table row TTL cleanup did not return a row cursor') + } + return { deleted, lastId: row.lastId ?? null } +} + +async function deleteExpiredTableRowBatch( + trx: DbTransaction, + tableId: string, + workspaceId: string, + columnKey: string, + nowEpochSeconds: number, + afterId?: string +): Promise> { + const rows = await trx.execute<{ count: number | string; lastId: string | null }>(sql` + WITH candidates AS MATERIALIZED ( + SELECT table_row.id + FROM ${userTableRows} AS table_row + WHERE table_row.table_id = ${tableId} + AND table_row.workspace_id = ${workspaceId} + ${afterId ? sql`AND table_row.id > ${afterId}` : sql``} + AND jsonb_typeof(table_row.data->${columnKey}) = 'number' + AND (table_row.data->>${columnKey})::numeric <= ${nowEpochSeconds} + ORDER BY table_row.id + LIMIT ${TTL_CLEANUP_BATCH_SIZE} + FOR UPDATE OF table_row SKIP LOCKED + ), deleted AS ( + DELETE FROM ${userTableRows} AS table_row + USING candidates + WHERE table_row.id = candidates.id + RETURNING table_row.id + ) + SELECT + count(*)::integer AS count, + max(id) AS "lastId" + FROM deleted + `) + return parseDeletedBatch(rows) +} + +async function deleteExpiredRowsForTable( + ref: ExpiredTtlTableRef, + nowEpochSeconds: number, + afterId?: string +): Promise { + try { + return await withLockedTable( + ref.id, + async (table, trx) => { + try { + assertRowDelete(table) + } catch (error) { + if (error instanceof TableLockedError) { + return { attempted: false, deleted: 0, lastId: null } + } + throw error + } + + const ttlColumn = table.schema.columns.find((column) => column.type === 'ttl') + if (!ttlColumn) return { attempted: false, deleted: 0, lastId: null } + + const batch = await deleteExpiredTableRowBatch( + trx, + table.id, + table.workspaceId, + getColumnId(ttlColumn), + nowEpochSeconds, + afterId + ) + return { attempted: true, ...batch } + }, + { expectedWorkspaceId: ref.workspaceId } + ) + } catch (error) { + if (asOrchestrationError(error)?.code === 'not_found') { + return { attempted: false, deleted: 0, lastId: null } + } + throw error + } +} + +/** Deletes rows whose table TTL cell is at or before the current Unix epoch second. */ +export async function runCleanupTableRowTtl( + signal?: AbortSignal +): Promise { + if (signal?.aborted) return { batches: 0, deleted: 0, limitReached: false } + + const nowEpochSeconds = Math.floor(Date.now() / 1000) + const tableRefs = await listExpiredTtlTables(nowEpochSeconds) + const tableStates: TtlTableCleanupState[] = tableRefs.map((ref) => ({ + ref, + deleted: 0, + complete: false, + })) + let deleted = 0 + let batches = 0 + + while ( + batches < TTL_CLEANUP_MAX_BATCHES && + !signal?.aborted && + tableStates.some((state) => !state.complete) + ) { + for (const state of tableStates) { + if (state.complete) continue + if (batches === TTL_CLEANUP_MAX_BATCHES || signal?.aborted) break + + const batch = await deleteExpiredRowsForTable(state.ref, nowEpochSeconds, state.afterId) + if (!batch.attempted) { + state.complete = true + continue + } + + batches++ + deleted += batch.deleted + state.deleted += batch.deleted + state.afterId = batch.lastId ?? undefined + if (batch.deleted < TTL_CLEANUP_BATCH_SIZE) state.complete = true + } + } + + for (const state of tableStates) { + if (state.deleted > 0) signalTableRowsChanged(state.ref.id) + } + + const limitReached = + batches === TTL_CLEANUP_MAX_BATCHES && + (tableStates.some((state) => !state.complete) || tableRefs.length === TTL_CLEANUP_MAX_BATCHES) + logger.info('Table row TTL cleanup completed', { batches, deleted, limitReached }) + return { batches, deleted, limitReached } +} + +export const cleanupTableRowTtlTask = task({ + id: 'cleanup-table-row-ttl', + queue: { concurrencyLimit: 1 }, + run: () => runCleanupTableRowTtl(), +}) diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index bbb44542619..866e43f4ac3 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -4171,7 +4171,7 @@ export const QueryUserTable: ToolCatalogEntry = { filter: { type: 'object', description: - 'Predicate filter object for query_rows. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}.', + 'Predicate filter object for query_rows. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are absolute whole Unix epoch seconds, never milliseconds. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}.', }, limit: { type: 'number', @@ -5433,7 +5433,7 @@ export const TableColumns: ToolCatalogEntry = { column: { type: 'object', description: - 'Column definition for add_column: { name, type, unique?, position? }; select (enum) columns also take { options: [names], multiple?: true } — options is required for select.', + 'Column definition for add_column: { name, type, unique?, position? }; type may be string, number, boolean, date, json, select, or ttl. Select (enum) columns also take { options: [names], multiple?: true } — options is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', }, columnName: { type: 'string', @@ -5454,7 +5454,7 @@ export const TableColumns: ToolCatalogEntry = { newType: { type: 'string', description: - 'New column type for update_column: string, number, boolean, date, json, select. Converting to select also requires options; conversion fails if an existing cell value matches no option. A multiple select round-trips through text as a comma-separated cell.', + 'New column type for update_column: string, number, boolean, date, json, select, ttl. Converting to select also requires options; conversion fails if an existing cell value matches no option. A multiple select round-trips through text as a comma-separated cell. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are absolute whole Unix epoch seconds, not milliseconds.', }, options: { type: 'array', @@ -5633,7 +5633,7 @@ export const TableManage: ToolCatalogEntry = { schema: { type: 'object', description: - 'Table schema with a columns array (required for create). Each column: { name, type, unique? }; a select (enum) column also requires options (display names) and takes multiple?.', + 'Table schema with a columns array (required for create). Each column: { name, type, unique? }; types are string, number, boolean, date, json, select, and ttl. A select (enum) column also requires options (display names) and takes multiple?. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', }, tableId: { type: 'string', @@ -5679,12 +5679,12 @@ export const TableRows: ToolCatalogEntry = { data: { type: 'object', description: - 'Row data as column → value pairs (required for insert_row, update_row; the patch object for update_rows_by_filter). Select (enum) cells take the option NAME.', + 'Row data as column → value pairs (required for insert_row, update_row; the patch object for update_rows_by_filter). Select (enum) cells take the option NAME. TTL cells take an absolute whole Unix epoch timestamp in seconds, never JavaScript milliseconds; missing or null TTL means no expiration.', }, filter: { type: 'object', description: - 'Predicate filter for update_rows_by_filter / delete_rows_by_filter: {"all":[...]} (AND) or {"any":[...]} (OR) of {field, op, value} leaves or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (* wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array. Single-select columns match by eq/ne/in/nin; multiple-select by contains/ncontains — values are option NAMES.', + 'Predicate filter for update_rows_by_filter / delete_rows_by_filter: {"all":[...]} (AND) or {"any":[...]} (OR) of {field, op, value} leaves or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (* wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array. Single-select columns match by eq/ne/in/nin; multiple-select by contains/ncontains — values are option NAMES. TTL filter values are absolute whole Unix epoch seconds.', }, limit: { type: 'number', @@ -5709,7 +5709,8 @@ export const TableRows: ToolCatalogEntry = { }, rows: { type: 'array', - description: 'Array of row data objects (required for batch_insert_rows)', + description: + 'Array of row data objects (required for batch_insert_rows). TTL cells take absolute whole Unix epoch timestamps in seconds, never JavaScript milliseconds.', }, tableId: { type: 'string', description: 'Table ID (required for every operation)' }, updates: { @@ -6038,7 +6039,7 @@ export const UserTable: ToolCatalogEntry = { column: { type: 'object', description: - 'Column definition for add_column: { name, type, unique?, position? }. For a select (enum) column also pass { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select.', + 'Column definition for add_column: { name, type, unique?, position? }. Type may be string, number, boolean, date, json, select, or ttl. For a select (enum) column also pass { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', }, columnName: { type: 'string', @@ -6057,7 +6058,8 @@ export const UserTable: ToolCatalogEntry = { }, data: { type: 'object', - description: 'Row data as key-value pairs (required for insert_row, update_row)', + description: + 'Row data as key-value pairs (required for insert_row, update_row). TTL cells take an absolute whole Unix epoch timestamp in seconds, never JavaScript milliseconds; missing or null TTL means no expiration.', }, dependencies: { type: 'object', @@ -6092,7 +6094,7 @@ export const UserTable: ToolCatalogEntry = { filter: { type: 'object', description: - 'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.', + 'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are absolute whole Unix epoch seconds, never milliseconds. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.', }, groupId: { type: 'string', @@ -6181,7 +6183,7 @@ export const UserTable: ToolCatalogEntry = { newType: { type: 'string', description: - 'New column type (optional for update_column). Types: string, number, boolean, date, json, select. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips.', + 'New column type (optional for update_column). Types: string, number, boolean, date, json, select, ttl. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are absolute whole Unix epoch seconds, not milliseconds.', }, options: { type: 'array', @@ -6259,7 +6261,8 @@ export const UserTable: ToolCatalogEntry = { }, rows: { type: 'array', - description: 'Array of row data objects (required for batch_insert_rows)', + description: + 'Array of row data objects (required for batch_insert_rows). TTL cells take absolute whole Unix epoch timestamps in seconds, never JavaScript milliseconds.', }, runMode: { type: 'string', @@ -6270,7 +6273,7 @@ export const UserTable: ToolCatalogEntry = { schema: { type: 'object', description: - 'Table schema with columns array (required for \'create\'). Each column: { name, type, unique? }. A select (enum) column also takes { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select.', + 'Table schema with columns array (required for \'create\'). Each column: { name, type, unique? }. Types are string, number, boolean, date, json, select, and ttl. A select (enum) column also takes { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', }, scope: { type: 'string', diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index c7f0cfcd3bf..a7c9619e182 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -4066,7 +4066,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { filter: { type: 'object', description: - 'Predicate filter object for query_rows. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}.', + 'Predicate filter object for query_rows. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are absolute whole Unix epoch seconds, never milliseconds. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}.', }, limit: { type: 'number', @@ -5325,7 +5325,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { column: { type: 'object', description: - 'Column definition for add_column: { name, type, unique?, position? }; select (enum) columns also take { options: [names], multiple?: true } — options is required for select.', + 'Column definition for add_column: { name, type, unique?, position? }; type may be string, number, boolean, date, json, select, or ttl. Select (enum) columns also take { options: [names], multiple?: true } — options is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', }, columnName: { type: 'string', @@ -5349,7 +5349,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { newType: { type: 'string', description: - 'New column type for update_column: string, number, boolean, date, json, select. Converting to select also requires options; conversion fails if an existing cell value matches no option. A multiple select round-trips through text as a comma-separated cell.', + 'New column type for update_column: string, number, boolean, date, json, select, ttl. Converting to select also requires options; conversion fails if an existing cell value matches no option. A multiple select round-trips through text as a comma-separated cell. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are absolute whole Unix epoch seconds, not milliseconds.', }, options: { type: 'array', @@ -5555,7 +5555,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { schema: { type: 'object', description: - 'Table schema with a columns array (required for create). Each column: { name, type, unique? }; a select (enum) column also requires options (display names) and takes multiple?.', + 'Table schema with a columns array (required for create). Each column: { name, type, unique? }; types are string, number, boolean, date, json, select, and ttl. A select (enum) column also requires options (display names) and takes multiple?. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', }, tableId: { type: 'string', @@ -5605,12 +5605,12 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { data: { type: 'object', description: - 'Row data as column → value pairs (required for insert_row, update_row; the patch object for update_rows_by_filter). Select (enum) cells take the option NAME.', + 'Row data as column → value pairs (required for insert_row, update_row; the patch object for update_rows_by_filter). Select (enum) cells take the option NAME. TTL cells take an absolute whole Unix epoch timestamp in seconds, never JavaScript milliseconds; missing or null TTL means no expiration.', }, filter: { type: 'object', description: - 'Predicate filter for update_rows_by_filter / delete_rows_by_filter: {"all":[...]} (AND) or {"any":[...]} (OR) of {field, op, value} leaves or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (* wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array. Single-select columns match by eq/ne/in/nin; multiple-select by contains/ncontains — values are option NAMES.', + 'Predicate filter for update_rows_by_filter / delete_rows_by_filter: {"all":[...]} (AND) or {"any":[...]} (OR) of {field, op, value} leaves or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (* wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array. Single-select columns match by eq/ne/in/nin; multiple-select by contains/ncontains — values are option NAMES. TTL filter values are absolute whole Unix epoch seconds.', }, limit: { type: 'number', @@ -5640,7 +5640,8 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, rows: { type: 'array', - description: 'Array of row data objects (required for batch_insert_rows)', + description: + 'Array of row data objects (required for batch_insert_rows). TTL cells take absolute whole Unix epoch timestamps in seconds, never JavaScript milliseconds.', }, tableId: { type: 'string', @@ -5984,7 +5985,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { column: { type: 'object', description: - 'Column definition for add_column: { name, type, unique?, position? }. For a select (enum) column also pass { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select.', + 'Column definition for add_column: { name, type, unique?, position? }. Type may be string, number, boolean, date, json, select, or ttl. For a select (enum) column also pass { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', }, columnName: { type: 'string', @@ -6003,7 +6004,8 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, data: { type: 'object', - description: 'Row data as key-value pairs (required for insert_row, update_row)', + description: + 'Row data as key-value pairs (required for insert_row, update_row). TTL cells take an absolute whole Unix epoch timestamp in seconds, never JavaScript milliseconds; missing or null TTL means no expiration.', }, dependencies: { type: 'object', @@ -6043,7 +6045,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { filter: { type: 'object', description: - 'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.', + 'Predicate filter object for query_rows, update_rows_by_filter, delete_rows_by_filter. A predicate is a tree: {"all":[...]} (AND) or {"any":[...]} (OR); members are leaves {field, op, value} or nested groups. Ops: eq, ne, gt, gte, lt, lte, in, nin, like, ilike (use * as the wildcard), nlike, nilike, contains, ncontains, startsWith, endsWith, isNull, isNotNull, isEmpty, isNotEmpty. in/nin take a non-empty array value. TTL filter values are absolute whole Unix epoch seconds, never milliseconds. Examples: {"all":[{"field":"status","op":"eq","value":"active"}]}; {"all":[{"field":"wins","op":"gte","value":18},{"field":"status","op":"eq","value":"pending"}]}; {"any":[{"field":"status","op":"eq","value":"active"},{"field":"status","op":"eq","value":"pending"}]}; {"all":[{"field":"name","op":"ilike","value":"*jo*"}]}; {"all":[{"field":"slack_user_id","op":"in","value":["U1","U2"]}]}.', }, groupId: { type: 'string', @@ -6140,7 +6142,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { newType: { type: 'string', description: - 'New column type (optional for update_column). Types: string, number, boolean, date, json, select. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips.', + 'New column type (optional for update_column). Types: string, number, boolean, date, json, select, ttl. Converting a column to select also requires options; the conversion fails if any existing cell value doesn\'t match one of them. Converting to a multiple: true select also accepts a comma-separated cell ("Open, Urgent"), which is the form a multi column converts to text as — so multiselect → text → multiselect round-trips. Converting to ttl enables row expiration and fails if the table already has another ttl column; TTL values are absolute whole Unix epoch seconds, not milliseconds.', }, options: { type: 'array', @@ -6228,7 +6230,8 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, rows: { type: 'array', - description: 'Array of row data objects (required for batch_insert_rows)', + description: + 'Array of row data objects (required for batch_insert_rows). TTL cells take absolute whole Unix epoch timestamps in seconds, never JavaScript milliseconds.', }, runMode: { type: 'string', @@ -6239,7 +6242,7 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { schema: { type: 'object', description: - 'Table schema with columns array (required for \'create\'). Each column: { name, type, unique? }. A select (enum) column also takes { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select.', + 'Table schema with columns array (required for \'create\'). Each column: { name, type, unique? }. Types are string, number, boolean, date, json, select, and ttl. A select (enum) column also takes { options: ["Open", "Closed"], multiple?: true } — options is a list of display names and is required for select. A table may have at most one ttl column; adding it enables row expiration, with cell values stored as absolute whole Unix epoch seconds rather than milliseconds.', }, scope: { type: 'string', diff --git a/apps/sim/lib/core/async-jobs/backends/trigger-dev.ts b/apps/sim/lib/core/async-jobs/backends/trigger-dev.ts index 4b3960740de..3b103314a62 100644 --- a/apps/sim/lib/core/async-jobs/backends/trigger-dev.ts +++ b/apps/sim/lib/core/async-jobs/backends/trigger-dev.ts @@ -181,6 +181,7 @@ const JOB_TYPE_TO_TASK_ID: Record = { 'workflow-group-cell': 'workflow-group-cell', 'cleanup-logs': 'cleanup-logs', 'cleanup-soft-deletes': 'cleanup-soft-deletes', + 'cleanup-table-row-ttl': 'cleanup-table-row-ttl', 'cleanup-tasks': 'cleanup-tasks', 'run-data-drain': 'run-data-drain', } diff --git a/apps/sim/lib/core/async-jobs/types.ts b/apps/sim/lib/core/async-jobs/types.ts index fb5facc4b13..793ce0d938a 100644 --- a/apps/sim/lib/core/async-jobs/types.ts +++ b/apps/sim/lib/core/async-jobs/types.ts @@ -44,6 +44,7 @@ export type JobType = | 'workflow-group-cell' | 'cleanup-logs' | 'cleanup-soft-deletes' + | 'cleanup-table-row-ttl' | 'cleanup-tasks' | 'run-data-drain' diff --git a/apps/sim/lib/table/__tests__/column-type-registry.test.ts b/apps/sim/lib/table/__tests__/column-type-registry.test.ts index 91ec369e31e..7fcdceeb148 100644 --- a/apps/sim/lib/table/__tests__/column-type-registry.test.ts +++ b/apps/sim/lib/table/__tests__/column-type-registry.test.ts @@ -121,6 +121,82 @@ describe('conversion write-back', () => { }) }) +describe('ttl columns', () => { + const column = { name: 'expires_at', type: 'ttl' } as ColumnDefinition + + it('stores integer epoch seconds while accepting date-shaped input', () => { + expect(COLUMN_TYPE_REGISTRY.ttl.coerce('2023-11-14T22:13:20Z', column)).toEqual({ + ok: true, + value: 1_700_000_000, + }) + expect(COLUMN_TYPE_REGISTRY.ttl.coerce(1_700_000_000, column)).toEqual({ + ok: true, + value: 1_700_000_000, + }) + expect(COLUMN_TYPE_REGISTRY.ttl.coerce('1700000000', column)).toEqual({ + ok: true, + value: 1_700_000_000, + }) + expect(COLUMN_TYPE_REGISTRY.ttl.coerce('2023-11-14T22:13:20.123Z', column)).toEqual({ + ok: true, + value: 1_700_000_000, + }) + expect(COLUMN_TYPE_REGISTRY.ttl.coerce('not-a-date', column)).toEqual({ ok: false }) + expect(COLUMN_TYPE_REGISTRY.ttl.coerce(1_700_000_000.5, column)).toEqual({ ok: false }) + }) + + it.each(['2023-02-29', '2023-02-29T12:00:00', '2023-02-29T12:00:00-05:00'])( + 'rejects a nonexistent ISO calendar input: %s', + (value) => { + expect(COLUMN_TYPE_REGISTRY.ttl.coerce(value, column, { timezone: 'UTC' })).toEqual({ + ok: false, + }) + } + ) + + it.each([ + ['2024-02-29', '2024-02-29T00:00:00Z'], + ['2024-02-29T12:00:00', '2024-02-29T12:00:00Z'], + ['2024-02-29T12:00:00-05:00', '2024-02-29T17:00:00Z'], + ])('accepts a valid leap-day ISO calendar input: %s', (value, expectedInstant) => { + expect(COLUMN_TYPE_REGISTRY.ttl.coerce(value, column, { timezone: 'UTC' })).toEqual({ + ok: true, + value: Math.floor(Date.parse(expectedInstant) / 1000), + }) + }) + + it('renders and edits epoch seconds as a date', () => { + expect(COLUMN_TYPE_REGISTRY.ttl.formatForDisplay(1_700_000_000, column)).toBe( + '11/14/2023 10:13:20 PM' + ) + expect(COLUMN_TYPE_REGISTRY.ttl.formatForInput(1_700_000_000, column)).toBe( + '2023-11-14T22:13:20Z' + ) + expect( + COLUMN_TYPE_REGISTRY.ttl.formatForInput(1_700_000_000, column, { + timezone: 'America/New_York', + }) + ).toBe('2023-11-14T17:13:20-05:00') + }) + + it('preserves the exact instant across both sides of a daylight-saving fold', () => { + expect( + COLUMN_TYPE_REGISTRY.ttl.formatForInput(1_699_162_200, column, { + timezone: 'America/New_York', + }) + ).toBe('2023-11-05T01:30:00-04:00') + expect( + COLUMN_TYPE_REGISTRY.ttl.formatForInput(1_699_165_800, column, { + timezone: 'America/New_York', + }) + ).toBe('2023-11-05T01:30:00-05:00') + }) + + it('limits a table to one ttl column', () => { + expect(COLUMN_TYPE_REGISTRY.ttl.maxPerTable).toBe(1) + }) +}) + describe('intentional divergences from the pre-registry behavior', () => { // A differential run of the registry against the pre-refactor implementations // (55 values x 7 column shapes) found ZERO coercion differences and exactly diff --git a/apps/sim/lib/table/__tests__/validation.test.ts b/apps/sim/lib/table/__tests__/validation.test.ts index 9d698c9d96b..ff39e690d87 100644 --- a/apps/sim/lib/table/__tests__/validation.test.ts +++ b/apps/sim/lib/table/__tests__/validation.test.ts @@ -195,6 +195,18 @@ describe('Validation', () => { expect(result.errors).toContain('Duplicate column names found') }) + it('rejects more than one TTL column', () => { + const result = validateTableSchema({ + columns: [ + { name: 'expires_at', type: 'ttl' }, + { name: 'delete_at', type: 'ttl' }, + ], + } as TableSchema) + + expect(result.valid).toBe(false) + expect(result.errors).toContain('A table can have at most 1 TTL column') + }) + it('should reject null schema', () => { const result = validateTableSchema(null as unknown as TableSchema) expect(result.valid).toBe(false) diff --git a/apps/sim/lib/table/column-types/registry.server.ts b/apps/sim/lib/table/column-types/registry.server.ts index 6ba27f5c616..5a6e23791ea 100644 --- a/apps/sim/lib/table/column-types/registry.server.ts +++ b/apps/sim/lib/table/column-types/registry.server.ts @@ -273,6 +273,7 @@ export const COLUMN_TYPE_SERVER_REGISTRY: Record = { number: numberColumnType, boolean: booleanColumnType, date: dateColumnType, + ttl: ttlColumnType, json: jsonColumnType, select: selectColumnType, currency: currencyColumnType, diff --git a/apps/sim/lib/table/column-types/ttl.test.ts b/apps/sim/lib/table/column-types/ttl.test.ts new file mode 100644 index 00000000000..db0984509db --- /dev/null +++ b/apps/sim/lib/table/column-types/ttl.test.ts @@ -0,0 +1,18 @@ +/** + * @vitest-environment node + */ + +import { describe, expect, it } from 'vitest' +import { retypeCellRewrite } from '@/lib/table/columns/service' +import type { ColumnDefinition } from '@/lib/table/types' + +const column = (over: Partial): ColumnDefinition => + ({ name: 'col', type: 'string', ...over }) as ColumnDefinition + +describe('TTL column type', () => { + it('converts epoch seconds to an ISO date before retyping', () => { + expect( + retypeCellRewrite(1_700_000_000, column({ type: 'date' }), column({ type: 'ttl' })) + ).toEqual({ value: '2023-11-14T22:13:20Z' }) + }) +}) diff --git a/apps/sim/lib/table/column-types/ttl.ts b/apps/sim/lib/table/column-types/ttl.ts new file mode 100644 index 00000000000..2303c852212 --- /dev/null +++ b/apps/sim/lib/table/column-types/ttl.ts @@ -0,0 +1,105 @@ +import { TypeTtl } from '@sim/emcn/icons' +import type { ColumnTypeDefinition } from '@/lib/table/column-types/types' +import { + formatDateCellDisplay, + formatInstantInTimeZone, + type NormalizeDateCellOptions, + normalizeDateCellValue, +} from '@/lib/table/dates' +import type { ColumnDefinition } from '@/lib/table/types' + +const NUMERIC_VALUE_PATTERN = /^-?\d+(?:\.\d+)?$/ +const ISO_DATE_PREFIX_PATTERN = /^(\d{4}-\d{2}-\d{2})(?:$|[T ])/i + +function isRepresentableEpochSeconds(value: number): boolean { + return Number.isSafeInteger(value) && !Number.isNaN(new Date(value * 1000).getTime()) +} + +/** Converts a TTL cell input to integer Unix epoch seconds. */ +export function parseTtlEpochSeconds( + value: unknown, + options?: NormalizeDateCellOptions +): number | null { + if (typeof value === 'number') return isRepresentableEpochSeconds(value) ? value : null + + if (value instanceof Date) { + const milliseconds = value.getTime() + return Number.isNaN(milliseconds) ? null : Math.floor(milliseconds / 1000) + } + + if (typeof value !== 'string') return null + const trimmed = value.trim() + if (!trimmed) return null + + if (NUMERIC_VALUE_PATTERN.test(trimmed)) { + const numeric = Number(trimmed) + return isRepresentableEpochSeconds(numeric) ? numeric : null + } + + const normalized = normalizeDateCellValue(trimmed, options) + if (normalized === null) return null + const instant = /^\d{4}-\d{2}-\d{2}$/.test(normalized) + ? normalizeDateCellValue(`${normalized}T00:00:00`, options) + : normalized + if (instant === null) return null + const inputIsoDate = trimmed.match(ISO_DATE_PREFIX_PATTERN)?.[1] + if (inputIsoDate && instant.slice(0, 10) !== inputIsoDate) return null + const milliseconds = Date.parse(instant) + if (Number.isNaN(milliseconds)) return null + const seconds = Math.floor(milliseconds / 1000) + return isRepresentableEpochSeconds(seconds) ? seconds : null +} + +function epochSecondsToIso(value: unknown): string | null { + const seconds = typeof value === 'number' ? value : Number(value) + if (!isRepresentableEpochSeconds(seconds)) return null + return new Date(seconds * 1000).toISOString().replace('.000Z', 'Z') +} + +function epochSecondsToEditable(value: unknown, timeZone?: string): string | null { + const iso = epochSecondsToIso(value) + if (!iso || !timeZone) return iso + return formatInstantInTimeZone(new Date(iso), timeZone) +} + +export const ttlColumnType: ColumnTypeDefinition = { + id: 'ttl', + label: 'TTL', + maxPerTable: 1, + icon: TypeTtl, + jsonbCast: 'numeric', + storesOpaqueIds: false, + supportsUnique: true, + sampleValue: 1_706_659_200, + ownedMetadata: [], + workflowInputType: 'number', + editor: 'date', + expandable: false, + typeaheadPattern: /[\d\-/]/, + parseErrorMessage: 'Invalid expiration date', + + coerce(value, _column, context) { + const seconds = parseTtlEpochSeconds(value, context) + return seconds === null ? { ok: false } : { ok: true, value: seconds } + }, + + valueForConversion(value, target: ColumnDefinition) { + if (target.type !== 'date') return value + return epochSecondsToIso(value) ?? value + }, + + validateCell(value, column) { + return typeof value === 'number' && isRepresentableEpochSeconds(value) + ? null + : `${column.name} must be valid epoch seconds` + }, + + formatForDisplay(value) { + const iso = epochSecondsToIso(value) + return iso === null ? String(value) : formatDateCellDisplay(iso, { seconds: true }) + }, + + formatForInput(value, _column, context) { + return epochSecondsToEditable(value, context?.timezone) ?? String(value) + }, +} diff --git a/apps/sim/lib/table/column-types/types.ts b/apps/sim/lib/table/column-types/types.ts index ad893acb809..72edeead0d0 100644 --- a/apps/sim/lib/table/column-types/types.ts +++ b/apps/sim/lib/table/column-types/types.ts @@ -37,6 +37,7 @@ export const COLUMN_TYPES = [ 'currency', 'boolean', 'date', + 'ttl', 'json', 'select', ] as const diff --git a/apps/sim/lib/table/columns/ttl-limit.test.ts b/apps/sim/lib/table/columns/ttl-limit.test.ts new file mode 100644 index 00000000000..30319a27843 --- /dev/null +++ b/apps/sim/lib/table/columns/ttl-limit.test.ts @@ -0,0 +1,74 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { TableDefinition, TableLocks } from '@/lib/table/types' + +const { mockTimeoutExecute, mockWithLockedTable } = vi.hoisted(() => ({ + mockTimeoutExecute: vi.fn(), + mockWithLockedTable: vi.fn(), +})) + +vi.mock('@/lib/table/service', () => ({ withLockedTable: mockWithLockedTable })) + +import { addTableColumn, updateColumnType } from '@/lib/table/columns/service' + +const UNLOCKED: TableLocks = { + schemaLocked: false, + insertLocked: false, + updateLocked: false, + deleteLocked: false, +} + +function makeTable(): TableDefinition { + return { + id: 'table-1', + name: 'Tasks', + schema: { + columns: [ + { id: 'col-name', name: 'name', type: 'string' }, + { id: 'col-ttl', name: 'expires_at', type: 'ttl' }, + ], + }, + rowCount: 0, + maxRows: 100, + workspaceId: 'workspace-1', + createdBy: 'user-1', + locks: UNLOCKED, + createdAt: new Date(), + updatedAt: new Date(), + } +} + +const transaction = new Proxy( + { execute: mockTimeoutExecute }, + { + get(target, property) { + if (property in target) return target[property as keyof typeof target] + throw new Error(`Unexpected transaction method: ${String(property)}`) + }, + } +) + +describe('TTL column mutation limit', () => { + beforeEach(() => { + vi.clearAllMocks() + mockTimeoutExecute.mockResolvedValue([]) + mockWithLockedTable.mockImplementation(async (_tableId, mutate) => + mutate(makeTable(), transaction) + ) + }) + + it('rejects adding a second TTL column before persistence', async () => { + await expect( + addTableColumn('table-1', { name: 'another_expiry', type: 'ttl' }, 'request-1') + ).rejects.toThrow('A table can have at most 1 TTL column') + }) + + it('rejects retyping another column to TTL before scanning cells', async () => { + await expect( + updateColumnType({ tableId: 'table-1', columnName: 'name', newType: 'ttl' }, 'request-1') + ).rejects.toThrow('A table can have at most 1 TTL column') + expect(mockTimeoutExecute).toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/table/dates.ts b/apps/sim/lib/table/dates.ts index a38eca7f21e..0c6360f63fb 100644 --- a/apps/sim/lib/table/dates.ts +++ b/apps/sim/lib/table/dates.ts @@ -164,6 +164,21 @@ function formatOffsetSuffix(offsetMinutes: number): string { return `${sign}${pad(Math.floor(abs / 60))}:${pad(abs % 60)}` } +/** Formats an instant as canonical wall time in an IANA timezone. */ +export function formatInstantInTimeZone(date: Date, timeZone: string): string { + const wall = getWallClockParts(date, timeZone) + const wallAsUtc = Date.UTC( + wall.year, + wall.month - 1, + wall.day, + wall.hour, + wall.minute, + wall.second + ) + const offsetMinutes = Math.round((wallAsUtc - date.getTime()) / 60_000) + return `${wall.year}-${pad(wall.month)}-${pad(wall.day)}T${pad(wall.hour)}:${pad(wall.minute)}:${pad(wall.second)}${formatOffsetSuffix(offsetMinutes)}` +} + /** * Trailing offset (minutes east of UTC) of a datetime string, or null when * naive. Recognizes exactly what `Date.parse` recognizes: numeric offsets, diff --git a/apps/sim/lib/table/import.test.ts b/apps/sim/lib/table/import.test.ts index d0aa04500e1..45463296049 100644 --- a/apps/sim/lib/table/import.test.ts +++ b/apps/sim/lib/table/import.test.ts @@ -172,6 +172,15 @@ describe('import', () => { ) expect(coerceValue('not-a-date', 'date')).toBe('not-a-date') }) + + it('coerces TTL imports to epoch seconds and preserves invalid input for row validation', () => { + expect(coerceValue('2023-11-14T22:13:20Z', 'ttl')).toBe(1_700_000_000) + expect(coerceValue('1700000000', 'ttl')).toBe(1_700_000_000) + expect(coerceValue('2023-11-14 17:13:20', 'ttl', { timezone: 'America/New_York' })).toBe( + 1_700_000_000 + ) + expect(coerceValue('not-a-date', 'ttl')).toBe('not-a-date') + }) }) describe('buildAutoMapping', () => { diff --git a/apps/sim/lib/table/import.ts b/apps/sim/lib/table/import.ts index d405fbb5e43..e1fa22d090c 100644 --- a/apps/sim/lib/table/import.ts +++ b/apps/sim/lib/table/import.ts @@ -15,6 +15,7 @@ import type { Options as CsvParseOptions } from 'csv-parse' import { OrchestrationError } from '@/lib/core/orchestration/types' import { getColumnId } from '@/lib/table/column-keys' import type { ColumnType } from '@/lib/table/column-types' +import { parseTtlEpochSeconds } from '@/lib/table/column-types/ttl' import { parseCurrencyInput } from '@/lib/table/currency' import { type NormalizeDateCellOptions, normalizeDateCellValue } from '@/lib/table/dates' import type { ColumnDefinition, RowData, TableSchema } from '@/lib/table/types' @@ -502,6 +503,9 @@ export function coerceValue( case 'date': { return normalizeDateCellValue(String(value), options) ?? String(value) } + case 'ttl': { + return parseTtlEpochSeconds(value, options) ?? String(value) + } case 'json': { if (typeof value === 'object') return value as Record | unknown[] try { diff --git a/docker/crontab b/docker/crontab index f945efafcca..9e3d3cead27 100644 --- a/docker/crontab +++ b/docker/crontab @@ -39,6 +39,9 @@ SHELL=/bin/sh # Enterprise data drains 0 * * * * curl -fsS -m 300 -o /dev/null -H "Authorization: Bearer $CRON_SECRET" "$SIM_URL/api/cron/run-data-drains" +# Deletes table rows whose TTL column has expired +*/5 * * * * curl -fsS -m 60 -o /dev/null -H "Authorization: Bearer $CRON_SECRET" "$SIM_URL/api/cron/cleanup-table-row-ttl" + # Microsoft Graph subscription renewal (Teams chat triggers expire after ~3 days) 0 */12 * * * curl -fsS -m 120 -o /dev/null -H "Authorization: Bearer $CRON_SECRET" "$SIM_URL/api/cron/renew-subscriptions" diff --git a/helm/sim/Chart.yaml b/helm/sim/Chart.yaml index 2b09f2bde3a..f675d1d7f6b 100644 --- a/helm/sim/Chart.yaml +++ b/helm/sim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: sim description: A Helm chart for Sim - the open-source AI workspace where teams build, deploy, and manage AI agents type: application -version: 1.6.2 +version: 1.6.3 appVersion: "v0.7.44" kubeVersion: ">=1.25.0-0" home: https://sim.ai diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index 785272fb8cb..9f97d5dbbb1 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -1454,6 +1454,16 @@ cronjobs: successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 1 + # Deletes table rows whose TTL column contains an expired Unix timestamp. + cleanupTableRowTtl: + enabled: true + name: cleanup-table-row-ttl + schedule: "*/5 * * * *" + path: "/api/cron/cleanup-table-row-ttl" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 1 + # Deletes prebuilt sandbox images that no workspace sandbox references and that # have gone unused past the retention window, from the provider and locally. # A no-op on deployments whose sandbox provider installs at run time. diff --git a/packages/emcn/src/icons/index.ts b/packages/emcn/src/icons/index.ts index 35fb8f96792..80a9efcbe2f 100644 --- a/packages/emcn/src/icons/index.ts +++ b/packages/emcn/src/icons/index.ts @@ -160,6 +160,7 @@ export { TypeCurrency } from './type-currency' export { TypeJson } from './type-json' export { TypeNumber } from './type-number' export { TypeText } from './type-text' +export { TypeTtl } from './type-ttl' export { Undo } from './undo' export { Unlink } from './unlink' export { Unlock } from './unlock' diff --git a/packages/emcn/src/icons/type-ttl.tsx b/packages/emcn/src/icons/type-ttl.tsx new file mode 100644 index 00000000000..3f7451ff1a7 --- /dev/null +++ b/packages/emcn/src/icons/type-ttl.tsx @@ -0,0 +1,22 @@ +import type { SVGProps } from 'react' + +export function TypeTtl(props: SVGProps) { + return ( + + ) +} diff --git a/packages/sim-cli/src/generated/v2-api.ts b/packages/sim-cli/src/generated/v2-api.ts index 2a5dd8f52b9..d8b09292932 100644 --- a/packages/sim-cli/src/generated/v2-api.ts +++ b/packages/sim-cli/src/generated/v2-api.ts @@ -181,7 +181,7 @@ export type AddTableColumnBody = { column: { id?: string name: string - type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'json' | 'select' + type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'ttl' | 'json' | 'select' required?: boolean unique?: boolean options?: Array<{ @@ -198,7 +198,7 @@ type AddTableColumnResponseRef0 = { columns: Array<{ id?: string name: string - type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'json' | 'select' + type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'ttl' | 'json' | 'select' required: boolean unique: boolean workflowGroupId?: string @@ -248,7 +248,7 @@ export type AddWorkflowGroupBody = { } outputColumns: Array<{ name: string - type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'json' | 'select' + type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'ttl' | 'json' | 'select' required?: boolean unique?: boolean }> @@ -283,7 +283,7 @@ type AddWorkflowGroupResponseRef1 = { columns: Array<{ id?: string name: string - type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'json' | 'select' + type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'ttl' | 'json' | 'select' required: boolean unique: boolean workflowGroupId?: string @@ -1874,7 +1874,7 @@ export type CreateTableBody = { columns: Array<{ id?: string name: string - type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'json' | 'select' + type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'ttl' | 'json' | 'select' required?: boolean unique?: boolean options?: Array<{ @@ -1906,7 +1906,7 @@ type CreateTableResponseRef1 = { columns: Array<{ id?: string name: string - type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'json' | 'select' + type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'ttl' | 'json' | 'select' required: boolean unique: boolean workflowGroupId?: string @@ -2858,7 +2858,7 @@ type DeleteTableColumnResponseRef0 = { columns: Array<{ id?: string name: string - type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'json' | 'select' + type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'ttl' | 'json' | 'select' required: boolean unique: boolean workflowGroupId?: string @@ -3119,7 +3119,7 @@ type DeleteWorkflowGroupResponseRef0 = { columns: Array<{ id?: string name: string - type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'json' | 'select' + type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'ttl' | 'json' | 'select' required: boolean unique: boolean workflowGroupId?: string @@ -4272,7 +4272,7 @@ type GetTableResponseRef1 = { columns: Array<{ id?: string name: string - type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'json' | 'select' + type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'ttl' | 'json' | 'select' required: boolean unique: boolean workflowGroupId?: string @@ -6028,7 +6028,7 @@ type ListTablesResponseRef0 = { columns: Array<{ id?: string name: string - type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'json' | 'select' + type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'ttl' | 'json' | 'select' required: boolean unique: boolean workflowGroupId?: string @@ -7232,7 +7232,7 @@ type RestoreTableResponseRef1 = { columns: Array<{ id?: string name: string - type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'json' | 'select' + type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'ttl' | 'json' | 'select' required: boolean unique: boolean workflowGroupId?: string @@ -8372,7 +8372,7 @@ type UpdateTableResponseRef1 = { columns: Array<{ id?: string name: string - type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'json' | 'select' + type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'ttl' | 'json' | 'select' required: boolean unique: boolean workflowGroupId?: string @@ -8414,7 +8414,7 @@ export type UpdateTableColumnBody = { columnName: string updates: { name?: string - type?: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'json' | 'select' + type?: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'ttl' | 'json' | 'select' required?: boolean unique?: boolean options?: Array<{ @@ -8430,7 +8430,7 @@ type UpdateTableColumnResponseRef0 = { columns: Array<{ id?: string name: string - type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'json' | 'select' + type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'ttl' | 'json' | 'select' required: boolean unique: boolean workflowGroupId?: string @@ -8693,7 +8693,7 @@ export type UpdateWorkflowGroupBody = { }> newOutputColumns?: Array<{ name: string - type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'json' | 'select' + type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'ttl' | 'json' | 'select' required?: boolean unique?: boolean }> @@ -8739,7 +8739,7 @@ type UpdateWorkflowGroupResponseRef1 = { columns: Array<{ id?: string name: string - type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'json' | 'select' + type: 'string' | 'number' | 'currency' | 'boolean' | 'date' | 'ttl' | 'json' | 'select' required: boolean unique: boolean workflowGroupId?: string diff --git a/scripts/check-api-validation-contracts.ts b/scripts/check-api-validation-contracts.ts index a5fb0b221f9..e3e7f759587 100644 --- a/scripts/check-api-validation-contracts.ts +++ b/scripts/check-api-validation-contracts.ts @@ -86,6 +86,7 @@ const INDIRECT_ZOD_ROUTES = new Set([ 'apps/sim/app/api/settings/allowed-mcp-domains/route.ts', 'apps/sim/app/api/cron/cleanup-tasks/route.ts', 'apps/sim/app/api/cron/cleanup-soft-deletes/route.ts', + 'apps/sim/app/api/cron/cleanup-table-row-ttl/route.ts', 'apps/sim/app/api/cron/cleanup-stale-executions/route.ts', 'apps/sim/app/api/cron/cleanup-sandbox-images/route.ts', 'apps/sim/app/api/cron/renew-subscriptions/route.ts',