From 8efb1dd6b7da21e8bbb7c02f3d64762b7b78c3ff Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Wed, 9 Sep 2026 15:52:58 -0700 Subject: [PATCH 1/3] feat(search): add GitHub installation indexing --- .../docs/search/connect-your-account.mdx | 1 + apps/docs/content/docs/search/github.mdx | 80 ++- apps/docs/content/docs/search/index.mdx | 9 +- apps/sim/.env.example | 11 + .../github/installations/route.test.ts | 223 ++++++++ .../knowledge/github/installations/route.ts | 53 ++ .../add-connector-modal.test.tsx | 57 ++- .../add-connector-modal.tsx | 32 +- .../github-installation-modal.test.tsx | 199 ++++++++ .../components/github-installation-modal.tsx | 220 ++++++++ .../components/search-source-setup.test.tsx | 2 +- apps/sim/connectors/github/github.test.ts | 48 ++ apps/sim/connectors/github/github.ts | 34 +- .../organization-account-providers.test.tsx | 113 +++- .../organization-account-providers.tsx | 54 +- .../github-search-installations.test.tsx | 114 +++++ .../queries/github-search-installations.ts | 53 ++ .../knowledge/github-installations.test.ts | 48 ++ .../knowledge/github-installations.ts | 79 +++ apps/sim/lib/core/config/env.ts | 3 + .../application/provider-catalog.ts | 11 + .../lib/credentials/service-account-secret.ts | 6 + .../credential-visibility.server.ts | 10 + .../github-member.integration.ts | 232 ++++++++- .../access/github-installation.test.ts | 247 +++++++++ .../knowledge/access/github-installation.ts | 297 +++++++++++ .../access/predicate.postgres.test.ts | 119 ++++- apps/sim/lib/knowledge/access/predicate.ts | 75 +++ apps/sim/lib/knowledge/access/scope.test.ts | 42 +- apps/sim/lib/knowledge/access/scope.ts | 47 +- apps/sim/lib/knowledge/access/types.ts | 10 + .../knowledge/application/connector-access.ts | 16 +- .../lib/knowledge/application/connectors.ts | 26 +- .../sim/lib/knowledge/application/contexts.ts | 20 +- .../github-installation-source.test.ts | 153 ++++++ .../application/github-installation-source.ts | 84 +++ .../github-installations.postgres.test.ts | 144 ++++++ .../application/github-installations.test.ts | 208 ++++++++ .../application/github-installations.ts | 242 +++++++++ .../lib/knowledge/application/operations.ts | 18 + apps/sim/lib/knowledge/application/search.ts | 12 +- .../lib/knowledge/chunks/keyset-sql.test.ts | 3 +- .../knowledge/connectors/access-token.test.ts | 18 + .../lib/knowledge/connectors/access-token.ts | 16 +- .../orchestration/connector-access.ts | 10 + .../lib/knowledge/orchestration/connectors.ts | 11 +- apps/sim/lib/oauth/credential-service.ts | 45 ++ .../github-installation-credential.test.ts | 99 ++++ .../lib/oauth/github-installation-types.ts | 23 + .../sim/lib/oauth/github-installation.test.ts | 331 ++++++++++++ apps/sim/lib/oauth/github-installation.ts | 481 ++++++++++++++++++ apps/sim/lib/oauth/github-repository.test.ts | 42 ++ apps/sim/lib/oauth/github-repository.ts | 19 + apps/sim/lib/oauth/oauth.ts | 2 + ...check-tool-registry-boundary.baseline.json | 14 +- 55 files changed, 4453 insertions(+), 113 deletions(-) create mode 100644 apps/sim/app/api/knowledge/github/installations/route.test.ts create mode 100644 apps/sim/app/api/knowledge/github/installations/route.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/search/components/github-installation-modal.test.tsx create mode 100644 apps/sim/app/workspace/[workspaceId]/search/components/github-installation-modal.tsx create mode 100644 apps/sim/hooks/queries/github-search-installations.test.tsx create mode 100644 apps/sim/hooks/queries/github-search-installations.ts create mode 100644 apps/sim/lib/api/contracts/knowledge/github-installations.test.ts create mode 100644 apps/sim/lib/api/contracts/knowledge/github-installations.ts create mode 100644 apps/sim/lib/knowledge/access/github-installation.test.ts create mode 100644 apps/sim/lib/knowledge/access/github-installation.ts create mode 100644 apps/sim/lib/knowledge/application/github-installation-source.test.ts create mode 100644 apps/sim/lib/knowledge/application/github-installation-source.ts create mode 100644 apps/sim/lib/knowledge/application/github-installations.postgres.test.ts create mode 100644 apps/sim/lib/knowledge/application/github-installations.test.ts create mode 100644 apps/sim/lib/knowledge/application/github-installations.ts create mode 100644 apps/sim/lib/oauth/github-installation-credential.test.ts create mode 100644 apps/sim/lib/oauth/github-installation-types.ts create mode 100644 apps/sim/lib/oauth/github-installation.test.ts create mode 100644 apps/sim/lib/oauth/github-installation.ts create mode 100644 apps/sim/lib/oauth/github-repository.test.ts create mode 100644 apps/sim/lib/oauth/github-repository.ts diff --git a/apps/docs/content/docs/search/connect-your-account.mdx b/apps/docs/content/docs/search/connect-your-account.mdx index 26508b493a7..025f7414568 100644 --- a/apps/docs/content/docs/search/connect-your-account.mdx +++ b/apps/docs/content/docs/search/connect-your-account.mdx @@ -56,6 +56,7 @@ For a source configured inside a workspace, join that workspace and use its **Se | Source setup | Your next step | | --- | --- | | Member accounts | Connect your own account, including when you are the admin. | +| GitHub App installation | Connect GitHub once for this Sim organization. The App handles indexing; your account establishes which repositories you may search. | | Confluence admin/service account | Connect Confluence to verify your identity; the administrator's account handles the crawl. | | Google Drive delegated service account | No personal connection is needed for that source. Your verified Sim email is matched to Drive permissions. | | GitLab instance administrator | No personal connection is needed. Your verified Sim email must match a confirmed GitLab email. | diff --git a/apps/docs/content/docs/search/github.mdx b/apps/docs/content/docs/search/github.mdx index 1bcc37159b6..7a390dcb017 100644 --- a/apps/docs/content/docs/search/github.mdx +++ b/apps/docs/content/docs/search/github.mdx @@ -1,24 +1,28 @@ --- title: GitHub -description: Search repository files through each member's GitHub account +description: Index repository files with a GitHub App while preserving each person's access --- import { Callout } from 'fumadocs-ui/components/callout' import { Step, Steps } from 'fumadocs-ui/components/steps' import { Image } from '@/components/ui/image' -GitHub Search indexes text files from a repository on `github.com`. An organization admin chooses the repository, then each person connects their GitHub account. Installing the GitHub App alone does not connect your teammates. +GitHub Search indexes text files from repositories on `github.com`. An organization admin can install the GitHub App once and use it to index selected repositories. Each person connects their own GitHub account once to search the repositories they can access. Installing the App does not connect teammates or give them the installer's permissions. -Admin setup uses your organization's **Settings → Sources** page. Teammates connect from **Integrations** in the main sidebar. For workspace Search, use **Search → Add source** instead; **Create & Invite** is the workspace equivalent of **Add source**. +Admin setup uses your organization's **Settings → Sources** page. Teammates connect from **Integrations** in the main sidebar. Installation indexing is available for organization Search. For workspace Search, use **Search → Add source** with member accounts or a dedicated user account; **Create & Invite** is the workspace equivalent of **Add source**. ## Before you start -Your Sim deployment needs a GitHub App configured as described below. The repository must contain at least one commit; initialize an empty repository with a README before adding it. For private repositories, an owner or administrator must install that App on them. Each person needs a verified GitHub email address matching their Sim account; the address can be private or secondary. +Your Sim deployment needs a GitHub App configured as described below. The repository must contain at least one commit. Each person needs a verified GitHub email address matching their Sim account; the address can be private or secondary. + +To connect an installation for central indexing, you must be a Sim organization admin and either own the GitHub personal account or be an owner of the GitHub organization where the App is installed. You must also be able to read the repository you add. ## Configure the GitHub App This step belongs to the Sim deployment administrator. If the App is already configured, continue to [Add a repository](#add-a-repository). +Create separate GitHub Apps for production and staging. Each deployment uses its own App, credentials, private key, and callback URL. This keeps test installations and authorizations separate from production. + @@ -31,9 +35,11 @@ Give the App a unique, recognizable name, such as **Your Company Sim Search**, a Under **Identifying and authorizing users → Redirect URI (callback URL)**, enter: ```text -https:///api/auth/oauth2/callback/github-repositories +/api/auth/oauth2/callback/github-repositories ``` +Replace `` with that deployment's configured public origin, without a trailing slash. For example, `https://sim.example.com` and `https://staging.sim.example.com` need different callbacks on their respective Apps. The scheme, hostname, port, and path must match exactly; `www` and non-`www` hosts are different. See GitHub's [callback matching rules](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/about-the-user-authorization-callback-url). + | GitHub setting | Value for Sim Search | |---|---| | Allow wildcard matching | Disabled | @@ -72,25 +78,38 @@ Leave every other permission at **No access**. Sim does not need issue, pull-req GitHub App user tokens use these permissions rather than OAuth scopes. An empty `scope` value in the token response is expected; see GitHub's [user token reference](https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/generating-a-user-access-token-for-a-github-app). -Under **Where can this GitHub App be installed?**, choose **Only on this account** for an organization-owned App used only by members of that organization. Choose **Any account** when teammates or repository owners are outside that organization, or the App is owned by your personal account. A private App owned by a personal account can only be authorized by its owner; see GitHub's [App visibility rules](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/making-a-github-app-public-or-private). +For production, set **Where can this GitHub App be installed? → Any account**. This lets any GitHub account install and authorize the App, subject to that account's organization policies. Making the App public does not make repositories public or grant anyone Search access. See GitHub's [App visibility rules](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/making-a-github-app-public-or-private). + +Use **Any account** for staging too if testing with accounts outside the App owner's organization. **Only on this account** restricts installations to the owner and authorization to that organization's members; a private personal App only works for its owner. Select **Create GitHub App**. -### Configure Sim and install the App +### Configure Sim + +On the App's **General** settings page, copy its numeric **App ID** and **Client ID**, select **Generate a new client secret**, and generate a **Private key**. The App slug is the final part of its public URL: `https://github.com/apps/`. -On the App's **General** settings page, copy its **Client ID**, then select **Generate a new client secret**. Configure these deployment variables and restart Sim: +Set all five variables for each deployment, using values from that deployment's App, then restart Sim: ```text +GITHUB_APP_ID= +GITHUB_APP_SLUG= GITHUB_APP_CLIENT_ID= GITHUB_APP_CLIENT_SECRET= +GITHUB_APP_PRIVATE_KEY= ``` -Use the **Client ID** and **client secret** from **Developer settings → GitHub Apps**. OAuth App credentials used for GitHub sign-in are not compatible. The numeric **App ID** and downloaded private key are not used by this connector. Keep expiring user tokens enabled so Sim can [refresh them](https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/refreshing-user-access-tokens). +The private key must include its PEM header, footer, and contents. Sim accepts actual newlines or escaped `\n` sequences. Keep the private key and client secret in the deployment's server configuration; organization admins select installations in Sim without entering these secrets. + +The **Client ID** is different from the numeric **App ID**. Use credentials from **Developer settings → GitHub Apps**. `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` belong to the separate GitHub sign-in integration and remain unchanged. Search does not read `GITHUB_REPO_CLIENT_ID` or `GITHUB_REPO_CLIENT_SECRET`. -In the App's sidebar, choose **Install App**, select the target account, and grant access to the repositories you want to search. Return to Sim and select **Connect account**. Every teammate must authorize from Sim too. +Keep **Expire user authorization tokens** enabled so Sim receives the refresh token it needs to [renew personal connections](https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/refreshing-user-access-tokens). + +Complete the installation from the Sim source setup below. + +If you replace a deployment's GitHub App, an organization admin must first open **Settings → Connected accounts → Providers → Update configurations**. This applies the deployment's current App configuration to the existing providers while preserving their saved identities. Accounts whose App configuration changed must reconnect. Then reconnect personal GitHub accounts and connect an installation of the new App. Reconnecting alone cannot update the organization's saved App configuration. @@ -107,11 +126,26 @@ Open **Settings → Sources** and turn on **GitHub** under **Allowed in Sim Sear +### Choose how to index + +In **Sync documents with**, choose **Connect GitHub App** to index through an installation: + +1. Select **Connect your GitHub account** if prompted. Finish authorization in the new tab, then return and select **Refresh**. +2. Select **Install GitHub App**. Choose your GitHub account or organization and the repositories to include. If it is already installed, check its repository selection. +3. Return to Sim and select **Refresh**. Choose the installation and select **Use installation**. Only installations on your own account or organizations you own are available. + +The installation is now selected under **Sync documents with**. You can reuse it when adding another repository source in the same Sim organization. + +Alternatively, leave **Connected members** selected to use members' accounts for indexing, or select an existing dedicated account. Each method still requires teammates to connect their own accounts for Search access. + + + + ### Choose what to index -Enter **Repository** as `owner/repo`. Open **More options** only if you need a different branch, path or extension filters, metadata tags, or a dedicated indexing account. **Sync documents with** defaults to **Connected members**. +Enter **Repository** as `owner/repo`. For installation indexing, it must belong to the installation's account and be included in the repositories granted to the App. Add one source per repository; installing on all repositories does not automatically create sources for them. -GitHub source setup with a required Repository field and More options +Open **More options** if you need a different branch, path or extension filters, or metadata tags. | Field | What to enter | |---|---| @@ -122,8 +156,6 @@ Enter **Repository** as `owner/repo`. Open **More options** only if you need a d **Metadata tags** controls the metadata stored with results. Its defaults are suitable for most sources. Select **Add source** to save the source. -You can instead select an existing account under **Sync documents with** to supply file contents centrally. Teammates still connect their own accounts to establish which files they may find. - @@ -131,24 +163,40 @@ You can instead select an existing account under **Sync documents with** to supp Open **Integrations** in the main sidebar, select **Connect account** on the GitHub source, and authorize the App. Teammates repeat this step after joining the Sim organization. For private repositories, both the person's account and the App installation must have access. GitHub also permits App user tokens to read public repositories without an installation; see [GitHub's permission rules](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/choosing-permissions-for-a-github-app). -With **Connected members**, indexing begins after someone connects. A dedicated indexing account can start syncing immediately; each teammate still connects before searching. Open **Settings → Sources**, select **Manage** beside **GitHub**, then open the source to inspect **Documents**, edit **Settings**, or review **Sync history**. +An App installation or dedicated indexing account can start syncing after the source is saved. With **Connected members**, indexing begins after someone connects. Each teammate still connects before searching. An existing GitHub connection in the same Sim organization is reused across its GitHub sources. + +Open **Settings → Sources**, select **Manage** beside **GitHub**, then open the source to inspect **Documents**, edit **Settings**, or review **Sync history**. Use **GitHub → Accounts → Request connections** to send provider-specific connection requests. These requests do not grant organization membership. For another repository, add another source; members can also use **Add another GitHub source** in the main Integrations page. +## How access is enforced + +GitHub App installation access supplies file contents for indexing. Each reader's own connected GitHub account determines which repository's indexed content they can search. Sim organization admins follow the same rule as other readers. + +For installation-indexed sources, Sim checks the installation's current status and verifies repository content access with the reader's GitHub account before returning results or opening indexed content. If GitHub cannot confirm access, that repository's content is withheld. Removing a person's repository access, disconnecting their account, or removing the repository from the App's access prevents subsequent reads once GitHub reflects the change. File edits still appear after background indexing. + +This is an installation plus personal authorization flow. GitHub Search does not impersonate everyone in an email domain. [Google Drive delegation and GitLab administrator indexing](/search#choose-the-right-connection-method) use different supported identity and permission models. + ## Troubleshooting | Problem | Next step | |---|---| | GitHub is unavailable in Search | Ask the deployment admin to configure the App client credentials and enable member connections. | +| GitHub App indexing is unavailable | Configure all five `GITHUB_APP_*` variables above. Confirm the App ID, slug, client credentials, and valid RSA PEM key all belong to the same App. | | GitHub rejects `redirect_uri` | Register the exact callback on the GitHub App whose Client ID Sim uses: `http://localhost:3000/api/auth/oauth2/callback/github-repositories` for the default local server, or your production Sim origin followed by `/api/auth/oauth2/callback/github-repositories`. The scheme, host, port, and path must match; keep wildcard matching disabled. See [callback matching](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/about-the-user-authorization-callback-url). | | Wrong app credentials | Copy the Client ID and client secret from **GitHub Apps**, not **OAuth Apps**. Set `GITHUB_APP_CLIENT_ID` and `GITHUB_APP_CLIENT_SECRET`, then restart Sim. A numeric App ID or private-key file cannot replace them. | | Repository cannot be read | Confirm the App is installed on that repository and your GitHub account has access. For SAML organizations, establish your GitHub SSO session before reconnecting. | -| A teammate cannot authorize the App | Check **Where can this GitHub App be installed?** and the App owner. A private organization App accepts only organization members; a private personal App accepts only its owner. | +| A teammate cannot authorize the App | Set the production App to **Any account**. For a restricted staging App, confirm the tester meets its visibility rules. Check the GitHub organization's App policies and required approvals. | +| No eligible installations found | Finish connecting your own GitHub account, install the configured App on your account or an organization you own, then select **Refresh**. An installation of a different App or one you only have repository access to cannot be selected. | +| Repository is not accepted for an installation | Check `owner/repo`, the installation's account and repository selection, and your own access. Update the source's Repository field after a rename. After a transfer, add a source using an installation for the new owner. | | Identity verification fails | Verify the email used by your Sim account in GitHub's email settings, then reconnect. A public profile email alone is insufficient. | | Authorization fails after installation | Return to Sim and start **Connect account** there. Do not enable authorization during installation. | +| Account authorization did not complete | Start the connection again from Sim. If it repeats, verify the exact callback, matching GitHub App client credentials, **Email addresses: Read-only**, and enabled expiring user tokens. The deployment admin can inspect server logs for the underlying failure. | +| Update GitHub in Connected accounts before connecting this source | An organization admin must select **Settings → Connected accounts → Providers → Update configurations**, then reconnect GitHub. This is required when the deployment's GitHub App has changed. | +| Indexed files no longer appear | Confirm your own repository access, App repository selection, and connection status. Installation-indexed content is also withheld when GitHub cannot verify current access; retry once GitHub is available. | | Sync is incomplete | Review the source status. Very large Git trees, file size limits, and unreadable files can limit indexing. | | Empty repository returns an error | Add an initial commit, then sync again. GitHub does not return a file tree for an uninitialized repository. | diff --git a/apps/docs/content/docs/search/index.mdx b/apps/docs/content/docs/search/index.mdx index 3673675951a..cfbd2812709 100644 --- a/apps/docs/content/docs/search/index.mdx +++ b/apps/docs/content/docs/search/index.mdx @@ -45,17 +45,18 @@ Source availability depends on the deployment and organization policy. An unavai ## Choose the right connection method -Most sources use member accounts. Google Drive and Confluence also support a central administrator connection; GitLab requires an administrator token for a self-managed instance. +Most sources use member accounts. GitHub supports a central App installation with a personal connection for each reader. Google Drive and Confluence also support a central administrator connection; GitLab requires an administrator token for a self-managed instance. | Method | What the admin does | What teammates do | | --- | --- | --- | | **Member accounts** | Sets the source's filters once. | Connect their own accounts. Sim lists documents using each member's access. | +| **GitHub App installation** | Installs the App, selects it under **Sync documents with**, and adds repository sources. | Connect GitHub once. Sim checks each reader's current repository access before returning installation-indexed content. | | **Service account** (Drive) / **Admin or service account** (Confluence) | Connects an account that can read the content and the source's permissions or directory. | Join the organization with a matching verified identity. Confluence also requires each person to connect their account. | | **Administrator token** (GitLab) | Connects a self-managed instance administrator token and selects projects to index. | Join the organization with a verified Sim email matching GitLab. No personal connection is needed. | Adding a Google Drive or Confluence source from the admin page starts central setup. For personal connections, use **Integrations → Connect account** in the main sidebar. An approved provider can create its first member source there; required repository, site, or project fields are collected before authorization. Admins can edit that source's filters afterward in its **Settings** tab. -Some member sources offer **More options → Sync documents with**. **Connected members** uses members' accounts for both content and access checks. Selecting a dedicated account uses it to fetch content; members still connect to establish which documents they may search. **Account for browsing** only helps an admin pick source options—it does not enroll that account for Search. +Some member sources offer **Sync documents with**, either directly in setup or under **More options**. **Connected members** uses members' accounts for both content and access checks. Selecting a dedicated account uses it to fetch content; members still connect to establish which documents they may search. For GitHub organization sources, choose **Connect GitHub App** in this field to [connect an installation](/search/github#add-a-repository). **Account for browsing** only helps an admin pick source options—it does not enroll that account for Search. An administrator connection does not grant everyone access to everything. Search applies the source's supported permission rules. It also does not automatically discover every employee's data: the indexing account must be able to read the configured content. @@ -66,7 +67,7 @@ Some member sources offer **More options → Sync documents with**. **Connected | Source | Content | Connection in Search | | --- | --- | --- | | [Confluence](/search/confluence) | Pages and blog posts | Admin/service account or member accounts; each teammate connects | -| [GitHub](/search/github) | Repository text files | GitHub App installation plus each member's authorization | +| [GitHub](/search/github) | Repository text files | App installation or member indexing; each teammate connects | | [GitLab](/search/gitlab) | Repository files, wikis, issues, merge requests | Self-managed instance administrator token; no member connection | | [Gmail](/search/gmail) | Email thread text | Each member's Gmail account | | [Google Calendar](/search/google-calendar) | Calendar events | Each member's Google Calendar account | @@ -133,7 +134,7 @@ Workspace Search remains separate. Workspace admins add sources through **Search 3. Ask a teammate with different source access to repeat the search. Documents restricted to you should not appear for them. 4. Change or remove a test document's access in the source and check again after the next completed content and permission refresh. -Search runs background syncs on an hourly schedule. Large sources, provider limits, and indexing queues can delay completion. Results are indexed copies, so edits and access changes are not fetched live for every query. +Search runs background syncs on an hourly schedule. Large sources, provider limits, and indexing queues can delay completion. Results are indexed copies, so edits appear after syncing. Permission refresh behavior depends on the connector: GitHub sources indexed through an App installation also check the reader's current repository access before returning indexed content. ## If indexing needs attention diff --git a/apps/sim/.env.example b/apps/sim/.env.example index f00c68f8094..4f2b7a58cee 100644 --- a/apps/sim/.env.example +++ b/apps/sim/.env.example @@ -143,6 +143,17 @@ CRON_SECRET=your_cron_secret # Use `openssl rand -hex 32` to generate. Authentic # S3_ENDPOINT= # Custom endpoint for S3-compatible storage (Cloudflare R2, MinIO, Backblaze B2). Leave unset for AWS S3 # S3_FORCE_PATH_STYLE=true # Required for MinIO/Ceph RGW. Leave unset for AWS S3 and R2 +# GitHub Search (Optional - credentials from a GitHub App with expiring user tokens) +# Use a separate app for production and staging. Allow installation by any account for a public app. +# Grant repository Contents: read and Metadata: read, plus user Email addresses: read. +# Callback: /api/auth/oauth2/callback/github-repositories +# GITHUB_APP_CLIENT_ID= # GitHub App client ID; distinct from sign-in OAuth credentials +# GITHUB_APP_CLIENT_SECRET= +# Optional organization indexing through an app installation; readers still connect their own GitHub account. +# GITHUB_APP_ID= # Numeric GitHub App ID +# GITHUB_APP_SLUG= # App slug from https://github.com/apps/ +# GITHUB_APP_PRIVATE_KEY= # RSA PEM private key; literal \\n sequences are accepted + # Instagram OAuth (Optional - Instagram App ID/Secret from Meta App Dashboard > Instagram > API setup with Instagram login) # INSTAGRAM_CLIENT_ID= # INSTAGRAM_CLIENT_SECRET= diff --git a/apps/sim/app/api/knowledge/github/installations/route.test.ts b/apps/sim/app/api/knowledge/github/installations/route.test.ts new file mode 100644 index 00000000000..1352bd78276 --- /dev/null +++ b/apps/sim/app/api/knowledge/github/installations/route.test.ts @@ -0,0 +1,223 @@ +/** + * @vitest-environment node + */ +import { authMockFns } from '@sim/testing' +import { NextRequest, NextResponse } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ list: vi.fn(), connect: vi.fn(), rateLimit: vi.fn() })) + +vi.mock('@/lib/core/rate-limiter', () => ({ + enforceUserRateLimit: mocks.rateLimit, + RateLimiter: class {}, +})) +vi.mock('@/lib/knowledge/application/github-installations', () => ({ + listGitHubSearchInstallations: { + operation: { id: 'knowledge.github.installations.list' }, + execute: mocks.list, + }, + connectGitHubSearchInstallation: { + operation: { id: 'knowledge.github.installations.connect' }, + execute: mocks.connect, + }, +})) +vi.mock('@/lib/oauth/github-installation', () => ({ + GitHubInstallationError: class extends Error { + constructor( + message: string, + readonly status?: number + ) { + super(message) + } + }, +})) +vi.mock('@/lib/credentials/managed-oauth', () => ({ + ManagedOAuthCredentialError: class extends Error { + constructor( + readonly code: string, + message: string, + readonly statusCode: number + ) { + super(message) + } + }, +})) + +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { ManagedOAuthCredentialError } from '@/lib/credentials/managed-oauth' +import { GitHubInstallationError } from '@/lib/oauth/github-installation' +import { GET, POST } from '@/app/api/knowledge/github/installations/route' + +const URL = 'http://localhost/api/knowledge/github/installations' +const installation = { + installationId: '123', + accountId: '456', + accountLogin: 'acme', + accountType: 'Organization', +} + +beforeEach(() => { + vi.clearAllMocks() + authMockFns.mockGetSession.mockResolvedValue({ + user: { id: 'admin-1' }, + session: { id: 'session-1' }, + }) + mocks.rateLimit.mockResolvedValue(null) + mocks.list.mockResolvedValue({ + available: true, + installUrl: 'https://github.com/apps/sim-search/installations/new', + needsUserConnection: false, + installations: [installation], + }) + mocks.connect.mockResolvedValue({ credential: { id: 'cred-1', displayName: 'GitHub · acme' } }) +}) + +describe('GitHub installation route boundary', () => { + it.each(['GET', 'POST'] as const)( + 'authenticates %s before parsing or calling the use case', + async (method) => { + authMockFns.mockGetSession.mockResolvedValue(null) + const request = new NextRequest(URL, method === 'POST' ? { method, body: '{' } : undefined) + const json = vi.spyOn(request, 'json') + const response = await (method === 'GET' ? GET(request) : POST(request)) + expect(response.status).toBe(401) + expect(response.headers.get('Cache-Control')).toBe('private, no-store') + expect(json).not.toHaveBeenCalled() + expect(mocks.rateLimit).not.toHaveBeenCalled() + expect(mocks.list).not.toHaveBeenCalled() + expect(mocks.connect).not.toHaveBeenCalled() + } + ) + + it('applies admission before parsing the POST body', async () => { + mocks.rateLimit.mockResolvedValue( + NextResponse.json({ error: 'Rate limit exceeded' }, { status: 429 }) + ) + const request = new NextRequest(URL, { method: 'POST', body: '{' }) + const json = vi.spyOn(request, 'json') + expect((await POST(request)).status).toBe(429) + expect(json).not.toHaveBeenCalled() + expect(mocks.connect).not.toHaveBeenCalled() + expect(mocks.rateLimit).toHaveBeenCalledWith( + 'github-search-installations', + 'admin-1', + undefined + ) + }) + + it.each(['0', '-1', '1.5', '123/path', ''])( + 'rejects invalid installation ID %s before the use case', + async (installationId) => { + const response = await POST( + new NextRequest(URL, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ organizationId: 'org-1', installationId }), + }) + ) + expect(response.status).toBe(400) + expect(mocks.connect).not.toHaveBeenCalled() + } + ) + + it('requires organization scope for GET', async () => { + expect((await GET(new NextRequest(URL))).status).toBe(400) + expect(mocks.list).not.toHaveBeenCalled() + }) + + it('forwards GET identity and cancellation and projects a private installation list', async () => { + const controller = new AbortController() + const request = new NextRequest(`${URL}?organizationId=org-1`, { signal: controller.signal }) + mocks.list.mockResolvedValue({ + available: true, + installUrl: 'https://github.com/apps/sim-search/installations/new', + needsUserConnection: false, + installations: [{ ...installation, accessToken: 'private' }], + privateKey: 'private', + }) + const response = await GET(request) + expect(response.status).toBe(200) + expect(response.headers.get('Cache-Control')).toBe('private, no-store') + expect(mocks.list).toHaveBeenCalledWith( + expect.objectContaining({ + principal: { kind: 'session', userId: 'admin-1', sessionId: 'session-1' }, + input: { organizationId: 'org-1', signal: request.signal }, + }) + ) + expect(await response.json()).toEqual({ + success: true, + available: true, + installUrl: 'https://github.com/apps/sim-search/installations/new', + needsUserConnection: false, + installations: [installation], + }) + }) + + it('forwards POST cancellation and only returns the safe credential projection', async () => { + const request = new NextRequest(URL, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ organizationId: 'org-1', installationId: '123' }), + }) + mocks.connect.mockResolvedValue({ + credential: { + id: 'cred-1', + displayName: 'GitHub · acme', + encryptedServiceAccountKey: 'private', + }, + created: true, + }) + const response = await POST(request) + expect(response.status).toBe(200) + expect(response.headers.get('Cache-Control')).toBe('private, no-store') + expect(mocks.connect).toHaveBeenCalledWith( + expect.objectContaining({ + input: { organizationId: 'org-1', installationId: '123', signal: request.signal }, + }) + ) + expect(await response.json()).toEqual({ + success: true, + credential: { id: 'cred-1', displayName: 'GitHub · acme' }, + }) + }) + + it.each([ + [ + new OrchestrationError('forbidden', 'Organization administrator access is required'), + 403, + 'Organization administrator access is required', + ], + [ + new GitHubInstallationError('Installation permission denied', 403), + 403, + 'Installation permission denied', + ], + [ + new GitHubInstallationError('GitHub is temporarily unavailable', 503), + 502, + 'GitHub is temporarily unavailable', + ], + [ + new ManagedOAuthCredentialError( + 'MANAGED_CREDENTIAL_NEEDS_REAUTH', + 'private refresh details', + 401 + ), + 401, + 'Reconnect your GitHub account to continue installation setup', + ], + [new Error('private database details'), 500, 'Internal server error'], + ] as const)( + 'projects %s without successful installation data', + async (error, status, message) => { + mocks.list.mockRejectedValue(error) + const response = await GET(new NextRequest(`${URL}?organizationId=org-1`)) + expect(response.status).toBe(status) + expect(response.headers.get('Cache-Control')).toBe('private, no-store') + const body = await response.json() + expect(body.error).toBe(message) + expect(body).not.toHaveProperty('installations') + expect(body).not.toHaveProperty('credential') + } + ) +}) diff --git a/apps/sim/app/api/knowledge/github/installations/route.ts b/apps/sim/app/api/knowledge/github/installations/route.ts new file mode 100644 index 00000000000..1ec3700a125 --- /dev/null +++ b/apps/sim/app/api/knowledge/github/installations/route.ts @@ -0,0 +1,53 @@ +import { + connectGitHubSearchInstallationContract, + listGitHubSearchInstallationsContract, +} from '@/lib/api/contracts/knowledge/github-installations' +import { + defineInternalJsonRoute, + extendInternalErrorPolicy, + internalErrorResponse, + internalOrchestrationErrorPolicy, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { ManagedOAuthCredentialError } from '@/lib/credentials/managed-oauth' +import { + connectGitHubSearchInstallation, + listGitHubSearchInstallations, +} from '@/lib/knowledge/application/github-installations' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { GitHubInstallationError } from '@/lib/oauth/github-installation' + +const errorPolicy = extendInternalErrorPolicy(internalOrchestrationErrorPolicy, (error) => { + if (error instanceof GitHubInstallationError) + return internalErrorResponse(error.status === 403 ? 403 : 502, { error: error.message }) + if (error instanceof ManagedOAuthCredentialError) + return internalErrorResponse(error.statusCode, { + error: 'Reconnect your GitHub account to continue installation setup', + }) + return null +}) + +export const GET = defineInternalJsonRoute({ + contract: listGitHubSearchInstallationsContract, + auth: internalSessionAuth, + operation: knowledgeOperations.listGitHubInstallations, + rateLimit: internalRateLimits.user({ bucketName: 'github-search-installations' }), + errorPolicy, + mapInput: ({ query }, { request }) => ({ ...query, signal: request.signal }), + useCase: listGitHubSearchInstallations, + present: (result) => ({ success: true, ...result }), + staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, +}) + +export const POST = defineInternalJsonRoute({ + contract: connectGitHubSearchInstallationContract, + auth: internalSessionAuth, + operation: knowledgeOperations.connectGitHubInstallation, + rateLimit: internalRateLimits.user({ bucketName: 'github-search-installations' }), + errorPolicy, + mapInput: ({ body }, { request }) => ({ ...body, signal: request.signal }), + useCase: connectGitHubSearchInstallation, + present: ({ credential }) => ({ success: true, credential }), + staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, +}) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.test.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.test.tsx index 925e601b84a..f1e727121d1 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.test.tsx @@ -24,6 +24,7 @@ const mocks = vi.hoisted(() => ({ refetchCredentials: vi.fn(), oauthModal: vi.fn(), serviceAccountModal: vi.fn(), + githubInstallationModal: vi.fn(), serviceAccountTarget: null as ServiceAccountConnectTarget | null, memberAccess: true, mirroredAccess: true, @@ -173,6 +174,26 @@ vi.mock('@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-confi return null }, })) +vi.mock('@/app/workspace/[workspaceId]/search/components/github-installation-modal', () => ({ + GitHubInstallationModal: (props: { + organizationId: string + onConnected: (id: string) => void + }) => { + mocks.githubInstallationModal(props) + return ( + + ) + }, +})) vi.mock('@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields', () => ({ useConnectorConfigFields: () => ({ sourceConfig: mocks.sourceConfig, @@ -390,6 +411,36 @@ describe('Slack member setup readiness', () => { }) describe('Search methods requiring member identity', () => { + it('selects a GitHub installation for content while preserving member access', async () => { + mocks.resolveSourceConfig.mockReturnValue({ repository: 'acme/docs' }) + await render({ + initialConnectorType: 'github', + initialAccessMode: 'members', + scope: { kind: 'organization', organizationId: 'org-1' }, + }) + expect(document.body.textContent).toContain('Sync documents with') + await act(async () => combobox('Connected members').click()) + const option = Array.from(document.querySelectorAll('[role="option"]')).find( + (node) => node.textContent?.trim() === 'Connect GitHub App' + ) + if (!option) throw new Error('Missing GitHub App option') + await act(async () => option.dispatchEvent(new MouseEvent('mousedown', { bubbles: true }))) + expect(mocks.githubInstallationModal).toHaveBeenCalledWith( + expect.objectContaining({ organizationId: 'org-1' }) + ) + await act(async () => button('Use GitHub installation').click()) + expect(combobox('GitHub App: acme')).toBeDefined() + await act(async () => button('Add source').click()) + expect(mocks.create).toHaveBeenCalledWith( + expect.objectContaining({ + credentialId: 'github-app-credential', + accessMode: 'members', + sourceConfig: { repository: 'acme/docs' }, + }), + expect.any(Object) + ) + }) + it.each(['members', 'admin'] as const)( 'honors the locked %s entry point over a draft for the other access mode', async (accessMode) => { @@ -754,7 +805,7 @@ describe('Search setup options', () => { }) describe('Account connection dropdown', () => { - it('keeps GitHub browsing credentials out of the primary form while preserving optional indexing-account connection', async () => { + it('offers GitHub indexing accounts directly without requiring a browsing credential', async () => { mocks.credentials = [] await render({ initialConnectorType: 'github', @@ -763,9 +814,7 @@ describe('Account connection dropdown', () => { setupDraftKey: 'github-members', }) expect(document.body.textContent).not.toContain('Account for browsing') - expect(document.body.textContent).not.toContain('Sync documents with') - expect(document.querySelector('[role="combobox"]')).toBeNull() - await act(async () => button('More options').click()) + expect(document.body.textContent).toContain('Sync documents with') await act(async () => combobox('Connected members').click()) const option = Array.from(document.querySelectorAll('[role="option"]')).find( (node) => node.textContent?.trim() === 'Connect GitHub account' diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx index f2a6d0cd486..66a272d8228 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx @@ -53,6 +53,7 @@ import { import { MaxBadge } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/max-badge' import { useConnectorConfigFields } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields' import { useConnectorScope } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-scope' +import { GitHubInstallationModal } from '@/app/workspace/[workspaceId]/search/components/github-installation-modal' import { SettingsEmptyState, SettingsQueryErrorState, @@ -146,6 +147,7 @@ export function AddConnectorModal({ const [error, setError] = useState(null) const [showOAuthModal, setShowOAuthModal] = useState(false) const [showServiceAccountModal, setShowServiceAccountModal] = useState(false) + const [showGitHubInstallationModal, setShowGitHubInstallationModal] = useState(false) const [apiKeyValue, setApiKeyValue] = useState('') const [useApiKey, setUseApiKey] = useState(!isSearchIndex) @@ -353,6 +355,8 @@ export function AddConnectorModal({ } : null + const canSetUpGitHubInstallation = + canAdmin && isSearchIndex && selectedType === 'github' && scope.kind === 'organization' const contentCredentialField = isMembersMode && connectorConfig?.supportsSeparateContentCredential ? ( <> @@ -384,6 +388,16 @@ export function AddConnectorModal({ }, ] : []), + ...(canSetUpGitHubInstallation + ? [ + { + value: '__github_installation__', + label: 'Connect GitHub App', + icon: Plus, + onSelect: () => setShowGitHubInstallationModal(true), + }, + ] + : []), ]} isLoading={credentialsLoading} disabled={isCreating} @@ -726,7 +740,7 @@ export function AddConnectorModal({ ) : null} - {!isSearchIndex && contentCredentialField} + {(!isSearchIndex || canSetUpGitHubInstallation) && contentCredentialField} {configFieldsProps && ( {showMetadata && ( <> - {isSearchIndex && contentCredentialField} + {isSearchIndex && !canSetUpGitHubInstallation && contentCredentialField} {configFieldsProps && hasOptionalSetupFields && ( )} + {showGitHubInstallationModal && + canSetUpGitHubInstallation && + isMembersMode && + scope.kind === 'organization' && ( + setShowGitHubInstallationModal(false)} + onConnected={(credentialId) => { + setContentCredentialId(credentialId) + setShowGitHubInstallationModal(false) + }} + /> + )} {showOAuthModal && connectorConfig && connectorConfig.auth.mode === 'oauth' && diff --git a/apps/sim/app/workspace/[workspaceId]/search/components/github-installation-modal.test.tsx b/apps/sim/app/workspace/[workspaceId]/search/components/github-installation-modal.test.tsx new file mode 100644 index 00000000000..af00ad3f4ec --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/search/components/github-installation-modal.test.tsx @@ -0,0 +1,199 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { ListGitHubSearchInstallationsResponse } from '@/lib/api/contracts/knowledge/github-installations' + +const mocks = vi.hoisted(() => ({ + data: undefined as ListGitHubSearchInstallationsResponse | undefined, + error: null as Error | null, + fetching: false, + pending: false, + refetch: vi.fn(), + connectInstallation: vi.fn(), + ensureAccounts: vi.fn(), + connectAccount: vi.fn(), + onConnected: vi.fn(), +})) + +vi.mock('@/hooks/queries/github-search-installations', () => ({ + useGitHubSearchInstallations: () => ({ + data: mocks.data, + error: mocks.error, + isSuccess: Boolean(mocks.data) && !mocks.error, + isError: Boolean(mocks.error), + isFetching: mocks.fetching, + refetch: mocks.refetch, + }), + useConnectGitHubSearchInstallation: () => ({ + mutate: mocks.connectInstallation, + isPending: mocks.pending, + error: null, + }), +})) +vi.mock('@/hooks/queries/organization-accounts', () => ({ + useEnsureOrganizationAccounts: () => ({ + mutate: mocks.ensureAccounts, + isPending: false, + error: null, + }), + useConnectOrganizationAccount: () => ({ + mutate: mocks.connectAccount, + isPending: false, + error: null, + }), +})) + +import { GitHubInstallationModal } from '@/app/workspace/[workspaceId]/search/components/github-installation-modal' + +let root: Root +let container: HTMLDivElement + +async function render() { + await act(async () => { + root.render( + + ) + }) +} + +function button(label: string): HTMLButtonElement { + const match = Array.from(document.querySelectorAll('button')).find( + (node) => node.textContent?.trim() === label + ) + if (!match) throw new Error(`Missing button: ${label}`) + return match +} + +beforeEach(() => { + vi.clearAllMocks() + mocks.data = { + success: true, + available: true, + installUrl: 'https://github.com/apps/sim-search/installations/new', + needsUserConnection: false, + installations: [ + { + installationId: '123', + accountId: '456', + accountLogin: 'acme', + accountType: 'Organization', + }, + ], + } + mocks.error = null + mocks.fetching = false + mocks.pending = false + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + vi.stubGlobal( + 'ResizeObserver', + class { + observe() {} + unobserve() {} + disconnect() {} + } + ) + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) +}) + +afterEach(async () => { + await act(async () => root.unmount()) + container.remove() + vi.restoreAllMocks() +}) + +describe('GitHub installation setup', () => { + it('uses only a server-verified installation and returns its credential', async () => { + await render() + const installLink = document.querySelector('a[href*="installations/new"]') + expect(installLink?.href).toBe('https://github.com/apps/sim-search/installations/new') + expect(installLink?.target).toBe('_blank') + expect(installLink?.rel).toContain('noopener') + await act(async () => button('Use installation').click()) + expect(mocks.connectInstallation).toHaveBeenCalledWith( + { organizationId: 'org-1', installationId: '123' }, + expect.any(Object) + ) + const callbacks = mocks.connectInstallation.mock.calls[0][1] + callbacks.onSuccess({ success: true, credential: { id: 'credential-1', displayName: 'acme' } }) + expect(mocks.onConnected).toHaveBeenCalledWith('credential-1') + }) + + it('requires a connected personal account before selecting an installation', async () => { + mocks.data!.needsUserConnection = true + await render() + expect(button('Use installation').disabled).toBe(true) + expect(button('Connect your GitHub account')).toBeTruthy() + expect(document.querySelector('a[href*="installations/new"]')).toBeNull() + expect(document.querySelector('[role="combobox"]')).toBeNull() + }) + + it('opens the existing managed-account connection after preserving provider setup', async () => { + mocks.data!.needsUserConnection = true + const tab = { opener: {}, closed: false, location: { href: '' }, close: vi.fn() } + vi.spyOn(window, 'open').mockReturnValue(tab as unknown as Window) + await render() + await act(async () => button('Connect your GitHub account').click()) + expect(tab.opener).toBeNull() + expect(mocks.ensureAccounts).toHaveBeenCalledWith( + { + organizationId: 'org-1', + option: { provider: 'github-repositories', label: 'GitHub', required: false }, + }, + expect.any(Object) + ) + const setup = mocks.ensureAccounts.mock.calls[0][1] + await act(async () => + setup.onSuccess({ + credentialGroup: { + options: [ + { id: 'other-option', provider: 'confluence', status: 'active' }, + { id: 'github-option', provider: 'github-repositories', status: 'active' }, + ], + }, + }) + ) + expect(mocks.connectAccount).toHaveBeenCalledWith( + { organizationId: 'org-1', optionId: 'github-option' }, + expect.any(Object) + ) + await act(async () => + mocks.connectAccount.mock.calls[0][1].onSuccess({ + invitationLink: 'https://sim.ai/credential-groups/invite/test', + }) + ) + expect(tab.location.href).toBe('https://sim.ai/credential-groups/invite/test') + expect(document.body.textContent).toContain('Finish connecting your account in the other tab') + }) + + it('does not create an enrollment when the browser blocks the account tab', async () => { + mocks.data!.needsUserConnection = true + vi.spyOn(window, 'open').mockReturnValue(null) + await render() + await act(async () => button('Connect your GitHub account').click()) + expect(mocks.ensureAccounts).not.toHaveBeenCalled() + expect(document.body.textContent).toContain('Allow pop-ups') + }) + + it.each(['unavailable', 'error', 'refreshing', 'empty'] as const)( + 'refuses installation changes while %s', + async (state) => { + if (state === 'unavailable') mocks.data!.available = false + if (state === 'error') mocks.error = new Error('Installation lookup failed') + if (state === 'refreshing') mocks.fetching = true + if (state === 'empty') mocks.data!.installations = [] + await render() + expect(button('Use installation').disabled).toBe(true) + await act(async () => button('Use installation').click()) + expect(mocks.connectInstallation).not.toHaveBeenCalled() + } + ) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/search/components/github-installation-modal.tsx b/apps/sim/app/workspace/[workspaceId]/search/components/github-installation-modal.tsx new file mode 100644 index 00000000000..b17d8e72c45 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/search/components/github-installation-modal.tsx @@ -0,0 +1,220 @@ +'use client' + +import { useState } from 'react' +import { + Chip, + ChipCombobox, + ChipLink, + ChipModal, + ChipModalBody, + ChipModalError, + ChipModalField, + ChipModalFooter, + ChipModalHeader, +} from '@sim/emcn' +import { + SettingsEmptyState, + SettingsQueryErrorState, +} from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' +import { + useConnectGitHubSearchInstallation, + useGitHubSearchInstallations, +} from '@/hooks/queries/github-search-installations' +import { + useConnectOrganizationAccount, + useEnsureOrganizationAccounts, +} from '@/hooks/queries/organization-accounts' + +interface GitHubInstallationModalProps { + organizationId: string + onClose: () => void + onConnected: (credentialId: string) => void +} + +/** Installs a content account while preserving each reader's own GitHub authorization. */ +export function GitHubInstallationModal({ + organizationId, + onClose, + onConnected, +}: GitHubInstallationModalProps) { + const installations = useGitHubSearchInstallations(organizationId) + const connectInstallation = useConnectGitHubSearchInstallation() + const ensureAccounts = useEnsureOrganizationAccounts() + const connectAccount = useConnectOrganizationAccount() + const [installationId, setInstallationId] = useState(null) + const [waitingForAccount, setWaitingForAccount] = useState(false) + const [connectionError, setConnectionError] = useState(null) + const data = installations.isSuccess ? installations.data : undefined + const choices = data?.installations ?? [] + const selected = + choices.find((item) => item.installationId === installationId) ?? + (choices.length === 1 ? choices[0] : undefined) + const pending = + connectInstallation.isPending || ensureAccounts.isPending || connectAccount.isPending + const canConnect = + data?.available === true && + !data.needsUserConnection && + selected !== undefined && + !installations.isFetching && + !pending + + const connectGitHubAccount = () => { + const tab = window.open('about:blank', '_blank') + if (!tab) { + setConnectionError('Allow pop-ups for this site to connect your GitHub account.') + return + } + tab.opener = null + setConnectionError(null) + ensureAccounts.mutate( + { + organizationId, + option: { provider: 'github-repositories', label: 'GitHub', required: false }, + }, + { + onSuccess: ({ credentialGroup }) => { + const option = credentialGroup.options.find( + (item) => item.provider === 'github-repositories' && item.status === 'active' + ) + if (!option) { + tab.close() + setConnectionError('GitHub account setup is unavailable. Refresh and try again.') + return + } + if (tab.closed) return + connectAccount.mutate( + { organizationId, optionId: option.id }, + { + onSuccess: ({ invitationLink }) => { + if (tab.closed) return + tab.location.href = invitationLink + setWaitingForAccount(true) + }, + onError: () => tab.close(), + } + ) + }, + onError: () => tab.close(), + } + ) + } + + return ( + { + if (!open && !pending) onClose() + }} + > + Connect GitHub App + + + {installations.isError ? ( + void installations.refetch()} + variant='inline' + /> + ) : !data ? ( + Loading GitHub setup… + ) : !data.available ? ( + + GitHub App indexing is unavailable in this deployment. + + ) : data.needsUserConnection ? ( +
+

+ {waitingForAccount + ? 'Finish connecting your account in the other tab, then refresh.' + : 'Connect your GitHub account to verify the installations you can manage.'} +

+
+ + {waitingForAccount ? 'Open account connection' : 'Connect your GitHub account'} + + void installations.refetch()} + > + Refresh + +
+
+ ) : ( +
+

+ Install the app on your account or organization and choose its repositories, then + refresh. +

+
+ {data.installUrl && ( + + Install GitHub App + + )} + void installations.refetch()} + > + Refresh + +
+
+ )} + + {data?.available && !data.needsUserConnection && ( + + {choices.length > 0 ? ( + ({ + value: item.installationId, + label: item.accountLogin, + }))} + value={selected?.installationId} + onChange={setInstallationId} + placeholder='Select an installation' + disabled={pending || installations.isFetching} + /> + ) : ( + + No eligible installations found. + + )} + + )} + + {connectionError ?? + ensureAccounts.error?.message ?? + connectAccount.error?.message ?? + connectInstallation.error?.message} + + + { + if (!canConnect || !selected) return + connectInstallation.mutate( + { organizationId, installationId: selected.installationId }, + { onSuccess: ({ credential }) => onConnected(credential.id) } + ) + }, + }} + /> + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/search/components/search-source-setup.test.tsx b/apps/sim/app/workspace/[workspaceId]/search/components/search-source-setup.test.tsx index 8dc2b768cc2..838cb7aec2f 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/components/search-source-setup.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/components/search-source-setup.test.tsx @@ -430,7 +430,7 @@ describe('organization setup entry points', () => { expect(mocks.replace).not.toHaveBeenCalled() expect(document.querySelector('button[aria-label="Choose another source"]')).toBeNull() expect(document.body.textContent).not.toContain('Sync using') - expect(document.body.textContent).not.toContain('Sync documents with') + expect(document.body.textContent).toContain('Sync documents with') expect(button('Add source')).toBeEnabled() await click(button('Add source')) expect(mocks.create).toHaveBeenCalledWith( diff --git a/apps/sim/connectors/github/github.test.ts b/apps/sim/connectors/github/github.test.ts index f51615a78ee..53eaa6594d9 100644 --- a/apps/sim/connectors/github/github.test.ts +++ b/apps/sim/connectors/github/github.test.ts @@ -49,6 +49,40 @@ describe('githubConnector member listing', () => { expect(hydrated?.contentHash).toBe(listing.documents[0]?.contentHash) }) + it.each(['master', 'develop'])( + 'uses the actual %s default for an installation content pass with a blank branch', + async (defaultBranch) => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(JSON.stringify({ default_branch: defaultBranch }))) + .mockResolvedValueOnce(treeResponse([treeFile('readme.md', 'sha')])) + .mockResolvedValueOnce(new Response('text')) + vi.stubGlobal('fetch', fetchMock) + const context = {} + const config = { repository: 'owner/repo', githubRepositoryId: '101', branch: ' ' } + const listing = await githubConnector.listDocuments( + 'installation-token', + config, + undefined, + context + ) + const hydrated = await githubConnector.getDocument( + 'installation-token', + config, + 'readme.md', + context + ) + expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([ + 'https://api.github.com/repos/owner/repo', + `https://api.github.com/repos/owner/repo/git/trees/${defaultBranch}?recursive=1`, + 'https://api.github.com/repos/owner/repo/git/blobs/sha', + ]) + expect(listing.documents[0]?.metadata?.branch).toBe(defaultBranch) + expect(hydrated?.metadata?.branch).toBe(defaultBranch) + expect(hydrated?.contentHash).toBe(listing.documents[0]?.contentHash) + } + ) + it('preserves the default main branch for existing general KB sources', async () => { const fetchMock = vi.fn().mockResolvedValue(treeResponse([])) vi.stubGlobal('fetch', fetchMock) @@ -72,6 +106,20 @@ describe('githubConnector member listing', () => { ) }) + it('uses an explicitly configured installation branch without a repository metadata lookup', async () => { + const fetchMock = vi.fn().mockResolvedValue(treeResponse([])) + vi.stubGlobal('fetch', fetchMock) + await githubConnector.listDocuments( + 'installation-token', + { repository: 'owner/repo', githubRepositoryId: '101', branch: 'release/docs' }, + undefined, + {} + ) + expect(fetchMock.mock.calls[0]?.[0]).toBe( + 'https://api.github.com/repos/owner/repo/git/trees/release%2Fdocs?recursive=1' + ) + }) + it('validates a member source against its actual default branch', async () => { const fetchMock = vi .fn() diff --git a/apps/sim/connectors/github/github.ts b/apps/sim/connectors/github/github.ts index 374b88826e8..7a2c71d0218 100644 --- a/apps/sim/connectors/github/github.ts +++ b/apps/sim/connectors/github/github.ts @@ -4,6 +4,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { z } from 'zod' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { type RetryOptions, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { parseGitHubRepository } from '@/lib/oauth/github-repository' import { githubConnectorMeta } from '@/connectors/github/meta' import { fetchGitHubWithRetry as fetchWithRetry } from '@/connectors/github/request' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' @@ -54,28 +55,6 @@ function isBinaryBuffer(buf: Buffer): boolean { return false } -/** - * Parses the repository string into owner and repo. - */ -function parseRepo(repository: string): { owner: string; repo: string } { - const cleaned = repository - .trim() - .replace(/^https?:\/\/github\.com\//i, '') - .replace(/\/$/, '') - .replace(/\.git$/, '') - const parts = cleaned.split('/') - if ( - parts.length !== 2 || - !/^[a-z\d](?:[a-z\d-]*[a-z\d])?$/i.test(parts[0] ?? '') || - !/^[a-z\d_.-]+$/i.test(parts[1] ?? '') || - parts[1] === '.' || - parts[1] === '..' - ) { - throw new Error(`Invalid repository format: "${repository}". Use "owner/repo".`) - } - return { owner: parts[0], repo: parts[1] } -} - /** * File extension filter set from user config. Returns null if no filter (accept all). */ @@ -180,7 +159,7 @@ async function repositoryRequestError( return new GitHubApiError(message, response.status) } -/** Member sources follow the repository default; existing workspace sources retain main. */ +/** Search sources follow the repository default; existing workspace sources retain main. */ async function resolveBranch( accessToken: string, owner: string, @@ -191,7 +170,8 @@ async function resolveBranch( ): Promise { const configuredBranch = typeof sourceConfig.branch === 'string' ? sourceConfig.branch.trim() : '' if (configuredBranch) return configuredBranch - if (!isPerMemberListing(syncContext)) return 'main' + const isInstallationSource = typeof sourceConfig.githubRepositoryId === 'string' + if (!isPerMemberListing(syncContext) && !isInstallationSource) return 'main' if (typeof syncContext?.githubBranch === 'string') return syncContext.githubBranch const response = await fetchWithRetry( @@ -405,7 +385,7 @@ export const githubConnector: ConnectorConfig = { cursor?: string, syncContext?: Record ): Promise => { - const { owner, repo } = parseRepo(sourceConfig.repository as string) + const { owner, repo } = parseGitHubRepository(sourceConfig.repository as string) const position = readCursor(cursor, syncContext) const branch = position?.branch ?? (await resolveBranch(accessToken, owner, repo, sourceConfig, syncContext)) @@ -510,7 +490,7 @@ export const githubConnector: ConnectorConfig = { externalId: string, syncContext?: Record ): Promise => { - const { owner, repo } = parseRepo(sourceConfig.repository as string) + const { owner, repo } = parseGitHubRepository(sourceConfig.repository as string) const path = externalId try { @@ -578,7 +558,7 @@ export const githubConnector: ConnectorConfig = { let owner: string let repo: string try { - const parsed = parseRepo(repository) + const parsed = parseGitHubRepository(repository) owner = parsed.owner repo = parsed.repo } catch (error) { diff --git a/apps/sim/ee/credential-groups/components/organization-account-providers.test.tsx b/apps/sim/ee/credential-groups/components/organization-account-providers.test.tsx index e3961080663..5bb56a7e043 100644 --- a/apps/sim/ee/credential-groups/components/organization-account-providers.test.tsx +++ b/apps/sim/ee/credential-groups/components/organization-account-providers.test.tsx @@ -14,15 +14,19 @@ const mocks = vi.hoisted(() => ({ addAsync: vi.fn(), configure: vi.fn(), setup: vi.fn(), + accounts: vi.fn(), update: vi.fn(), remove: vi.fn(), reset: vi.fn(), slack: vi.fn<(props: unknown) => null>(() => null), addError: null as Error | null, + updatePending: false, })) vi.mock('@/hooks/queries/organization-accounts', () => ({ + useOrganizationAccounts: mocks.accounts, + useEnsureOrganizationAccounts: () => ({ isPending: false, error: null }), useUpdateOrganizationAccounts: () => ({ - isPending: false, + isPending: mocks.updatePending, mutate: mocks.update, reset: mocks.reset, }), @@ -44,8 +48,15 @@ vi.mock('@/hooks/queries/organization-accounts', () => ({ vi.mock('@/ee/credential-groups/components/slack-managed-users-modal', () => ({ SlackManagedUsersModal: mocks.slack, })) +vi.mock('@/ee/credential-groups/components/organization-account-people', () => ({ + OrganizationAccountPeople: () => null, +})) +vi.mock('@/ee/credential-groups/components/organization-account-workspace-access', () => ({ + OrganizationAccountWorkspaceAccess: () => null, +})) import { OrganizationAccountProviders } from '@/ee/credential-groups/components/organization-account-providers' +import { OrganizationConnectedAccounts } from '@/ee/credential-groups/components/organization-connected-accounts' const group: NonNullable = { id: 'group-1', @@ -83,6 +94,13 @@ const gmail: NonNullable['optio status: 'active', configurationStatus: 'ready', } +const github: NonNullable['options'][number] = { + ...gmail, + id: 'github-option', + provider: 'github-repositories', + label: 'Engineering GitHub', + required: true, +} describe('organization provider configuration UI', () => { let root: Root @@ -99,6 +117,7 @@ describe('organization provider configuration UI', () => { mocks.add.mockImplementation((_input, { onSuccess }) => onSuccess()) mocks.update.mockImplementation((_input, { onSuccess }) => onSuccess()) mocks.addError = null + mocks.updatePending = false container = document.createElement('div') document.body.appendChild(container) root = createRoot(container) @@ -205,6 +224,98 @@ describe('organization provider configuration UI', () => { expect(container.textContent).not.toContain('Indexing') }) + it('updates current provider configurations while preserving their IDs and saved settings', async () => { + const slack = { + ...gmail, + id: 'slack-option', + provider: 'slack' as const, + label: 'Company Slack', + slackBotCredentialId: 'bot-1', + requiredScopes: ['search:read'], + } + await render([], [github, gmail, slack]) + await clickButton('Update configurations') + expect(mocks.update).toHaveBeenCalledExactlyOnceWith( + { + organizationId: 'org-1', + groupId: 'group-1', + update: { + options: [ + { + id: github.id, + provider: github.provider, + label: github.label, + required: github.required, + }, + { + id: gmail.id, + provider: gmail.provider, + label: gmail.label, + required: gmail.required, + }, + { + id: slack.id, + provider: slack.provider, + label: slack.label, + required: slack.required, + slackBotCredentialId: slack.slackBotCredentialId, + requiredScopes: slack.requiredScopes, + }, + ], + }, + }, + expect.any(Object) + ) + expect(mocks.add).not.toHaveBeenCalled() + expect(mocks.remove).not.toHaveBeenCalled() + expect(toast.success).toHaveBeenCalledWith('Provider configurations updated') + }) + + it('disables configuration updates while a provider mutation is pending', async () => { + mocks.updatePending = true + await render([], [github]) + const button = Array.from(document.querySelectorAll('button')).find( + (node) => node.textContent === 'Update configurations' + ) + expect(button?.disabled).toBe(true) + await act(async () => button?.click()) + expect(mocks.update).not.toHaveBeenCalled() + }) + + it('surfaces a configuration update failure without removing the provider', async () => { + mocks.update.mockImplementation((_input, { onError }) => + onError(new Error('GitHub App configuration is unavailable')) + ) + await render([], [github]) + await clickButton('Update configurations') + expect(toast.error).toHaveBeenCalledWith('GitHub App configuration is unavailable') + expect(toast.success).not.toHaveBeenCalled() + expect(container.textContent).toContain('GitHub') + expect(mocks.remove).not.toHaveBeenCalled() + }) + + it.each([false, true])( + 'shows configuration updates only to administrators: %s', + async (canManage) => { + mocks.accounts.mockReturnValue({ + data: { + canManage, + credentialGroup: { ...group, options: [github] }, + availableProviders: ['github-repositories'], + }, + }) + await act(async () => + root.render( + + + + ) + ) + expect(container.textContent?.includes('Update configurations')).toBe(canManage) + expect(mocks.update).not.toHaveBeenCalled() + } + ) + it('opens Slack app configuration directly with the existing scopes', async () => { await render( [], diff --git a/apps/sim/ee/credential-groups/components/organization-account-providers.tsx b/apps/sim/ee/credential-groups/components/organization-account-providers.tsx index f5cce773c30..fb6cf96561a 100644 --- a/apps/sim/ee/credential-groups/components/organization-account-providers.tsx +++ b/apps/sim/ee/credential-groups/components/organization-account-providers.tsx @@ -64,10 +64,25 @@ export function OrganizationAccountProviders({ (option) => { const common = { id: option.id, label: option.label, required: option.required } return option.provider === 'slack' - ? { ...common, provider: 'slack', requiredScopes: option.requiredScopes } + ? { + ...common, + provider: 'slack', + slackBotCredentialId: option.slackBotCredentialId, + requiredScopes: option.requiredScopes, + } : { ...common, provider: option.provider } } ) + const updateConfigurations = () => { + if (pending) return + update.mutate( + { organizationId, groupId: group.id, update: { options } }, + { + onSuccess: () => toast.success('Provider configurations updated'), + onError: (error) => toast.error(error.message), + } + ) + } const addProvider = (choice: OrganizationAccountProviderChoice) => { if (choice.kind === 'mcp') { if (choice.connectorId === 'databricks') { @@ -160,20 +175,33 @@ export function OrganizationAccountProviders({ } - disabled={pending} - onClick={() => { - update.reset() - addMcp.reset() - removeMcp.reset() - setCatalogOpen(true) - }} - > - Add provider - +
+ {options.length > 0 && ( + + Update configurations + + )} + } + disabled={pending} + onClick={() => { + update.reset() + addMcp.reset() + removeMcp.reset() + setCatalogOpen(true) + }} + > + Add provider + +
} > + {options.length > 0 && ( +

+ Apply the current app configuration. Accounts whose app configuration changed will need + to reconnect. +

+ )}
{rows.map(({ id, name, icon: Icon, configure, choice }) => ( ({ requestJson: vi.fn() })) +vi.mock('@/lib/api/client/request', () => ({ requestJson: mocks.requestJson })) + +import { + githubSearchInstallationKeys, + useConnectGitHubSearchInstallation, + useGitHubSearchInstallations, +} from '@/hooks/queries/github-search-installations' +import { oauthCredentialKeys } from '@/hooks/queries/oauth/oauth-credentials' + +let root: Root +let queryClient: QueryClient + +beforeEach(() => { + vi.clearAllMocks() + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + root = createRoot(document.createElement('div')) + queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, + }) +}) + +afterEach(async () => { + await act(async () => root.unmount()) + queryClient.clear() +}) + +describe('GitHub installation queries', () => { + it('forwards cancellation and keeps installations scoped to the organization', async () => { + mocks.requestJson.mockResolvedValue({ + success: true, + available: false, + installUrl: null, + needsUserConnection: false, + installations: [], + }) + function Probe() { + useGitHubSearchInstallations('org-1') + return null + } + await act(async () => + root.render( + + + + ) + ) + expect(mocks.requestJson).toHaveBeenCalledWith(listGitHubSearchInstallationsContract, { + query: { organizationId: 'org-1' }, + signal: expect.any(AbortSignal), + }) + expect(queryClient.getQueryData(githubSearchInstallationKeys.list('org-1'))).toBeDefined() + expect(queryClient.getQueryData(githubSearchInstallationKeys.list('org-2'))).toBeUndefined() + }) + + it('does not list installations without an organization', async () => { + function Probe() { + useGitHubSearchInstallations() + return null + } + await act(async () => + root.render( + + + + ) + ) + expect(mocks.requestJson).not.toHaveBeenCalled() + }) + + it('refreshes the selected organization credential picker after connecting', async () => { + const ownKey = oauthCredentialKeys.list('github-repositories', '', '', 'org-1') + const otherKey = oauthCredentialKeys.list('github-repositories', '', '', 'org-2') + const installationsKey = githubSearchInstallationKeys.list('org-1') + for (const key of [ownKey, otherKey, installationsKey]) queryClient.setQueryData(key, []) + mocks.requestJson.mockResolvedValue({ + success: true, + credential: { id: 'cred-1', displayName: 'acme' }, + }) + let mutation: ReturnType | undefined + function Probe() { + mutation = useConnectGitHubSearchInstallation() + return null + } + await act(async () => + root.render( + + + + ) + ) + await act(async () => { + await mutation!.mutateAsync({ organizationId: 'org-1', installationId: '123' }) + }) + expect(mocks.requestJson).toHaveBeenCalledWith(connectGitHubSearchInstallationContract, { + body: { organizationId: 'org-1', installationId: '123' }, + }) + expect(queryClient.getQueryState(ownKey)?.isInvalidated).toBe(true) + expect(queryClient.getQueryState(installationsKey)?.isInvalidated).toBe(true) + expect(queryClient.getQueryState(otherKey)?.isInvalidated).toBe(false) + }) +}) diff --git a/apps/sim/hooks/queries/github-search-installations.ts b/apps/sim/hooks/queries/github-search-installations.ts new file mode 100644 index 00000000000..f7788e2d199 --- /dev/null +++ b/apps/sim/hooks/queries/github-search-installations.ts @@ -0,0 +1,53 @@ +'use client' + +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { requestJson } from '@/lib/api/client/request' +import { + type ConnectGitHubSearchInstallationBody, + connectGitHubSearchInstallationContract, + listGitHubSearchInstallationsContract, +} from '@/lib/api/contracts/knowledge/github-installations' +import { oauthCredentialKeys } from '@/hooks/queries/oauth/oauth-credentials' + +export const GITHUB_SEARCH_INSTALLATIONS_STALE_TIME = 30_000 + +export const githubSearchInstallationKeys = { + all: ['github-search-installations'] as const, + lists: () => [...githubSearchInstallationKeys.all, 'list'] as const, + list: (organizationId?: string) => + [...githubSearchInstallationKeys.lists(), organizationId ?? ''] as const, +} + +export function useGitHubSearchInstallations(organizationId?: string) { + return useQuery({ + queryKey: githubSearchInstallationKeys.list(organizationId), + queryFn: ({ signal }) => { + if (!organizationId) throw new Error('Organization is required') + return requestJson(listGitHubSearchInstallationsContract, { + query: { organizationId }, + signal, + }) + }, + enabled: Boolean(organizationId), + staleTime: GITHUB_SEARCH_INSTALLATIONS_STALE_TIME, + refetchOnWindowFocus: 'always', + retry: false, + }) +} + +export function useConnectGitHubSearchInstallation() { + const queryClient = useQueryClient() + return useMutation({ + mutationFn: (body: ConnectGitHubSearchInstallationBody) => + requestJson(connectGitHubSearchInstallationContract, { body }), + onSuccess: (_result, { organizationId }) => + Promise.all([ + queryClient.invalidateQueries({ + queryKey: githubSearchInstallationKeys.list(organizationId), + }), + queryClient.invalidateQueries({ + queryKey: oauthCredentialKeys.list('github-repositories', '', '', organizationId), + }), + ]), + }) +} diff --git a/apps/sim/lib/api/contracts/knowledge/github-installations.test.ts b/apps/sim/lib/api/contracts/knowledge/github-installations.test.ts new file mode 100644 index 00000000000..c9d69bf5d6e --- /dev/null +++ b/apps/sim/lib/api/contracts/knowledge/github-installations.test.ts @@ -0,0 +1,48 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + connectGitHubSearchInstallationBodySchema, + listGitHubSearchInstallationsResponseSchema, +} from '@/lib/api/contracts/knowledge/github-installations' + +describe('GitHub installation setup contracts', () => { + it('accepts installation identifiers without permitting credential or owner overrides', () => { + const body = { organizationId: 'org-1', installationId: '12345' } + expect(connectGitHubSearchInstallationBodySchema.parse(body)).toEqual(body) + expect( + connectGitHubSearchInstallationBodySchema.safeParse({ ...body, credentialId: 'foreign' }) + .success + ).toBe(false) + for (const installationId of ['', '0', '-1', '1.5', '123/456']) { + expect( + connectGitHubSearchInstallationBodySchema.safeParse({ ...body, installationId }).success + ).toBe(false) + } + }) + + it('accepts only a fixed GitHub App installation destination', () => { + const response = { + success: true, + available: true, + needsUserConnection: false, + installations: [], + } + expect( + listGitHubSearchInstallationsResponseSchema.safeParse({ + ...response, + installUrl: 'https://github.com/apps/sim-search/installations/new', + }).success + ).toBe(true) + for (const installUrl of [ + 'https://example.com/apps/sim-search/installations/new', + 'https://github.com.evil.example/apps/sim-search/installations/new', + 'https://github.com/apps/sim-search/installations/new?redirect_uri=https://example.com', + ]) { + expect( + listGitHubSearchInstallationsResponseSchema.safeParse({ ...response, installUrl }).success + ).toBe(false) + } + }) +}) diff --git a/apps/sim/lib/api/contracts/knowledge/github-installations.ts b/apps/sim/lib/api/contracts/knowledge/github-installations.ts new file mode 100644 index 00000000000..28fcbdc57ab --- /dev/null +++ b/apps/sim/lib/api/contracts/knowledge/github-installations.ts @@ -0,0 +1,79 @@ +import { z } from 'zod' +import { organizationIdSchema } from '@/lib/api/contracts/primitives' +import { defineRouteContract } from '@/lib/api/contracts/types' + +export const githubInstallationIdSchema = z + .string() + .max(32) + .regex(/^[1-9]\d*$/, 'GitHub installation ID must be a positive integer') + +export const githubSearchInstallationSchema = z.object({ + installationId: githubInstallationIdSchema, + accountId: z + .string() + .max(32) + .regex(/^[1-9]\d*$/, 'GitHub account ID must be a positive integer'), + accountLogin: z.string().min(1).max(100), + accountType: z.enum(['User', 'Organization']), +}) +export type GitHubSearchInstallation = z.output + +export const listGitHubSearchInstallationsQuerySchema = z.object({ + organizationId: organizationIdSchema, +}) +export type ListGitHubSearchInstallationsQuery = z.input< + typeof listGitHubSearchInstallationsQuerySchema +> + +export const listGitHubSearchInstallationsResponseSchema = z.object({ + success: z.literal(true), + available: z.boolean(), + installUrl: z + .string() + .max(2000) + .regex( + /^https:\/\/github\.com\/apps\/[a-z0-9-]+\/installations\/new$/, + 'GitHub installation URL must use the configured GitHub App' + ) + .nullable(), + needsUserConnection: z.boolean(), + installations: z.array(githubSearchInstallationSchema).max(1000), +}) +export type ListGitHubSearchInstallationsResponse = z.output< + typeof listGitHubSearchInstallationsResponseSchema +> + +export const listGitHubSearchInstallationsContract = defineRouteContract({ + method: 'GET', + path: '/api/knowledge/github/installations', + query: listGitHubSearchInstallationsQuerySchema, + response: { mode: 'json', schema: listGitHubSearchInstallationsResponseSchema }, +}) + +export const connectGitHubSearchInstallationBodySchema = z + .object({ + organizationId: organizationIdSchema, + installationId: githubInstallationIdSchema, + }) + .strict() +export type ConnectGitHubSearchInstallationBody = z.input< + typeof connectGitHubSearchInstallationBodySchema +> + +export const connectGitHubSearchInstallationResponseSchema = z.object({ + success: z.literal(true), + credential: z.object({ + id: z.string().min(1).max(200), + displayName: z.string().min(1).max(500), + }), +}) +export type ConnectGitHubSearchInstallationResponse = z.output< + typeof connectGitHubSearchInstallationResponseSchema +> + +export const connectGitHubSearchInstallationContract = defineRouteContract({ + method: 'POST', + path: '/api/knowledge/github/installations', + body: connectGitHubSearchInstallationBodySchema, + response: { mode: 'json', schema: connectGitHubSearchInstallationResponseSchema }, +}) diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 55b06fbac57..ae0f3ddbc71 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -492,6 +492,9 @@ export const env = createEnv({ GITHUB_CLIENT_SECRET: z.string().optional(), // GitHub OAuth client secret GITHUB_APP_CLIENT_ID: z.string().optional(), GITHUB_APP_CLIENT_SECRET: z.string().optional(), + GITHUB_APP_ID: z.string().optional(), + GITHUB_APP_PRIVATE_KEY: z.string().optional(), + GITHUB_APP_SLUG: z.string().optional(), DISABLE_GOOGLE_AUTH: z.boolean().optional(), // Disable Google OAuth login even when credentials are configured DISABLE_GITHUB_AUTH: z.boolean().optional(), // Disable GitHub OAuth login even when credentials are configured DISABLE_MICROSOFT_AUTH: z.boolean().optional(), // Disable Microsoft OAuth login even when credentials are configured diff --git a/apps/sim/lib/credentials/application/provider-catalog.ts b/apps/sim/lib/credentials/application/provider-catalog.ts index 7da948017bc..6fad8d50de4 100644 --- a/apps/sim/lib/credentials/application/provider-catalog.ts +++ b/apps/sim/lib/credentials/application/provider-catalog.ts @@ -15,6 +15,7 @@ import { allowedOrganizationIntegrationTypes, principalUserId, } from '@/lib/integrations/principal-scope.server' +import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' import { ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID, GOOGLE_SERVICE_ACCOUNT_PROVIDER_ID, @@ -115,6 +116,16 @@ function providerField( } function getServiceAccountDescriptor(providerId: string): ServiceAccountDescriptor { + if (providerId === GITHUB_INSTALLATION_PROVIDER_ID) { + return { + name: 'GitHub App installation', + description: 'Index repository content with a GitHub App installation.', + docsUrl: 'https://docs.sim.ai/search/github', + helpText: + 'Connect an installation through your organization’s Search integrations. Each person connects their own GitHub account to establish access.', + fields: [], + } + } if (providerId === GOOGLE_SERVICE_ACCOUNT_PROVIDER_ID) { return { name: 'Google service account', diff --git a/apps/sim/lib/credentials/service-account-secret.ts b/apps/sim/lib/credentials/service-account-secret.ts index 3996c82c868..4fc12acab7b 100644 --- a/apps/sim/lib/credentials/service-account-secret.ts +++ b/apps/sim/lib/credentials/service-account-secret.ts @@ -34,6 +34,7 @@ import { getTokenServiceAccountValidator, type TokenServiceAccountSecretBlob, } from '@/lib/credentials/token-service-accounts/server' +import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' import { ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID, ATLASSIAN_SERVICE_ACCOUNT_SECRET_TYPE, @@ -371,6 +372,11 @@ export async function verifyAndBuildServiceAccountSecret( providerId: string, fields: ServiceAccountSecretFields ): Promise { + if (providerId === GITHUB_INSTALLATION_PROVIDER_ID) { + throw new ServiceAccountSecretError( + 'Connect a GitHub App installation through your organization’s Search integrations' + ) + } const builder = Object.hasOwn(SERVICE_ACCOUNT_SECRET_BUILDERS, providerId) ? SERVICE_ACCOUNT_SECRET_BUILDERS[providerId] : undefined diff --git a/apps/sim/lib/integrations/credential-visibility.server.ts b/apps/sim/lib/integrations/credential-visibility.server.ts index dabef6dea0a..41d024a4445 100644 --- a/apps/sim/lib/integrations/credential-visibility.server.ts +++ b/apps/sim/lib/integrations/credential-visibility.server.ts @@ -9,6 +9,8 @@ import { getIntegrationAvailability, isOAuthServiceDeploymentAvailable, } from '@/lib/integrations/availability.server' +import { getGitHubInstallationConfiguration } from '@/lib/oauth/github-installation' +import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' import type { OAuthServiceMetadata } from '@/lib/oauth/types' import { getAllOAuthServices } from '@/lib/oauth/utils' import { getBlock } from '@/blocks/registry' @@ -115,6 +117,14 @@ export function createIntegrationCredentialVisibility({ providerId: string, owners: readonly OAuthServiceMetadata[] ): boolean => { + if (providerId === GITHUB_INSTALLATION_PROVIDER_ID) { + return ( + getGitHubInstallationConfiguration().configured && + owners.some( + (service) => isServiceAllowed(service) && visibleAvailability(service).length > 0 + ) + ) + } const gatingBlockType = getServiceAccountGatingBlockType(providerId) if (gatingBlockType) { const gatingBlock = getBlock(gatingBlockType) diff --git a/apps/sim/lib/knowledge/__integration__/github-member.integration.ts b/apps/sim/lib/knowledge/__integration__/github-member.integration.ts index f7f5f8da4cb..f75c1919953 100644 --- a/apps/sim/lib/knowledge/__integration__/github-member.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/github-member.integration.ts @@ -3,7 +3,7 @@ * connector registry, member sync, storage, chunking, and application authorization. * Provider replies and embeddings are deterministic; no live GitHub account is used. */ -import { createHash } from 'node:crypto' +import { createHash, generateKeyPairSync, verify } from 'node:crypto' import { posix } from 'node:path' import type { Principal } from '@sim/auth/principal' import { db } from '@sim/db' @@ -11,12 +11,15 @@ import { credential, credentialGroup, credentialGroupEnrollment, + credentialMember, document, embedding, knowledgeBase, knowledgeConnector, knowledgeConnectorMember, knowledgeDocumentObservation, + member, + organization, rateLimitBucket, resourcePolicy, user, @@ -40,9 +43,13 @@ vi.mock('@/lib/embeddings', async () => ({ }), })) -import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution' +import { + resolveBillingAttribution, + resolveOrganizationBillingAttribution, +} from '@/lib/billing/core/billing-attribution' import { env } from '@/lib/core/config/env' import { closeRedisConnection, getRedisClient } from '@/lib/core/config/redis' +import { encryptSecret } from '@/lib/core/security/encryption' import { resetStorageMethod } from '@/lib/core/storage' import { compileCredentialGroupWorkflowAccessPolicy } from '@/lib/credential-groups/application/workflow-access-policy' import { @@ -95,6 +102,8 @@ if (redisUrl) { /** Private repositories require the intersection of installation access and member access. */ interface RepositoryFixture { + id: number + public: boolean installed: boolean readers: Set defaultBranch: string @@ -117,7 +126,22 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { id: env.GITHUB_APP_CLIENT_ID, secret: env.GITHUB_APP_CLIENT_SECRET, redis: env.REDIS_URL, + appId: env.GITHUB_APP_ID, + privateKey: env.GITHUB_APP_PRIVATE_KEY, + slug: env.GITHUB_APP_SLUG, } + const { privateKey, publicKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }) + let organizationSource = false + let installationSuspended = false + const installation = () => ({ + id: 42, + app_id: 1, + client_id: 'github-fixture-client', + account: { id: 90, login: 'fixture', type: 'Organization' }, + repository_selection: 'selected', + permissions: { contents: 'read', metadata: 'read' }, + suspended_at: installationSuspended ? new Date().toISOString() : null, + }) let oauthStateKey: string | undefined let oauthVerification: { codeVerifier: string; redirectUri: string } | undefined const tokenFor = (userId: string) => `ghu_fixture_${userId}` @@ -137,6 +161,8 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { function repository(name: string, readers = [ids.aliceId, ids.bobId]) { const value: RepositoryFixture = { + id: 9001 + repositories.size, + public: false, installed: true, readers: new Set(readers), defaultBranch: 'trunk', @@ -194,18 +220,62 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { refresh_token_expires_in: 15897600, }) } - if (url.origin !== 'https://api.github.com' || request.method !== 'GET') + if (url.origin !== 'https://api.github.com') throw new Error(`Unexpected outbound request: ${request.method} ${url.origin}${url.pathname}`) + const bearer = request.headers.get('authorization')?.slice(7) ?? '' + if (url.pathname.startsWith('/app/installations/') || url.pathname.endsWith('/installation')) { + const [header, payload, signature] = bearer.split('.') + expect( + verify( + 'RSA-SHA256', + Buffer.from(`${header}.${payload}`), + publicKey, + Buffer.from(signature, 'base64url') + ) + ).toBe(true) + expect(JSON.parse(Buffer.from(payload, 'base64url').toString()).iss).toBe( + 'github-fixture-client' + ) + requests.push({ userId: 'app', path: url.pathname }) + if (url.pathname === '/app/installations/42/access_tokens') { + expect(request.method).toBe('POST') + const body = await request.json() + expect(body).toEqual({ + permissions: { contents: 'read', metadata: 'read' }, + repository_ids: [9001], + }) + return Response.json({ + token: 'ghs_fixture_installation', + expires_at: new Date(Date.now() + 60 * 60_000).toISOString(), + permissions: body.permissions, + repositories: [{ id: 9001 }], + }) + } + expect(request.method).toBe('GET') + if ( + url.pathname === '/repos/fixture/shared/installation' && + !repositories.get('shared')?.installed + ) + return Response.json({ message: 'Not Found' }, { status: 404 }) + expect(['/app/installations/42', '/repos/fixture/shared/installation']).toContain( + url.pathname + ) + return Response.json(installation()) + } + if (request.method !== 'GET') throw new Error(`Unexpected GitHub method: ${request.method}`) + const installationToken = bearer === 'ghs_fixture_installation' const member = enrolled.members.find((candidate) => [tokenFor(candidate.userId), `${tokenFor(candidate.userId)}_refreshed`].some( (token) => request.headers.get('authorization') === `Bearer ${token}` ) ) - if (!member) throw new Error('GitHub request did not use an enrolled member token') + if (!member && !installationToken) + throw new Error('GitHub request did not use an enrolled member or installation token') + const actingId = installationToken ? 'installation' : member!.userId expect(request.headers.get('x-github-api-version')).toBe('2022-11-28') - requests.push({ userId: member.userId, path: `${url.pathname}${url.search}` }) + requests.push({ userId: actingId, path: `${url.pathname}${url.search}` }) if (url.pathname === '/user') { - expect(member.userId).toBe(ids.aliceId) + expect(actingId).toBe(ids.aliceId) return Response.json({ id: 101, login: 'github-fixture-alice', @@ -218,24 +288,40 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { expect(url.searchParams.get('page')).toBe('1') return Response.json([ { email: 'personal@github-fixture.test', primary: true, verified: true }, - { email: `${member.userId}@fixture.test`, primary: false, verified: true }, + { email: `${actingId}@fixture.test`, primary: false, verified: true }, ]) } const match = url.pathname.match(/^\/repos\/fixture\/([^/]+)(.*)$/) if (!match) throw new Error(`Unexpected GitHub endpoint: ${url.pathname}`) const source = repositories.get(match[1]) if (!source) throw new Error('Unexpected GitHub repository') - if (source.throttledReaders.has(member.userId)) + if (source.throttledReaders.has(actingId)) return Response.json( { message: 'You have exceeded a secondary rate limit.' }, { status: 403 } ) - if (!source.installed || !source.readers.has(member.userId)) + if ( + (installationToken && !source.installed) || + (!installationToken && !source.public && (!source.installed || !source.readers.has(actingId))) + ) return Response.json( { message: 'Resource not accessible by integration' }, { status: source.deniedStatus } ) - if (!match[2]) return Response.json({ private: true, default_branch: source.defaultBranch }) + if (!match[2]) + return Response.json({ + id: source.id, + owner: { id: 90 }, + full_name: `fixture/${match[1]}`, + private: !source.public, + default_branch: source.defaultBranch, + }) + if (match[2].startsWith('/git/ref/heads/')) { + const ref = decodeURIComponent(match[2].slice('/git/ref/heads/'.length)) + return ref === source.defaultBranch + ? Response.json({ ref: `refs/heads/${ref}`, object: { type: 'commit', sha: shaFor(ref) } }) + : Response.json({ message: 'Not Found' }, { status: 404 }) + } if (match[2].startsWith('/git/trees/')) { const ref = decodeURIComponent(match[2].slice('/git/trees/'.length)) const treeSha = shaFor(JSON.stringify([[...source.files], [...source.symlinks]])) @@ -264,7 +350,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { }) } if (match[2].startsWith('/git/blobs/')) { - if (source.throttledBlobReaders.has(member.userId)) + if (source.throttledBlobReaders.has(actingId)) return Response.json( { message: 'You have exceeded a secondary rate limit.' }, { status: 403 } @@ -311,6 +397,8 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { repositories.clear() requests.length = 0 refreshedUsers.clear() + organizationSource = false + installationSuspended = false oauthStateKey = undefined oauthVerification = undefined Object.assign(env, { @@ -435,6 +523,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { ) ) await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) + await db.delete(organization).where(eq(organization.id, ids.organizationId)) await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) vi.unstubAllGlobals() } @@ -443,6 +532,9 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { Object.assign(env, { GITHUB_APP_CLIENT_ID: previousClient.id, GITHUB_APP_CLIENT_SECRET: previousClient.secret, + GITHUB_APP_ID: previousClient.appId, + GITHUB_APP_PRIVATE_KEY: previousClient.privateKey, + GITHUB_APP_SLUG: previousClient.slug, }) await db.$client.end() }) @@ -497,7 +589,9 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { const result = await searchKnowledge.execute({ principal, input: { - workspaceId: ids.workspaceId, + ...(organizationSource + ? { organizationId: ids.organizationId } + : { workspaceId: ids.workspaceId }), knowledgeBaseIds: [ids.knowledgeBaseId], query: 'Orion', searchMode: 'hybrid', @@ -528,6 +622,120 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { } } + it('indexes an organization installation once and denies live user, app, and org revocations before search or reads', async () => { + organizationSource = true + Object.assign(env, { + GITHUB_APP_ID: '1', + GITHUB_APP_PRIVATE_KEY: privateKey.export({ type: 'pkcs8', format: 'pem' }).toString(), + GITHUB_APP_SLUG: 'github-fixture', + }) + await db.insert(member).values([ + { id: generateId(), organizationId: ids.organizationId, userId: ids.aliceId, role: 'owner' }, + { id: generateId(), organizationId: ids.organizationId, userId: ids.bobId, role: 'member' }, + ]) + await db + .update(knowledgeBase) + .set({ workspaceId: null, organizationId: ids.organizationId }) + .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) + await db + .update(credentialGroup) + .set({ workspaceId: null, organizationId: ids.organizationId }) + .where(eq(credentialGroup.id, enrolled.groupId)) + await db + .update(credential) + .set({ workspaceId: null, organizationId: ids.organizationId }) + .where( + inArray( + credential.id, + enrolled.members.map((entry) => entry.credentialId) + ) + ) + await db + .update(knowledgeConnectorMember) + .set({ workspaceId: null, organizationId: ids.organizationId }) + .where(eq(knowledgeConnectorMember.connectorId, enrolled.connectorId)) + const installationCredentialId = generateId() + const { encrypted } = await encryptSecret( + JSON.stringify({ + type: 'github_app_installation', + version: 1, + appId: '1', + appClientId: 'github-fixture-client', + installationId: '42', + accountId: '90', + accountType: 'Organization', + accountLogin: 'fixture', + repositorySelection: 'selected', + }) + ) + await db.insert(credential).values({ + id: installationCredentialId, + organizationId: ids.organizationId, + type: 'service_account', + providerId: 'github-app-installation', + providerSubjectId: '42', + providerTenantId: '90', + encryptedServiceAccountKey: encrypted, + displayName: 'GitHub fixture installation', + createdBy: ids.aliceId, + }) + await db.insert(credentialMember).values({ + id: generateId(), + credentialId: installationCredentialId, + userId: ids.aliceId, + role: 'admin', + status: 'active', + }) + await db + .update(knowledgeConnector) + .set({ + credentialId: installationCredentialId, + sourceConfig: { repository: 'fixture/shared', githubRepositoryId: '9001', maxFiles: 0 }, + }) + .where(eq(knowledgeConnector.id, enrolled.connectorId)) + billing = await resolveOrganizationBillingAttribution({ + actorUserId: ids.aliceId, + organizationId: ids.organizationId, + }) + const result = await sync() + expect(result.error).toBeUndefined() + expect(result.docsHydratedOnce).toBe(1) + const [indexed] = await rows() + expect(indexed).toBeDefined() + expect( + requests.filter((entry) => entry.path.includes('/git/blobs/')).map((entry) => entry.userId) + ).toEqual(['installation']) + expect(await search(actor(ids.aliceId))).toEqual([indexed.id]) + expect(await search(actor(ids.bobId))).toEqual([indexed.id]) + await assertAccess(actor(ids.bobId), indexed, true) + const source = repositories.get('shared')! + source.readers.delete(ids.bobId) + expect(await search(actor(ids.bobId))).toEqual([]) + await assertAccess(actor(ids.bobId), indexed, false) + expect(await search(actor(ids.aliceId))).toEqual([indexed.id]) + expect( + await db + .select() + .from(knowledgeDocumentObservation) + .where(eq(knowledgeDocumentObservation.documentId, indexed.id)) + ).toHaveLength(2) + source.readers.add(ids.bobId) + source.public = true + source.installed = false + expect(await search(actor(ids.aliceId))).toEqual([]) + await assertAccess(actor(ids.aliceId), indexed, false) + source.installed = true + installationSuspended = true + expect(await search(actor(ids.aliceId))).toEqual([]) + installationSuspended = false + expect(await search(actor(ids.aliceId))).toEqual([indexed.id]) + await db + .delete(member) + .where(and(eq(member.organizationId, ids.organizationId), eq(member.userId, ids.bobId))) + await expect(search(actor(ids.bobId))).rejects.toThrow() + await assertAccess(actor(ids.bobId), indexed, false) + }) + it.runIf(Boolean(redisUrl))( 'completes a PKCE OAuth attempt through Redis and persists a searchable scopeless credential', async () => { diff --git a/apps/sim/lib/knowledge/access/github-installation.test.ts b/apps/sim/lib/knowledge/access/github-installation.test.ts new file mode 100644 index 00000000000..22d5eb560ff --- /dev/null +++ b/apps/sim/lib/knowledge/access/github-installation.test.ts @@ -0,0 +1,247 @@ +/** @vitest-environment node */ +import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + GITHUB_READ_CONCURRENCY, + GITHUB_READ_RESPONSE_MAX_BYTES, + GITHUB_READ_SOURCE_LIMIT, + resolveGitHubInstallationReadGrants, +} from '@/lib/knowledge/access/github-installation' + +const mocks = vi.hoisted(() => ({ + token: vi.fn(), + installation: vi.fn(), + repositoryInstallation: vi.fn(), + decrypt: vi.fn(), + fetch: vi.fn(), +})) +vi.mock('@/lib/credentials/managed-oauth', () => ({ resolveManagedOAuthToken: mocks.token })) +vi.mock('@/lib/core/security/encryption', () => ({ decryptSecret: mocks.decrypt })) +vi.mock('@/lib/oauth/github-installation', () => ({ + parseGitHubInstallationBinding: (value: unknown) => value, + assertGitHubInstallationActive: mocks.installation, + assertGitHubInstallationRepositoryActive: mocks.repositoryInstallation, +})) + +const input = { + scope: { kind: 'organization' as const, organizationId: 'org-1' }, + readers: [{ credentialId: 'alice-credential', subjectToken: 's:github-repositories:-:alice' }], + knowledgeBaseIds: ['index-1'], +} +const source = { + connectorId: 'source-1', + contentCredentialId: 'installation-credential', + memberCredentialId: 'alice-credential', + subjectToken: 's:github-repositories:-:alice', + repository: 'company/private', + repositoryId: '123', + branch: null as string | null, +} +const binding = { installationId: '42', accountId: '90' } +const contentCredential = { + id: 'installation-credential', + key: 'encrypted-installation', + installationId: '42', + accountId: '90', +} +const grant = { + connectorId: source.connectorId, + contentCredentialId: source.contentCredentialId, + readerCredentialId: source.memberCredentialId, + readerSubjectToken: source.subjectToken, + repositoryId: source.repositoryId, +} +const metadata = { id: 123, owner: { id: 90 }, default_branch: 'main' } +const reference = { ref: 'refs/heads/main', object: { type: 'commit', sha: 'a'.repeat(40) } } + +function queueSources(rows: (typeof source)[] = [source]) { + queueTableRows(schemaMock.knowledgeConnector, rows) + queueTableRows(schemaMock.credential, [contentCredential]) +} + +beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + vi.stubGlobal('fetch', mocks.fetch) + mocks.token.mockResolvedValue({ accessToken: 'ghu_alice' }) + mocks.installation.mockResolvedValue(binding) + mocks.repositoryInstallation.mockResolvedValue(undefined) + mocks.decrypt.mockResolvedValue({ decrypted: JSON.stringify(binding) }) + mocks.fetch.mockImplementation(async (url: string) => + Response.json(url.includes('/git/ref/') ? reference : metadata) + ) +}) + +describe('live GitHub installation reader access', () => { + it('requires both current installation and personal Contents access for the immutable repository', async () => { + queueSources() + await expect(resolveGitHubInstallationReadGrants(input)).resolves.toEqual([grant]) + expect(mocks.token).toHaveBeenCalledWith({ + credentialId: 'alice-credential', + organizationId: 'org-1', + expectedProviderId: 'github-repositories', + requiredScopes: [], + }) + expect(mocks.fetch.mock.calls.map(([url]) => url)).toEqual([ + 'https://api.github.com/repos/company/private', + 'https://api.github.com/repos/company/private/git/ref/heads/main', + ]) + for (const [, init] of mocks.fetch.mock.calls) + expect(init).toMatchObject({ + headers: { Authorization: 'Bearer ghu_alice' }, + cache: 'no-store', + redirect: 'error', + }) + }) + + it('does not reuse a positive check after upstream access is revoked without a sync', async () => { + queueSources() + await expect(resolveGitHubInstallationReadGrants(input)).resolves.toEqual([grant]) + queueSources() + mocks.fetch.mockResolvedValueOnce(new Response(null, { status: 404 })) + await expect(resolveGitHubInstallationReadGrants(input)).resolves.toEqual([]) + expect(mocks.installation).toHaveBeenCalledTimes(2) + expect(mocks.token).toHaveBeenCalledTimes(2) + }) + + it('denies a public repository removed from the app installation even if the user could read it', async () => { + queueSources() + mocks.repositoryInstallation.mockRejectedValue(new Error('Repository installation not found')) + mocks.fetch.mockResolvedValue(Response.json({ ...metadata, private: false })) + await expect(resolveGitHubInstallationReadGrants(input)).resolves.toEqual([]) + expect(mocks.repositoryInstallation).toHaveBeenCalledWith(binding, source.repository, { + signal: expect.any(AbortSignal), + }) + expect(mocks.fetch).not.toHaveBeenCalled() + }) + + it.each([401, 403, 404, 429, 500, 503])( + 'denies a %i response rather than trusting stored observations', + async (status) => { + queueSources() + mocks.fetch.mockResolvedValueOnce(new Response(null, { status })) + await expect(resolveGitHubInstallationReadGrants(input)).resolves.toEqual([]) + } + ) + + it('denies metadata-only access even when repository lookup succeeds', async () => { + queueSources() + mocks.fetch + .mockResolvedValueOnce(Response.json(metadata)) + .mockResolvedValueOnce(new Response(null, { status: 403 })) + await expect(resolveGitHubInstallationReadGrants(input)).resolves.toEqual([]) + }) + + it.each([ + { ...metadata, id: 456 }, + { ...metadata, owner: { id: 91 } }, + { ...metadata, default_branch: null }, + ])( + 'denies changed repository or account identity and incomplete provider data', + async (response) => { + queueSources() + mocks.fetch.mockResolvedValueOnce(Response.json(response)) + await expect(resolveGitHubInstallationReadGrants(input)).resolves.toEqual([]) + expect(mocks.fetch).toHaveBeenCalledTimes(1) + } + ) + + it('never uses the installer token or another enrolled person as a fallback', async () => { + queueSources([{ ...source, subjectToken: 's:github-repositories:-:bob' }]) + await expect(resolveGitHubInstallationReadGrants(input)).resolves.toEqual([]) + expect(mocks.token).not.toHaveBeenCalled() + expect(mocks.fetch).not.toHaveBeenCalled() + }) + + it('denies a reader who never connected without resolving an installation', async () => { + await expect(resolveGitHubInstallationReadGrants({ ...input, readers: [] })).resolves.toEqual( + [] + ) + expect(dbChainMockFns.select).not.toHaveBeenCalled() + expect(mocks.installation).not.toHaveBeenCalled() + }) + + it('denies suspended installations and mismatched stored bindings', async () => { + queueSources() + mocks.installation.mockRejectedValueOnce(new Error('suspended')) + await expect(resolveGitHubInstallationReadGrants(input)).resolves.toEqual([]) + queueSources() + mocks.decrypt.mockResolvedValueOnce({ + decrypted: JSON.stringify({ ...binding, accountId: '91' }), + }) + await expect(resolveGitHubInstallationReadGrants(input)).resolves.toEqual([]) + expect(mocks.fetch).not.toHaveBeenCalled() + }) + + it('keeps an allowed repository when a different repository is denied', async () => { + queueSources([ + source, + { ...source, connectorId: 'source-2', repository: 'company/denied', repositoryId: '456' }, + ]) + mocks.fetch.mockImplementation(async (url: string) => + url.includes('/denied') + ? new Response(null, { status: 404 }) + : Response.json(url.includes('/git/ref/') ? reference : metadata) + ) + await expect(resolveGitHubInstallationReadGrants(input)).resolves.toEqual([grant]) + expect(mocks.installation).toHaveBeenCalledTimes(1) + expect(mocks.token).toHaveBeenCalledTimes(1) + }) + + it('deduplicates identical repository checks only inside the current admission', async () => { + queueSources([source, { ...source, connectorId: 'source-2' }]) + await expect(resolveGitHubInstallationReadGrants(input)).resolves.toHaveLength(2) + expect(mocks.fetch).toHaveBeenCalledTimes(2) + }) + + it('bounds sources before any provider request', async () => { + queueSources( + Array.from({ length: GITHUB_READ_SOURCE_LIMIT + 1 }, (_, index) => ({ + ...source, + connectorId: `source-${index}`, + })) + ) + await expect(resolveGitHubInstallationReadGrants(input)).resolves.toEqual([]) + expect(dbChainMockFns.limit).toHaveBeenCalledWith(GITHUB_READ_SOURCE_LIMIT + 1) + expect(mocks.installation).not.toHaveBeenCalled() + }) + + it('bounds concurrent source checks and never buffers unbounded response bytes', async () => { + queueSources( + Array.from({ length: GITHUB_READ_CONCURRENCY + 2 }, (_, index) => ({ + ...source, + connectorId: `source-${index}`, + repository: `company/repo-${index}`, + })) + ) + let active = 0 + let peak = 0 + mocks.fetch.mockImplementation(async (url: string) => { + active += 1 + peak = Math.max(peak, active) + await Promise.resolve() + active -= 1 + return Response.json(url.includes('/git/ref/') ? reference : metadata) + }) + await expect(resolveGitHubInstallationReadGrants(input)).resolves.toHaveLength( + GITHUB_READ_CONCURRENCY + 2 + ) + expect(peak).toBeLessThanOrEqual(GITHUB_READ_CONCURRENCY) + queueSources() + mocks.fetch.mockResolvedValueOnce(new Response('x'.repeat(GITHUB_READ_RESPONSE_MAX_BYTES + 1))) + await expect(resolveGitHubInstallationReadGrants(input)).resolves.toEqual([]) + }) + + it('stops on cancellation while a credential refresh remains pending', async () => { + queueSources() + const controller = new AbortController() + mocks.token.mockImplementation(() => { + controller.abort(new Error('cancelled')) + return new Promise(() => {}) + }) + await expect( + resolveGitHubInstallationReadGrants({ ...input, signal: controller.signal }) + ).resolves.toEqual([]) + expect(mocks.fetch).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/access/github-installation.ts b/apps/sim/lib/knowledge/access/github-installation.ts new file mode 100644 index 00000000000..ecd411bc912 --- /dev/null +++ b/apps/sim/lib/knowledge/access/github-installation.ts @@ -0,0 +1,297 @@ +import { db } from '@sim/db' +import { + credential, + knowledgeBase, + knowledgeConnector, + knowledgeConnectorMember, +} from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { isPlainRecord } from '@sim/utils/object' +import { and, asc, eq, inArray, isNull, sql } from 'drizzle-orm' +import { type ResourceScope, resourceScopeFields } from '@/lib/core/resource-scope' +import { resourceScopeCondition } from '@/lib/core/resource-scope.server' +import { decryptSecret } from '@/lib/core/security/encryption' +import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import { resolveManagedOAuthToken } from '@/lib/credentials/managed-oauth' +import type { GitHubInstallationReadGrant } from '@/lib/knowledge/access/types' +import { + assertGitHubInstallationActive, + assertGitHubInstallationRepositoryActive, + parseGitHubInstallationBinding, +} from '@/lib/oauth/github-installation' +import { + GITHUB_INSTALLATION_PROVIDER_ID, + type GitHubInstallationBinding, +} from '@/lib/oauth/github-installation-types' + +const logger = createLogger('GitHubInstallationReadAccess') +export const GITHUB_READ_SOURCE_LIMIT = 100 +export const GITHUB_READ_CONCURRENCY = 4 +export const GITHUB_READ_TIMEOUT_MS = 8000 +export const GITHUB_READ_RESPONSE_MAX_BYTES = 64 * 1024 +const INSTALLATION_BINDING_MAX_BYTES = 16 * 1024 + +export interface GitHubReaderCredential { + credentialId: string + subjectToken: string +} + +interface GitHubReadSource { + connectorId: string + contentCredentialId: string | null + memberCredentialId: string + subjectToken: string + repository: string | null + repositoryId: string | null + branch: string | null +} + +/** Token refresh may outlive its caller, but a timed-out admission must stop waiting or fetching. */ +function withinAdmission(pending: Promise, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + const abort = () => reject(signal.reason) + signal.addEventListener('abort', abort, { once: true }) + pending.then(resolve, reject).finally(() => signal.removeEventListener('abort', abort)) + if (signal.aborted) abort() + }) +} + +function positiveId(value: unknown): string | null { + if (typeof value === 'number' && Number.isSafeInteger(value) && value > 0) return String(value) + return typeof value === 'string' && /^[1-9]\d{0,19}$/.test(value) ? value : null +} + +async function readGitHubJson(path: string, accessToken: string, signal: AbortSignal) { + signal.throwIfAborted() + const response = await fetch(`https://api.github.com${path}`, { + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + }, + redirect: 'error', + cache: 'no-store', + signal, + }) + if (!response.ok) { + await response.body?.cancel() + throw new Error('GitHub did not confirm current repository access') + } + return readResponseJsonWithLimit(response, { + maxBytes: GITHUB_READ_RESPONSE_MAX_BYTES, + label: 'GitHub repository authorization response', + signal, + }) +} + +/** A metadata response alone does not prove Contents permission; the Git ref endpoint does. */ +async function verifyRepository( + source: GitHubReadSource, + accountId: string, + accessToken: string, + signal: AbortSignal +): Promise { + if ( + !source.repository || + !/^[A-Za-z0-9_.-]{1,100}\/[A-Za-z0-9_.-]{1,100}$/.test(source.repository) + ) + return false + if (source.repository.split('/').some((segment) => segment === '.' || segment === '..')) + return false + if (!positiveId(source.repositoryId)) return false + const path = `/repos/${source.repository.split('/').map(encodeURIComponent).join('/')}` + const repository = await readGitHubJson(path, accessToken, signal) + if ( + !isPlainRecord(repository) || + positiveId(repository.id) !== source.repositoryId || + !isPlainRecord(repository.owner) || + positiveId(repository.owner.id) !== accountId + ) + return false + const branch = source.branch?.trim() || repository.default_branch + if (typeof branch !== 'string' || !branch || branch.length > 1024) return false + if (branch.split('/').some((segment) => !segment || segment === '.' || segment === '..')) + return false + const reference = await readGitHubJson( + `${path}/git/ref/heads/${branch.split('/').map(encodeURIComponent).join('/')}`, + accessToken, + signal + ) + return ( + isPlainRecord(reference) && + reference.ref === `refs/heads/${branch}` && + isPlainRecord(reference.object) && + reference.object.type === 'commit' && + typeof reference.object.sha === 'string' && + /^[a-f0-9]{40,64}$/.test(reference.object.sha) + ) +} + +/** + * Proves current reader access before any installation-backed indexed content is selected. + * The caller supplies credentials already bound to the verified current member. All positive + * evidence and token reuse are local to this admission; failures grant nothing for that source. + */ +export async function resolveGitHubInstallationReadGrants(input: { + scope: ResourceScope + readers: readonly GitHubReaderCredential[] + knowledgeBaseIds?: readonly string[] + signal?: AbortSignal +}): Promise { + if ( + !input.readers.length || + input.readers.length > GITHUB_READ_SOURCE_LIMIT || + (input.knowledgeBaseIds && + (input.knowledgeBaseIds.length === 0 || + input.knowledgeBaseIds.length > GITHUB_READ_SOURCE_LIMIT)) + ) + return [] + const readers = new Map(input.readers.map((reader) => [reader.credentialId, reader.subjectToken])) + const sources: GitHubReadSource[] = await db + .select({ + connectorId: knowledgeConnector.id, + contentCredentialId: knowledgeConnector.credentialId, + memberCredentialId: knowledgeConnectorMember.credentialId, + subjectToken: knowledgeConnectorMember.subjectToken, + repository: sql`left(${knowledgeConnector.sourceConfig}->>'repository', 202)`, + repositoryId: sql< + string | null + >`left(${knowledgeConnector.sourceConfig}->>'githubRepositoryId', 21)`, + branch: sql`left(${knowledgeConnector.sourceConfig}->>'branch', 1025)`, + }) + .from(knowledgeConnector) + .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) + .innerJoin( + knowledgeConnectorMember, + eq(knowledgeConnectorMember.connectorId, knowledgeConnector.id) + ) + .where( + and( + resourceScopeCondition(knowledgeBase, input.scope), + input.knowledgeBaseIds ? inArray(knowledgeBase.id, [...input.knowledgeBaseIds]) : undefined, + isNull(knowledgeBase.deletedAt), + eq(knowledgeConnector.connectorType, 'github'), + eq(knowledgeConnector.accessMode, 'members'), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt), + eq(knowledgeConnectorMember.status, 'active'), + inArray(knowledgeConnectorMember.credentialId, [...readers.keys()]), + sql`${knowledgeConnector.sourceConfig}::jsonb ? 'githubRepositoryId'` + ) + ) + .orderBy(asc(knowledgeConnector.id), asc(knowledgeConnectorMember.id)) + .limit(GITHUB_READ_SOURCE_LIMIT + 1) + if (sources.length > GITHUB_READ_SOURCE_LIMIT || !sources.length) return [] + const contentCredentialIds = [ + ...new Set( + sources.flatMap((source) => (source.contentCredentialId ? [source.contentCredentialId] : [])) + ), + ] + if (!contentCredentialIds.length) return [] + const credentials = await db + .select({ + id: credential.id, + key: credential.encryptedServiceAccountKey, + installationId: credential.providerSubjectId, + accountId: credential.providerTenantId, + }) + .from(credential) + .where( + and( + inArray(credential.id, contentCredentialIds), + resourceScopeCondition(credential, input.scope), + eq(credential.type, 'service_account'), + eq(credential.providerId, GITHUB_INSTALLATION_PROVIDER_ID), + isNull(credential.revokedAt), + sql`octet_length(${credential.encryptedServiceAccountKey}) <= ${INSTALLATION_BINDING_MAX_BYTES}` + ) + ) + .limit(GITHUB_READ_SOURCE_LIMIT) + const contentById = new Map(credentials.map((entry) => [entry.id, entry])) + const timeout = AbortSignal.timeout(GITHUB_READ_TIMEOUT_MS) + const signal = input.signal ? AbortSignal.any([input.signal, timeout]) : timeout + const installations = new Map>() + const tokens = new Map>() + const proofs = new Map>() + const grants = new Map() + for ( + let offset = 0; + offset < sources.length && !signal.aborted; + offset += GITHUB_READ_CONCURRENCY + ) { + await Promise.all( + sources.slice(offset, offset + GITHUB_READ_CONCURRENCY).map(async (source) => { + if (readers.get(source.memberCredentialId) !== source.subjectToken) return + const content = source.contentCredentialId + ? contentById.get(source.contentCredentialId) + : undefined + if (!content?.key || !source.repositoryId) return + try { + let installation = installations.get(content.id) + if (!installation) { + installation = (async () => { + const { decrypted } = await decryptSecret(content.key!) + const binding = parseGitHubInstallationBinding(JSON.parse(decrypted)) + if ( + binding.installationId !== content.installationId || + binding.accountId !== content.accountId + ) + throw new Error('GitHub installation credential identity mismatch') + await assertGitHubInstallationActive(binding, { signal }) + return binding + })() + installations.set(content.id, installation) + } + const binding = await installation + signal.throwIfAborted() + let token = tokens.get(source.memberCredentialId) + if (!token) { + token = resolveManagedOAuthToken({ + credentialId: source.memberCredentialId, + ...resourceScopeFields(input.scope), + expectedProviderId: 'github-repositories', + requiredScopes: [], + }).then(({ accessToken }) => { + if (!accessToken.startsWith('ghu_')) + throw new Error('A GitHub App user token is required') + return accessToken + }) + tokens.set(source.memberCredentialId, token) + } + const accessToken = await withinAdmission(token, signal) + signal.throwIfAborted() + const key = JSON.stringify([ + content.id, + source.memberCredentialId, + source.repositoryId, + source.repository, + source.branch, + ]) + let proof = proofs.get(key) + if (!proof) { + proof = (async () => { + if (!source.repository) return false + await assertGitHubInstallationRepositoryActive(binding, source.repository, { signal }) + return verifyRepository(source, binding.accountId, accessToken, signal) + })() + proofs.set(key, proof) + } + if (await proof) + grants.set(source.connectorId, { + connectorId: source.connectorId, + contentCredentialId: content.id, + readerCredentialId: source.memberCredentialId, + readerSubjectToken: source.subjectToken, + repositoryId: source.repositoryId, + }) + } catch { + logger.warn('GitHub did not confirm current Search access', { + connectorId: source.connectorId, + }) + } + }) + ) + } + /** Once admission expires, none of its partial proofs may authorize a later content query. */ + return signal.aborted ? [] : [...grants.values()] +} diff --git a/apps/sim/lib/knowledge/access/predicate.postgres.test.ts b/apps/sim/lib/knowledge/access/predicate.postgres.test.ts index bcefa6889b6..5137d8404ef 100644 --- a/apps/sim/lib/knowledge/access/predicate.postgres.test.ts +++ b/apps/sim/lib/knowledge/access/predicate.postgres.test.ts @@ -5,6 +5,7 @@ import { readFile } from 'node:fs/promises' import type postgres from 'postgres' import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' import { createEnterpriseSearchMigrationFixture } from '@/lib/knowledge/__integration__/migration-fixture' +import type { GitHubInstallationReadGrant } from '@/lib/knowledge/access/types' vi.unmock('drizzle-orm') vi.unmock('@sim/db/schema') @@ -46,6 +47,15 @@ describe.runIf(Boolean(databaseUrl))('knowledge ACLs in PostgreSQL', () => { ) const [{ current_schema: schemaName }] = await client`SELECT current_schema()` await client.unsafe(approvalMigration.replaceAll('"public".', `"${schemaName}".`)) + await client.unsafe(` + ALTER TABLE knowledge_connector ADD COLUMN credential_id text, + ADD COLUMN source_config json NOT NULL DEFAULT '{}', + ADD COLUMN credential_group_id text, ADD COLUMN credential_group_option_id text; + ALTER TABLE credential ADD COLUMN revoked_at timestamp, ADD COLUMN credential_group_option_id text; + ALTER TABLE credential_group ADD COLUMN options jsonb NOT NULL DEFAULT '[]'; + ALTER TABLE credential_group_enrollment ADD COLUMN user_id text; + CREATE TABLE member (id text PRIMARY KEY, organization_id text, user_id text); + `) expect(await readable(['ws'], 'before-migration')).toBe(true) await connection.unsafe("INSERT INTO document(id) VALUES ('old-writer-after-migration')") expect(await readable(['ws'], 'old-writer-after-migration')).toBe(true) @@ -64,9 +74,15 @@ describe.runIf(Boolean(databaseUrl))('knowledge ACLs in PostgreSQL', () => { ) }) - async function readable(tokens: string[], documentId: string, join = false): Promise { + async function readable( + tokens: string[], + documentId: string, + join = false, + githubInstallationGrants?: GitHubInstallationReadGrant[], + userId = 'reader' + ): Promise { const query = new PgDialect().sqlToQuery( - knowledgeAccessCondition({ kind: 'user', userId: 'reader', tokens }) + knowledgeAccessCondition({ kind: 'user', userId, tokens, githubInstallationGrants }) ) const values = query.params.map((value: unknown) => { if (typeof value === 'string' || typeof value === 'number') return value @@ -88,6 +104,105 @@ describe.runIf(Boolean(databaseUrl))('knowledge ACLs in PostgreSQL', () => { ) } + it('requires live GitHub proof and rechecks exact source, reader, credential and organization at every content query', async () => { + const token = 's:github-repositories:-:alice' + await connection.unsafe(` + INSERT INTO organization(id) VALUES ('github-org'); + INSERT INTO "user"(id,email,email_verified) VALUES ('reader','alice@example.com',true), ('bob','bob@example.com',true); + INSERT INTO member VALUES ('alice-membership','github-org','reader'), ('bob-membership','github-org','bob'); + INSERT INTO knowledge_base(id,organization_id,name,is_search_index) VALUES ('github-index','github-org','Search',true); + INSERT INTO credential_group(id,organization_id,name,status,options) + VALUES ('github-group','github-org','GitHub','active','[{"id":"github-option","status":"active"}]'); + INSERT INTO credential_group_enrollment(id,credential_group_id,email,status,user_id) + VALUES ('alice-enrollment','github-group','alice@example.com','completed','reader'); + INSERT INTO credential(id,organization_id,type,provider_id,provider_subject_id,provider_tenant_id,encrypted_service_account_key) + VALUES ('github-installation','github-org','service_account','github-app-installation','42','90','encrypted'); + INSERT INTO credential(id,organization_id,type,provider_id,provider_subject_id,authorization_app_id, + managed_oauth_status,granted_scopes,encrypted_oauth_token_set,granted_at,credential_group_enrollment_id,credential_group_option_id) + VALUES ('alice-github','github-org','managed_oauth','github-repositories','alice','github-app','active', + ARRAY[]::text[],'encrypted',now(),'alice-enrollment','github-option'); + INSERT INTO knowledge_connector(id,knowledge_base_id,connector_type,access_mode,credential_id,source_config,credential_group_id,credential_group_option_id) + VALUES ('github-source','github-index','github','members','github-installation', + '{"repository":"company/private","githubRepositoryId":"123"}','github-group','github-option'); + INSERT INTO knowledge_connector_member(id,organization_id,connector_id,subject_token,status,member_synced_through) + VALUES ('github-member','github-org','github-source','${token}','active',now()); + INSERT INTO document(id,knowledge_base_id,connector_id,acl) + VALUES ('github-document','github-index','github-source',ARRAY['${token}']); + INSERT INTO knowledge_document_observation(document_id,member_id,last_seen_at) + VALUES ('github-document','github-member',now()); + INSERT INTO embedding(id,document_id,content) VALUES ('github-chunk','github-document','private content'); + `) + const grants = [ + { + connectorId: 'github-source', + contentCredentialId: 'github-installation', + readerCredentialId: 'alice-github', + readerSubjectToken: token, + repositoryId: '123', + }, + ] + for (const join of [false, true]) { + expect(await readable([token], 'github-document', join)).toBe(false) + expect(await readable([token], 'github-document', join, grants)).toBe(true) + expect(await readable([token], 'github-document', join, grants, 'bob')).toBe(false) + expect( + await readable([token], 'github-document', join, [{ ...grants[0], repositoryId: '456' }]) + ).toBe(false) + expect( + await readable([token], 'github-document', join, [ + { ...grants[0], contentCredentialId: 'other-installation' }, + ]) + ).toBe(false) + } + await connection.unsafe( + "UPDATE credential_group_enrollment SET status='revoked' WHERE id='alice-enrollment'" + ) + expect(await readable([token], 'github-document', true, grants)).toBe(false) + await connection.unsafe( + "UPDATE credential_group_enrollment SET status='completed' WHERE id='alice-enrollment'" + ) + await connection.unsafe( + "UPDATE credential_group_enrollment SET revoked_at=now() WHERE id='alice-enrollment'" + ) + expect(await readable([token], 'github-document', true, grants)).toBe(false) + await connection.unsafe( + "UPDATE credential_group_enrollment SET revoked_at=NULL WHERE id='alice-enrollment'" + ) + await connection.unsafe( + "UPDATE credential SET provider_subject_id='bob' WHERE id='alice-github'" + ) + expect(await readable([token], 'github-document', true, grants)).toBe(false) + await connection.unsafe( + "UPDATE credential SET provider_subject_id='alice' WHERE id='alice-github'" + ) + await connection.unsafe("UPDATE credential SET revoked_at=now() WHERE id='github-installation'") + expect(await readable([token], 'github-document', true, grants)).toBe(false) + await connection.unsafe("UPDATE credential SET revoked_at=NULL WHERE id='github-installation'") + await connection.unsafe( + "UPDATE credential SET provider_id='other-provider', type='oauth' WHERE id='github-installation'" + ) + expect(await readable([token], 'github-document', true, grants)).toBe(false) + expect(await readable([token], 'github-document', true)).toBe(false) + await connection.unsafe( + "UPDATE credential SET provider_id='github-app-installation', type='service_account' WHERE id='github-installation'" + ) + await connection.unsafe( + "UPDATE knowledge_connector SET access_mode='admin' WHERE id='github-source'" + ) + expect(await readable([token], 'github-document', true, grants)).toBe(false) + await connection.unsafe( + "UPDATE knowledge_connector SET access_mode='members' WHERE id='github-source'" + ) + await connection.unsafe("DELETE FROM member WHERE id='alice-membership'") + expect(await readable([token], 'github-document', true, grants)).toBe(false) + await connection.unsafe("INSERT INTO member VALUES ('alice-membership','github-org','reader')") + await connection.unsafe( + "UPDATE knowledge_connector SET credential_id=NULL WHERE id='github-source'" + ) + expect(await readable([token], 'github-document', true, grants)).toBe(false) + expect(await readable([token], 'github-document', true)).toBe(false) + }) + it('revokes every source of one integration without changing ACLs or another organization', async () => { await connection.unsafe("INSERT INTO organization(id) VALUES ('approval-org'), ('other-org')") await connection.unsafe( diff --git a/apps/sim/lib/knowledge/access/predicate.ts b/apps/sim/lib/knowledge/access/predicate.ts index f652ade1301..1f734782142 100644 --- a/apps/sim/lib/knowledge/access/predicate.ts +++ b/apps/sim/lib/knowledge/access/predicate.ts @@ -1,13 +1,87 @@ import { + credential, + credentialGroup, + credentialGroupEnrollment, document, + knowledgeBase, knowledgeConnector, knowledgeConnectorMember, knowledgeDocumentObservation, + member, + user, } from '@sim/db/schema' import { type SQL, sql } from 'drizzle-orm' import { SOURCE_ACL_MAX_AGE_MS } from '@/lib/knowledge/access/freshness' import type { KnowledgeAccessScope, SystemAccessScope } from '@/lib/knowledge/access/types' import { searchIntegrationAccessCondition } from '@/lib/knowledge/search/integration-policy' +import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' + +/** Missing credentials or missing live evidence must never downgrade an installation source. */ +function githubInstallationAccessCondition(scope: KnowledgeAccessScope): SQL { + const grants = scope.kind === 'user' ? (scope.githubInstallationGrants ?? []) : [] + const allowed = + scope.kind !== 'user' || grants.length === 0 + ? sql`false` + : sql`EXISTS ( + SELECT 1 FROM (VALUES ${sql.join( + grants.map( + (grant) => sql`( + ${grant.connectorId}, ${grant.contentCredentialId}, ${grant.readerCredentialId}, ${grant.repositoryId}, ${grant.readerSubjectToken} + )` + ), + sql`, ` + )}) AS github_read_grant(connector_id, content_credential_id, reader_credential_id, repository_id, reader_subject_token) + JOIN ${credential} ON ${credential.id} = github_read_grant.content_credential_id + JOIN ${knowledgeBase} ON ${knowledgeBase.id} = ${knowledgeConnector.knowledgeBaseId} + WHERE github_read_grant.connector_id = ${knowledgeConnector.id} + AND github_read_grant.content_credential_id = ${knowledgeConnector.credentialId} + AND github_read_grant.repository_id = ${knowledgeConnector.sourceConfig}->>'githubRepositoryId' + AND ${knowledgeConnector.accessMode} = 'members' + AND ${knowledgeConnector.archivedAt} IS NULL AND ${knowledgeConnector.deletedAt} IS NULL + AND ${knowledgeBase.deletedAt} IS NULL + AND ${credential.type} = 'service_account' + AND ${credential.providerId} = ${GITHUB_INSTALLATION_PROVIDER_ID} + AND ${credential.revokedAt} IS NULL + AND ${credential.organizationId} IS NOT DISTINCT FROM ${knowledgeBase.organizationId} + AND ${credential.workspaceId} IS NOT DISTINCT FROM ${knowledgeBase.workspaceId} + AND (${knowledgeBase.organizationId} IS NULL OR EXISTS ( + SELECT 1 FROM ${member} WHERE ${member.organizationId} = ${knowledgeBase.organizationId} + AND ${member.userId} = ${scope.userId} + )) + AND EXISTS ( + SELECT 1 FROM ${credential} + JOIN ${credentialGroupEnrollment} ON ${credentialGroupEnrollment.id} = ${credential.credentialGroupEnrollmentId} + JOIN ${credentialGroup} ON ${credentialGroup.id} = ${credentialGroupEnrollment.credentialGroupId} + JOIN ${user} ON ${user.id} = ${scope.userId} + WHERE ${credential.id} = github_read_grant.reader_credential_id + AND ${credential.type} = 'managed_oauth' AND ${credential.providerId} = 'github-repositories' + AND ${credential.managedOauthStatus} = 'active' AND ${credential.revokedAt} IS NULL + AND ('s:github-repositories:' || COALESCE(NULLIF(${credential.providerTenantId}, ''), '-') || ':' || ${credential.providerSubjectId}) = github_read_grant.reader_subject_token + AND ${credential.organizationId} IS NOT DISTINCT FROM ${knowledgeBase.organizationId} + AND ${credential.workspaceId} IS NOT DISTINCT FROM ${knowledgeBase.workspaceId} + AND ${credentialGroup.organizationId} IS NOT DISTINCT FROM ${knowledgeBase.organizationId} + AND ${credentialGroup.workspaceId} IS NOT DISTINCT FROM ${knowledgeBase.workspaceId} + AND ${credentialGroup.status} = 'active' + AND ${credentialGroup.id} = ${knowledgeConnector.credentialGroupId} + AND ${credential.credentialGroupOptionId} = ${knowledgeConnector.credentialGroupOptionId} + AND ${credentialGroupEnrollment.status} IN ('in_progress', 'completed') + AND ${credentialGroupEnrollment.revokedAt} IS NULL + AND ${user.emailVerified} = true + AND ((${knowledgeBase.organizationId} IS NOT NULL AND ${credentialGroupEnrollment.userId} = ${scope.userId}) + OR (${knowledgeBase.workspaceId} IS NOT NULL AND ${credentialGroupEnrollment.email} = lower(btrim(${user.email})))) + AND EXISTS (SELECT 1 FROM jsonb_array_elements(${credentialGroup.options}) AS option + WHERE option->>'id' = ${credential.credentialGroupOptionId} AND option->>'status' = 'active') + ) + )` + return sql`( + ${knowledgeConnector.connectorType} IS DISTINCT FROM 'github' + OR (NOT (${knowledgeConnector.sourceConfig}::jsonb ? 'githubRepositoryId') AND NOT EXISTS ( + SELECT 1 FROM ${credential} WHERE ${credential.id} = ${knowledgeConnector.credentialId} + AND ${credential.providerId} = ${GITHUB_INSTALLATION_PROVIDER_ID} + )) + OR ${allowed} + )` +} /** * The single read-side access predicate: the document's ACL overlaps the @@ -39,6 +113,7 @@ export function knowledgeAccessCondition(scope: KnowledgeAccessScope | SystemAcc SELECT 1 FROM ${knowledgeConnector} WHERE ${knowledgeConnector.id} = ${document.connectorId} AND ${searchIntegrationAccessCondition()} + AND ${githubInstallationAccessCondition(scope)} AND ( (${knowledgeConnector.accessMode} = 'workspace' AND ${document.acl} = ARRAY['ws']::text[]) OR (${document.acl} <> ARRAY['ws']::text[] AND ( diff --git a/apps/sim/lib/knowledge/access/scope.test.ts b/apps/sim/lib/knowledge/access/scope.test.ts index c2ddde49485..440cdb3ec5d 100644 --- a/apps/sim/lib/knowledge/access/scope.test.ts +++ b/apps/sim/lib/knowledge/access/scope.test.ts @@ -6,9 +6,10 @@ import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@s import { eq, inArray } from 'drizzle-orm' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockAvailability, mockCheckWorkspaceAccess } = vi.hoisted(() => ({ +const { mockAvailability, mockCheckWorkspaceAccess, mockGitHubReadGrants } = vi.hoisted(() => ({ mockAvailability: vi.fn(async () => ({ memberScoped: true, sourceMirrored: true })), mockCheckWorkspaceAccess: vi.fn(async () => ({ hasAccess: true })), + mockGitHubReadGrants: vi.fn(async () => []), })) vi.mock('@/lib/knowledge/access/availability', () => ({ @@ -17,6 +18,9 @@ vi.mock('@/lib/knowledge/access/availability', () => ({ vi.mock('@/lib/workspaces/permissions/utils', () => ({ checkWorkspaceAccess: mockCheckWorkspaceAccess, })) +vi.mock('@/lib/knowledge/access/github-installation', () => ({ + resolveGitHubInstallationReadGrants: mockGitHubReadGrants, +})) import { createKnowledgeAccessProvider, @@ -453,6 +457,42 @@ describe('organization document ACL scope', () => { vi.clearAllMocks() resetDbChainMock() }) + it('live-checks only the current member’s GitHub credentials within the canonical selected index', async () => { + queueTableRows(schemaMock.member, [{ id: 'membership-1' }]) + queueSubjects([ + { + email: 'viewer@example.com', + credentialId: 'personal-github', + providerId: 'github-repositories', + providerSubjectId: '42', + providerTenantId: null, + }, + ]) + const scope = await resolveKnowledgeAccessScope(SESSION, { + ...organization, + knowledgeBaseIds: ['index-1'], + }) + expect(mockGitHubReadGrants).toHaveBeenCalledWith({ + scope: { kind: 'organization', organizationId: 'org-1' }, + readers: [{ credentialId: 'personal-github', subjectToken: 's:github-repositories:-:42' }], + knowledgeBaseIds: ['index-1'], + signal: undefined, + }) + expect(scope).toMatchObject({ githubInstallationGrants: [] }) + }) + it('does not live-check retained provider credentials after organization removal', async () => { + queueTableRows(schemaMock.member, []) + queueSubjects([ + { + credentialId: 'personal-github', + providerId: 'github-repositories', + providerSubjectId: '42', + providerTenantId: null, + }, + ]) + expect(await resolveKnowledgeAccessScope(SESSION, organization)).toMatchObject({ tokens: [] }) + expect(mockGitHubReadGrants).not.toHaveBeenCalled() + }) it('uses current organization membership and org baseline without any workspace membership', async () => { queueTableRows(schemaMock.member, [{ id: 'membership-1' }]) queueSubjects([ diff --git a/apps/sim/lib/knowledge/access/scope.ts b/apps/sim/lib/knowledge/access/scope.ts index 9496b1e72e2..508696b4ff7 100644 --- a/apps/sim/lib/knowledge/access/scope.ts +++ b/apps/sim/lib/knowledge/access/scope.ts @@ -23,6 +23,10 @@ import { EXTERNAL_GROUP_STALE_AFTER_MS, emailDomain, } from '@/lib/knowledge/access/external-groups' +import { + type GitHubReaderCredential, + resolveGitHubInstallationReadGrants, +} from '@/lib/knowledge/access/github-installation' import { groupToken, sortAccessTokens, @@ -33,6 +37,7 @@ import { type KnowledgeAccessProvider, type KnowledgeAccessScope, ORGANIZATION_ACCESS_TOKENS, + type UserAccessScope, WORKSPACE_ACCESS_TOKENS, type WorkspaceAccessScope, } from '@/lib/knowledge/access/types' @@ -79,7 +84,7 @@ async function loadExternalGroupTokens( ): Promise { /** * A query of its own rather than a fourth join on the credential query in - * `loadUserAccessTokens`: + * `loadUserAccess`: * that one already fans out per managed credential, and joining groups onto * it would multiply the two — every credential row repeated for every group. * Two indexed reads cost less than one cross product. @@ -120,6 +125,9 @@ export interface KnowledgeAccessScopeContext { /** Exactly one workspace or organization owner is required at resolution. */ workspaceId?: string organizationId?: string + /** Canonical bases already selected by the application resolver, never caller assertions. */ + knowledgeBaseIds?: readonly string[] + signal?: AbortSignal } /** @@ -131,10 +139,10 @@ export interface KnowledgeAccessScopeContext { * really owns it. Nothing here is cached: revoking a credential or leaving a * group is visible on the next read. */ -async function loadUserAccessTokens( +async function loadUserAccess( userId: string, context: KnowledgeAccessScopeContext -): Promise { +): Promise> { const { workspaceId, organizationId } = context const scope = resourceScopeFromOwner(context) const baseline = organizationId ? ORGANIZATION_ACCESS_TOKENS : WORKSPACE_ACCESS_TOKENS @@ -150,10 +158,10 @@ async function loadUserAccessTokens( .from(member) .where(and(eq(member.organizationId, scope.organizationId), eq(member.userId, userId))) .limit(1) - if (!membership) return [] + if (!membership) return { tokens: [] } } else { const workspaceAccess = await checkWorkspaceAccess(scope.workspaceId, userId) - if (!workspaceAccess.hasAccess) return [] + if (!workspaceAccess.hasAccess) return { tokens: [] } } /** * An identity token only counts where permission-aware knowledge is on, so @@ -164,13 +172,14 @@ async function loadUserAccessTokens( */ const availability = await resolveKnowledgeAccessAvailability(context) if (!availability.memberScoped && !availability.sourceMirrored) { - return [...baseline] + return { tokens: [...baseline] } } const rows = await db .select({ emailIsAmbiguous: emailHeldByAnotherAccount, email: foldedEmail(user.email), + credentialId: credential.id, providerId: credential.providerId, providerTenantId: credential.providerTenantId, providerSubjectId: credential.providerSubjectId, @@ -218,14 +227,18 @@ async function loadUserAccessTokens( userId, workspaceId, }) - return [...baseline] + return { tokens: [...baseline] } } const identityTokens = new Set() + const githubReaders: GitHubReaderCredential[] = [] for (const row of rows) { if (!availability.memberScoped || !row.providerSubjectId) continue try { - identityTokens.add(subjectToken(row)) + const token = subjectToken(row) + identityTokens.add(token) + if (row.providerId === 'github-repositories' && row.credentialId) + githubReaders.push({ credentialId: row.credentialId, subjectToken: token }) } catch (error) { logger.warn('Skipping malformed managed credential subject', { userId, @@ -256,7 +269,19 @@ async function loadUserAccessTokens( } } - return sortAccessTokens(new Set([...baseline, ...identityTokens])) + return { + tokens: sortAccessTokens(new Set([...baseline, ...identityTokens])), + ...(githubReaders.length + ? { + githubInstallationGrants: await resolveGitHubInstallationReadGrants({ + scope, + readers: githubReaders, + knowledgeBaseIds: context.knowledgeBaseIds, + signal: context.signal, + }), + } + : {}), + } } /** @@ -286,7 +311,7 @@ export async function resolveKnowledgeAccessScope( return { kind: 'user', userId: subject.userId, - tokens: await loadUserAccessTokens(subject.userId, context), + ...(await loadUserAccess(subject.userId, context)), } } @@ -300,7 +325,7 @@ export async function resolveUserKnowledgeAccessScope( userId: string, workspaceId: string | undefined ): Promise { - return { kind: 'user', userId, tokens: await loadUserAccessTokens(userId, { workspaceId }) } + return { kind: 'user', userId, ...(await loadUserAccess(userId, { workspaceId })) } } /** Memoises {@link resolveKnowledgeAccessScope} for one operation; a failed lookup is retried on the next call. */ diff --git a/apps/sim/lib/knowledge/access/types.ts b/apps/sim/lib/knowledge/access/types.ts index c8aa195f50c..d09dfabe3ed 100644 --- a/apps/sim/lib/knowledge/access/types.ts +++ b/apps/sim/lib/knowledge/access/types.ts @@ -39,6 +39,16 @@ export interface UserAccessScope { * they belong to. */ tokens: readonly string[] + /** Live user-token evidence, scoped to the installation source's immutable repository. */ + githubInstallationGrants?: readonly GitHubInstallationReadGrant[] +} + +export interface GitHubInstallationReadGrant { + connectorId: string + contentCredentialId: string + readerCredentialId: string + readerSubjectToken: string + repositoryId: string } /** diff --git a/apps/sim/lib/knowledge/application/connector-access.ts b/apps/sim/lib/knowledge/application/connector-access.ts index bc4163d815d..ed5daa0312e 100644 --- a/apps/sim/lib/knowledge/application/connector-access.ts +++ b/apps/sim/lib/knowledge/application/connector-access.ts @@ -26,6 +26,7 @@ import { validateConnectorSourceConfig, } from '@/lib/knowledge/application/connectors' import { resolveActiveKnowledgeConnectorContext } from '@/lib/knowledge/application/contexts' +import { prepareGitHubInstallationSource } from '@/lib/knowledge/application/github-installation-source' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { type ConnectorAccessMode, @@ -198,7 +199,20 @@ export const updateKnowledgeConnectorAccess = defineAuthorizedKnowledgeUseCase({ 'Search sources must support per-person access or source permissions' ) } - const sourceConfig = connector.sourceConfig as Record + const previousConfig = connector.sourceConfig as Record + const sourceConfig = await prepareGitHubInstallationSource({ + connectorType: connector.connectorType, + credentialId: + input.credentialId === undefined && input.accessMode === connector.accessMode + ? connector.credentialId + : input.credentialId, + organizationId: context.organizationId, + isSearchIndex: context.knowledgeBase.isSearchIndex === true, + accessMode: input.accessMode, + actingUserId, + sourceConfig: previousConfig, + previousConfig, + }) let target: ConnectorAccessTarget if (input.accessMode === 'members') { diff --git a/apps/sim/lib/knowledge/application/connectors.ts b/apps/sim/lib/knowledge/application/connectors.ts index 853f08ca224..e8adeb13283 100644 --- a/apps/sim/lib/knowledge/application/connectors.ts +++ b/apps/sim/lib/knowledge/application/connectors.ts @@ -48,6 +48,7 @@ import { resolveActiveKnowledgeResourceContext, resolveKnowledgeWorkspaceContext, } from '@/lib/knowledge/application/contexts' +import { prepareGitHubInstallationSource } from '@/lib/knowledge/application/github-installation-source' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { type ConnectorAccessMode, @@ -765,13 +766,23 @@ async function executeCreateKnowledgeConnector( }) } } + const sourceConfig = await prepareGitHubInstallationSource({ + connectorType: input.connectorType, + credentialId: input.credentialId, + organizationId: context.organizationId, + isSearchIndex: context.knowledgeBase.isSearchIndex === true, + accessMode: input.accessMode ?? 'workspace', + actingUserId, + sourceConfig: membersBinding?.sourceConfig ?? input.sourceConfig, + }) + if (membersBinding) membersBinding = { ...membersBinding, sourceConfig } const outcome = await performCreateKnowledgeConnector({ knowledgeBase: connectorTarget(context), connectorType: input.connectorType, credentialId: input.credentialId, apiKey: input.apiKey, /** Members mode stores the config with its listing caps cleared. */ - sourceConfig: membersBinding?.sourceConfig ?? input.sourceConfig, + sourceConfig, syncIntervalMinutes: input.syncIntervalMinutes, membersBinding, accessMode: input.accessMode, @@ -787,7 +798,7 @@ async function executeCreateKnowledgeConnector( requestId, auth: connectorMeta.auth, accessMode: input.accessMode ?? 'workspace', - sourceConfig: input.sourceConfig, + sourceConfig, }), userId: actingUserId, source: input.source ?? 'agent', @@ -928,6 +939,17 @@ export const updateKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ knowledgeBase: connectorTarget(context), connectorId: context.connectorId, updates: input.updates, + prepareSourceConfig: (connector, sourceConfig) => + prepareGitHubInstallationSource({ + connectorType: connector.connectorType, + credentialId: connector.credentialId, + organizationId: context.organizationId, + isSearchIndex: context.knowledgeBase.isSearchIndex === true, + accessMode: connector.accessMode, + actingUserId, + sourceConfig, + previousConfig: connector.sourceConfig as Record, + }), resolveBillingAttribution: () => { const workspaceId = context.workspaceId return ( diff --git a/apps/sim/lib/knowledge/application/contexts.ts b/apps/sim/lib/knowledge/application/contexts.ts index 1738dbd70e5..0347e8fe312 100644 --- a/apps/sim/lib/knowledge/application/contexts.ts +++ b/apps/sim/lib/knowledge/application/contexts.ts @@ -164,7 +164,10 @@ export async function resolveActiveKnowledgeBaseContext( ...workspaceContext, knowledgeBaseId: knowledgeBase.id, knowledgeBase, - access: createKnowledgeAccessProvider(principal, { workspaceId: knowledgeBase.workspaceId }), + access: createKnowledgeAccessProvider(principal, { + workspaceId: knowledgeBase.workspaceId, + knowledgeBaseIds: [knowledgeBase.id], + }), } } @@ -185,7 +188,10 @@ export async function resolveActiveKnowledgeBaseInWorkspace( ...workspaceContext, knowledgeBaseId: knowledgeBase.id, knowledgeBase, - access: createKnowledgeAccessProvider(principal, { workspaceId: workspaceContext.workspaceId }), + access: createKnowledgeAccessProvider(principal, { + workspaceId: workspaceContext.workspaceId, + knowledgeBaseIds: [knowledgeBase.id], + }), } } @@ -251,7 +257,10 @@ export async function resolveActiveKnowledgeResourceContext( ...owner, knowledgeBaseId: knowledgeBase.id, knowledgeBase, - access: createKnowledgeAccessProvider(principal, owner), + access: createKnowledgeAccessProvider(principal, { + ...owner, + knowledgeBaseIds: [knowledgeBase.id], + }), } } if (!knowledgeBase.workspaceId) { @@ -263,7 +272,10 @@ export async function resolveActiveKnowledgeResourceContext( ...workspaceContext, knowledgeBaseId: knowledgeBase.id, knowledgeBase, - access: createKnowledgeAccessProvider(principal, { workspaceId: knowledgeBase.workspaceId }), + access: createKnowledgeAccessProvider(principal, { + workspaceId: knowledgeBase.workspaceId, + knowledgeBaseIds: [knowledgeBase.id], + }), } } diff --git a/apps/sim/lib/knowledge/application/github-installation-source.test.ts b/apps/sim/lib/knowledge/application/github-installation-source.test.ts new file mode 100644 index 00000000000..5eaee58ae6c --- /dev/null +++ b/apps/sim/lib/knowledge/application/github-installation-source.test.ts @@ -0,0 +1,153 @@ +/** @vitest-environment node */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const m = vi.hoisted(() => ({ + access: vi.fn(), + canUse: vi.fn(), + decrypt: vi.fn(), + parse: vi.fn(), + repository: vi.fn(), +})) +vi.mock('@/lib/credentials/access', () => ({ + getCredentialActorContext: m.access, + canUseCredential: m.canUse, +})) +vi.mock('@/lib/core/security/encryption', () => ({ decryptSecret: m.decrypt })) +vi.mock('@/lib/oauth/github-installation', () => ({ + parseGitHubInstallationBinding: m.parse, + resolveGitHubInstallationRepository: m.repository, +})) + +import { prepareGitHubInstallationSource } from '@/lib/knowledge/application/github-installation-source' + +const installed = { + id: 'installation-credential', + providerId: 'github-app-installation', + organizationId: 'org', + workspaceId: null, + type: 'service_account', + revokedAt: null, + encryptedServiceAccountKey: 'encrypted-binding', + providerSubjectId: '42', + providerTenantId: '7', +} +const input = { + connectorType: 'github', + credentialId: installed.id, + organizationId: 'org', + isSearchIndex: true, + accessMode: 'members', + actingUserId: 'admin', + sourceConfig: { repository: 'example/private' }, +} + +beforeEach(() => { + vi.clearAllMocks() + m.access.mockResolvedValue({ credential: installed }) + m.canUse.mockReturnValue(true) + m.decrypt.mockResolvedValue({ decrypted: '{}' }) + m.parse.mockReturnValue({ installationId: '42', accountId: '7' }) + m.repository.mockResolvedValue({ id: '123', fullName: 'example/private', defaultBranch: 'main' }) +}) + +describe('GitHub installation source identity', () => { + it('persists only the provider-attested repository ID even when the browser supplies another', async () => { + await expect( + prepareGitHubInstallationSource({ + ...input, + sourceConfig: { ...input.sourceConfig, githubRepositoryId: '999' }, + }) + ).resolves.toEqual({ repository: 'example/private', githubRepositoryId: '123' }) + expect(m.access).toHaveBeenCalledWith(installed.id, 'admin') + }) + it.each([ + { organizationId: undefined }, + { isSearchIndex: false }, + { accessMode: 'admin' }, + { accessMode: 'workspace' }, + ])('requires organization Search and member access: %j', async (change) => { + await expect(prepareGitHubInstallationSource({ ...input, ...change })).rejects.toMatchObject({ + code: 'validation', + }) + expect(m.repository).not.toHaveBeenCalled() + }) + it.each([ + { organizationId: 'other-org' }, + { workspaceId: 'workspace' }, + { type: 'oauth' }, + { revokedAt: new Date() }, + { encryptedServiceAccountKey: null }, + ])('refuses unusable or cross-scope installation credentials: %j', async (change) => { + m.access.mockResolvedValue({ credential: { ...installed, ...change } }) + await expect(prepareGitHubInstallationSource(input)).rejects.toMatchObject({ + code: 'forbidden', + }) + expect(m.decrypt).not.toHaveBeenCalled() + }) + it('refuses credentials the acting user cannot use', async () => { + m.canUse.mockReturnValue(false) + await expect(prepareGitHubInstallationSource(input)).rejects.toMatchObject({ + code: 'forbidden', + }) + }) + it('bounds encrypted binding data before decryption', async () => { + m.access.mockResolvedValue({ + credential: { ...installed, encryptedServiceAccountKey: 'x'.repeat(16_385) }, + }) + await expect(prepareGitHubInstallationSource(input)).rejects.toMatchObject({ + code: 'validation', + }) + expect(m.decrypt).not.toHaveBeenCalled() + }) + it('refuses mismatched installation identity', async () => { + m.parse.mockReturnValue({ installationId: 'evil', accountId: '7' }) + await expect(prepareGitHubInstallationSource(input)).rejects.toMatchObject({ + code: 'validation', + }) + expect(m.repository).not.toHaveBeenCalled() + }) + it('refuses repository replacement while allowing a rename of the same immutable repository', async () => { + await expect( + prepareGitHubInstallationSource({ ...input, previousConfig: { githubRepositoryId: '456' } }) + ).rejects.toMatchObject({ code: 'validation' }) + await expect( + prepareGitHubInstallationSource({ + ...input, + previousConfig: { repository: 'old/name', githubRepositoryId: '123' }, + }) + ).resolves.toMatchObject({ githubRepositoryId: '123' }) + }) + it('keeps the marker when an edit omits it', async () => { + await expect( + prepareGitHubInstallationSource({ ...input, previousConfig: { githubRepositoryId: '123' } }) + ).resolves.toMatchObject({ githubRepositoryId: '123' }) + }) + it.each([null, { credential: { providerId: 'github-repositories' } }])( + 'cannot downgrade an existing installation by replacing or deleting its credential', + async (access) => { + m.access.mockResolvedValue(access) + await expect( + prepareGitHubInstallationSource({ ...input, previousConfig: { githubRepositoryId: '123' } }) + ).rejects.toMatchObject({ code: 'validation' }) + } + ) + it('leaves ordinary GitHub member setup available without an installation', async () => { + await expect( + prepareGitHubInstallationSource({ ...input, credentialId: undefined }) + ).resolves.toEqual(input.sourceConfig) + expect(m.repository).not.toHaveBeenCalled() + }) + it('does not accept the installation marker on ordinary member setup', async () => { + await expect( + prepareGitHubInstallationSource({ + ...input, + credentialId: undefined, + sourceConfig: { ...input.sourceConfig, githubRepositoryId: '123' }, + }) + ).rejects.toMatchObject({ code: 'validation' }) + }) + it('propagates a provider denial without producing a source binding', async () => { + m.repository.mockRejectedValue(new Error('Repository access denied')) + await expect(prepareGitHubInstallationSource(input)).rejects.toThrow('Repository access denied') + }) +}) diff --git a/apps/sim/lib/knowledge/application/github-installation-source.ts b/apps/sim/lib/knowledge/application/github-installation-source.ts new file mode 100644 index 00000000000..c289b4accc4 --- /dev/null +++ b/apps/sim/lib/knowledge/application/github-installation-source.ts @@ -0,0 +1,84 @@ +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { decryptSecret } from '@/lib/core/security/encryption' +import { canUseCredential, getCredentialActorContext } from '@/lib/credentials/access' +import { + parseGitHubInstallationBinding, + resolveGitHubInstallationRepository, +} from '@/lib/oauth/github-installation' +import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' + +interface GitHubInstallationSourceInput { + connectorType: string + credentialId?: string | null + organizationId?: string + isSearchIndex: boolean + accessMode: string + actingUserId: string + sourceConfig: Record + previousConfig?: Record +} + +/** Pins installation sources to a provider-verified repository identity before persisting them. */ +export async function prepareGitHubInstallationSource( + input: GitHubInstallationSourceInput +): Promise> { + const wasInstallation = input.previousConfig?.githubRepositoryId !== undefined + const assertedId = input.sourceConfig.githubRepositoryId + if (input.connectorType !== 'github') { + if (assertedId !== undefined) + throw new OrchestrationError( + 'validation', + 'githubRepositoryId is reserved for GitHub installation sources' + ) + return input.sourceConfig + } + const access = input.credentialId + ? await getCredentialActorContext(input.credentialId, input.actingUserId) + : null + const contentCredential = access?.credential + if (contentCredential?.providerId !== GITHUB_INSTALLATION_PROVIDER_ID) { + if (wasInstallation || assertedId !== undefined) + throw new OrchestrationError( + 'validation', + 'This source requires its GitHub installation. Create a new source to change the indexing method.' + ) + return input.sourceConfig + } + if (!input.organizationId || !input.isSearchIndex || input.accessMode !== 'members') + throw new OrchestrationError( + 'validation', + 'GitHub installations require organization Search with connected member access' + ) + if ( + !access || + !canUseCredential(access) || + contentCredential.organizationId !== input.organizationId || + contentCredential.workspaceId !== null || + contentCredential.type !== 'service_account' || + contentCredential.revokedAt || + !contentCredential.encryptedServiceAccountKey + ) + throw new OrchestrationError( + 'forbidden', + 'This GitHub installation is not available in this organization' + ) + if (contentCredential.encryptedServiceAccountKey.length > 16_384) + throw new OrchestrationError('validation', 'Reconnect this GitHub installation before using it') + const repository = input.sourceConfig.repository + if (typeof repository !== 'string' || !repository.trim()) + throw new OrchestrationError('validation', 'Choose a GitHub repository for this source') + const { decrypted } = await decryptSecret(contentCredential.encryptedServiceAccountKey) + const binding = parseGitHubInstallationBinding(JSON.parse(decrypted)) + if ( + binding.installationId !== contentCredential.providerSubjectId || + binding.accountId !== contentCredential.providerTenantId + ) + throw new OrchestrationError('validation', 'Reconnect this GitHub installation before using it') + const resolved = await resolveGitHubInstallationRepository(binding, repository.trim()) + if (wasInstallation && input.previousConfig?.githubRepositoryId !== resolved.id) + throw new OrchestrationError( + 'validation', + 'Create a new source to index a different GitHub repository' + ) + return { ...input.sourceConfig, repository: resolved.fullName, githubRepositoryId: resolved.id } +} diff --git a/apps/sim/lib/knowledge/application/github-installations.postgres.test.ts b/apps/sim/lib/knowledge/application/github-installations.postgres.test.ts new file mode 100644 index 00000000000..04e594b2d2a --- /dev/null +++ b/apps/sim/lib/knowledge/application/github-installations.postgres.test.ts @@ -0,0 +1,144 @@ +/** @vitest-environment node */ +import { db } from '@sim/db' +import type { PostgresJsDatabase } from 'drizzle-orm/postgres-js' +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { createEnterpriseSearchMigrationFixture } from '@/lib/knowledge/__integration__/migration-fixture' + +vi.unmock('drizzle-orm') +vi.unmock('@sim/db/schema') + +const mocks = vi.hoisted(() => ({ token: vi.fn(), list: vi.fn() })) +vi.mock('@/lib/knowledge/application/authorized-knowledge-use-case', () => ({ + defineAuthorizedKnowledgeUseCase: (definition: { + execute: (args: { + principal: { userId: string } + input: { organizationId: string } + context: { organizationId: string } + }) => Promise + }) => ({ + execute: (args: { principal: { userId: string }; input: { organizationId: string } }) => + definition.execute({ ...args, context: { organizationId: args.input.organizationId } }), + }), +})) +vi.mock('@/lib/knowledge/application/operations', () => ({ + knowledgeOperations: { listGitHubInstallations: {}, connectGitHubInstallation: {} }, +})) +vi.mock('@/lib/knowledge/application/contexts', () => ({ + resolveKnowledgeOrganizationContext: vi.fn(), +})) +vi.mock('@/lib/knowledge/access/availability', () => ({ + requireOrganizationSearchAvailable: vi.fn(), +})) +vi.mock('@/lib/credentials/managed-oauth', () => ({ resolveManagedOAuthToken: mocks.token })) +vi.mock('@/lib/credential-groups/provider-registry', () => ({ + getCredentialGroupProviderAdapter: () => ({ + getPolicy: async () => ({ authorizationAppId: 'current-app', scopeVersion: 1 }), + }), +})) +vi.mock('@/lib/oauth/github-installation', () => ({ + getGitHubInstallationConfiguration: () => ({ + configured: true, + installUrl: 'https://github.com/apps/example/installations/new', + }), + listUserAdminGitHubInstallations: mocks.list, + verifyGitHubInstallationBinding: vi.fn(), +})) +vi.mock('@/lib/core/security/encryption', () => ({ encryptSecret: vi.fn() })) + +import { listGitHubSearchInstallations } from '@/lib/knowledge/application/github-installations' + +const { drizzle } = await import('drizzle-orm/postgres-js') +const schema = await import('@sim/db/schema') +const databaseUrl = process.env.KNOWLEDGE_ACL_TEST_DATABASE_URL + +/** Exercises the production reader-selection SQL against isolated local PostgreSQL tables. */ +describe.runIf(Boolean(databaseUrl))( + 'GitHub installation setup reader ownership in PostgreSQL', + () => { + let fixture: Awaited> + let executor: PostgresJsDatabase + + beforeAll(async () => { + fixture = await createEnterpriseSearchMigrationFixture(databaseUrl!) + await fixture.migrate() + await fixture.client.unsafe(` + ALTER TABLE credential ADD COLUMN revoked_at timestamp, + ADD COLUMN credential_group_option_id text, ADD COLUMN managed_oauth_scope_version integer; + ALTER TABLE credential_group ADD COLUMN options jsonb NOT NULL DEFAULT '[]'; + ALTER TABLE credential_group_enrollment ADD COLUMN user_id text; + INSERT INTO organization(id) VALUES ('setup-org'), ('other-org'); + `) + executor = drizzle(fixture.client, { schema }) + vi.mocked(db.select).mockImplementation(executor.select.bind(executor)) + }) + + afterAll(async () => { + await fixture?.cleanup() + }) + + beforeEach(async () => { + vi.clearAllMocks() + mocks.token.mockResolvedValue({ accessToken: 'ghu_alice' }) + mocks.list.mockResolvedValue([]) + await fixture.client.unsafe(` + TRUNCATE credential, credential_group_enrollment, credential_group CASCADE; + INSERT INTO credential_group(id,organization_id,name,status,options) + VALUES ('setup-group','setup-org','GitHub','active', + '[{"id":"github-option","provider":"github-repositories","status":"active"}]'); + INSERT INTO credential_group_enrollment(id,credential_group_id,email,status,user_id) + VALUES ('setup-enrollment','setup-group','alice@example.com','completed','alice'); + INSERT INTO credential(id,organization_id,type,provider_id,provider_subject_id,authorization_app_id, + managed_oauth_status,managed_oauth_scope_version,granted_scopes,encrypted_oauth_token_set,granted_at, + credential_group_enrollment_id,credential_group_option_id) + VALUES ('alice-github','setup-org','managed_oauth','github-repositories','123','current-app','active',1, + ARRAY[]::text[],'encrypted',now(),'setup-enrollment','github-option'); + `) + }) + + function list(userId = 'alice', organizationId = 'setup-org') { + return listGitHubSearchInstallations.execute({ + principal: { kind: 'session', userId, sessionId: 'session' }, + input: { organizationId }, + }) + } + + it('selects the acting person’s active credential in the canonical organization', async () => { + expect(await list()).toMatchObject({ needsUserConnection: false }) + expect(mocks.token).toHaveBeenCalledWith({ + credentialId: 'alice-github', + organizationId: 'setup-org', + expectedProviderId: 'github-repositories', + requiredScopes: [], + }) + expect(await list('bob')).toMatchObject({ needsUserConnection: true }) + expect(await list('alice', 'other-org')).toMatchObject({ needsUserConnection: true }) + expect(mocks.token).toHaveBeenCalledTimes(1) + }) + + it.each([ + ['credential owner', "UPDATE credential SET organization_id='other-org'"], + ['group owner', "UPDATE credential_group SET organization_id='other-org'"], + ['credential revocation', 'UPDATE credential SET revoked_at=now()'], + ['enrollment revocation', 'UPDATE credential_group_enrollment SET revoked_at=now()'], + ['revoked enrollment status', "UPDATE credential_group_enrollment SET status='revoked'"], + ['inactive credential', "UPDATE credential SET managed_oauth_status='revoked'"], + ['inactive group', "UPDATE credential_group SET status='archived'"], + [ + 'disabled option', + `UPDATE credential_group SET options='[{"id":"github-option","provider":"github-repositories","status":"disabled"}]'`, + ], + [ + 'different option provider', + `UPDATE credential_group SET options='[{"id":"github-option","provider":"other","status":"active"}]'`, + ], + ['different option', "UPDATE credential SET credential_group_option_id='other-option'"], + ['stale app identity', "UPDATE credential SET authorization_app_id='old-app'"], + ['stale scope policy', 'UPDATE credential SET managed_oauth_scope_version=0'], + ])('denies %s before using any GitHub token', async (_name, mutation) => { + await fixture.client.unsafe(mutation) + expect(await list()).toMatchObject({ needsUserConnection: true, installations: [] }) + expect(mocks.token).not.toHaveBeenCalled() + expect(mocks.list).not.toHaveBeenCalled() + }) + } +) diff --git a/apps/sim/lib/knowledge/application/github-installations.test.ts b/apps/sim/lib/knowledge/application/github-installations.test.ts new file mode 100644 index 00000000000..645180d446e --- /dev/null +++ b/apps/sim/lib/knowledge/application/github-installations.test.ts @@ -0,0 +1,208 @@ +/** @vitest-environment node */ +import { db } from '@sim/db' +import { credential, credentialGroup, member } from '@sim/db/schema' +import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const m = vi.hoisted(() => ({ + configuration: vi.fn(), + list: vi.fn(), + verify: vi.fn(), + token: vi.fn(), + encrypt: vi.fn(), + audit: vi.fn(), +})) +vi.mock('@sim/audit', () => ({ + AuditAction: { + CREDENTIAL_CREATED: 'credential.created', + CREDENTIAL_UPDATED: 'credential.updated', + }, + AuditResourceType: { CREDENTIAL: 'credential' }, + recordAudit: m.audit, +})) +vi.mock('@/lib/knowledge/application/contexts', () => ({ + resolveKnowledgeOrganizationContext: async ({ organizationId }: { organizationId: string }) => ({ + organizationId, + workspaceId: undefined, + }), +})) +vi.mock('@/lib/knowledge/access/availability', () => ({ + requireOrganizationSearchAvailable: vi.fn(), +})) +vi.mock('@/lib/permission-groups/resolve.server', () => ({ + getUserPermissionConfigForOrganization: async () => null, +})) +vi.mock('@/lib/oauth/github-installation', () => ({ + getGitHubInstallationConfiguration: m.configuration, + listUserAdminGitHubInstallations: m.list, + verifyGitHubInstallationBinding: m.verify, +})) +vi.mock('@/lib/credentials/managed-oauth', () => ({ resolveManagedOAuthToken: m.token })) +vi.mock('@/lib/credential-groups/provider-registry', () => ({ + getCredentialGroupProviderAdapter: () => ({ + getPolicy: async () => ({ authorizationAppId: 'current-app', scopeVersion: 1 }), + }), +})) +vi.mock('@/lib/core/security/encryption', () => ({ encryptSecret: m.encrypt })) + +import { + connectGitHubSearchInstallation, + listGitHubSearchInstallations, +} from '@/lib/knowledge/application/github-installations' + +const principal = { kind: 'session', userId: 'admin', sessionId: 'session' } as const +const input = { organizationId: 'org', installationId: '42' } +const reader = { id: 'reader', authorizationAppId: 'current-app', groupId: 'group', subjectId: '9' } +const binding = { + type: 'github_app_installation', + version: 1, + appId: '1', + appClientId: 'client', + installationId: '42', + accountId: '7', + accountLogin: 'example', + accountType: 'Organization', + repositorySelection: 'selected', +} +const connect = () => connectGitHubSearchInstallation.execute({ principal, input }) + +beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + m.configuration.mockReturnValue({ + configured: true, + installUrl: 'https://github.com/apps/example/installations/new', + }) + m.list.mockResolvedValue([binding]) + m.token.mockResolvedValue({ accessToken: 'ghu_reader' }) + m.verify.mockResolvedValue(binding) + m.encrypt.mockResolvedValue({ encrypted: 'encrypted-installation-binding' }) +}) +afterAll(resetDbChainMock) + +function setupReader() { + queueTableRows(member, [{ role: 'admin' }]) + queueTableRows(credential, [reader]) +} +function setupTransaction(currentReader = reader, existing: { id: string }[] = []) { + queueTableRows(member, [{ id: 'admin-membership' }]) + queueTableRows(credentialGroup, [{ id: 'group' }]) + queueTableRows(credential, [currentReader]) + queueTableRows(credential, existing) +} + +describe('GitHub Search installation application operations', () => { + it.each(['member', undefined])( + 'requires a current Sim organization admin before provider access: %s', + async (role) => { + queueTableRows(member, role ? [{ role }] : []) + await expect(connect()).rejects.toMatchObject({ code: role ? 'forbidden' : 'not_found' }) + expect(m.token).not.toHaveBeenCalled() + expect(m.verify).not.toHaveBeenCalled() + expect(db.transaction).not.toHaveBeenCalled() + } + ) + it('refuses API keys for installation setup', async () => { + await expect( + connectGitHubSearchInstallation.execute({ + principal: { kind: 'personal_api_key', userId: 'admin', keyId: 'key' }, + input, + }) + ).rejects.toMatchObject({ code: 'forbidden' }) + expect(m.verify).not.toHaveBeenCalled() + }) + it('reports unavailable configuration without exposing installations', async () => { + queueTableRows(member, [{ role: 'admin' }]) + m.configuration.mockReturnValue({ configured: false, installUrl: null }) + await expect(listGitHubSearchInstallations.execute({ principal, input })).resolves.toEqual({ + available: false, + installUrl: null, + needsUserConnection: false, + installations: [], + }) + expect(m.list).not.toHaveBeenCalled() + }) + it('asks for the current admin’s own connection when none is available', async () => { + queueTableRows(member, [{ role: 'admin' }]) + queueTableRows(credential, []) + await expect( + listGitHubSearchInstallations.execute({ principal, input }) + ).resolves.toMatchObject({ needsUserConnection: true, installations: [] }) + expect(m.token).not.toHaveBeenCalled() + }) + it('uses only the organization-bound managed reader token to list installations', async () => { + setupReader() + const signal = new AbortController().signal + await listGitHubSearchInstallations.execute({ principal, input: { ...input, signal } }) + expect(m.token).toHaveBeenCalledWith({ + credentialId: 'reader', + organizationId: 'org', + expectedProviderId: 'github-repositories', + requiredScopes: [], + }) + expect(m.list).toHaveBeenCalledWith('ghu_reader', { signal }) + }) + it('reverifies GitHub admin authority before persisting an installation', async () => { + setupReader() + m.verify.mockRejectedValue(new Error('GitHub administrator access required')) + await expect(connect()).rejects.toThrow('GitHub administrator access required') + expect(m.encrypt).not.toHaveBeenCalled() + expect(db.transaction).not.toHaveBeenCalled() + }) + it('persists an encrypted installation binding and grants management to the actual admin', async () => { + setupReader() + setupTransaction() + const result = await connect() + expect(result).toMatchObject({ + created: true, + credential: { displayName: 'GitHub App · example' }, + }) + expect(m.verify).toHaveBeenCalledWith('ghu_reader', '42', { signal: undefined }) + expect(m.encrypt).toHaveBeenCalledWith(JSON.stringify(binding)) + expect(dbChainMockFns.values).toHaveBeenCalledWith( + expect.objectContaining({ + organizationId: 'org', + workspaceId: null, + providerId: 'github-app-installation', + type: 'service_account', + createdBy: 'admin', + encryptedServiceAccountKey: 'encrypted-installation-binding', + }) + ) + expect(dbChainMockFns.values).toHaveBeenCalledWith( + expect.objectContaining({ + credentialId: result.credential.id, + userId: 'admin', + role: 'admin', + status: 'active', + }) + ) + expect(m.audit).toHaveBeenCalledWith( + expect.objectContaining({ + actorId: 'admin', + resourceId: result.credential.id, + action: 'credential.created', + }) + ) + }) + it('reuses the same installation credential on repeat setup', async () => { + setupReader() + setupTransaction(reader, [{ id: 'existing' }]) + await expect(connect()).resolves.toMatchObject({ + created: false, + credential: { id: 'existing' }, + }) + }) + it('refuses if Sim administrator access was removed during GitHub verification', async () => { + setupReader() + queueTableRows(member, []) + await expect(connect()).rejects.toMatchObject({ code: 'forbidden' }) + expect(dbChainMockFns.values).not.toHaveBeenCalled() + }) + it('refuses if the reader identity changed during GitHub verification', async () => { + setupReader() + setupTransaction({ ...reader, subjectId: 'different-person' }) + await expect(connect()).rejects.toMatchObject({ code: 'conflict' }) + expect(dbChainMockFns.values).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/application/github-installations.ts b/apps/sim/lib/knowledge/application/github-installations.ts new file mode 100644 index 00000000000..917e9569cfb --- /dev/null +++ b/apps/sim/lib/knowledge/application/github-installations.ts @@ -0,0 +1,242 @@ +import { AuditAction, AuditResourceType } from '@sim/audit' +import { db } from '@sim/db' +import { + credential, + credentialGroup, + credentialGroupEnrollment, + credentialMember, + member, +} from '@sim/db/schema' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray, isNull, sql } from 'drizzle-orm' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { encryptSecret } from '@/lib/core/security/encryption' +import { LIVE_ENROLLMENT_STATUSES } from '@/lib/credential-groups/credentials' +import { getCredentialGroupProviderAdapter } from '@/lib/credential-groups/provider-registry' +import { resolveManagedOAuthToken } from '@/lib/credentials/managed-oauth' +import type { DbOrTx } from '@/lib/db/types' +import { requireOrganizationSearchAvailable } from '@/lib/knowledge/access/availability' +import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' +import { resolveKnowledgeOrganizationContext } from '@/lib/knowledge/application/contexts' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { + getGitHubInstallationConfiguration, + listUserAdminGitHubInstallations, + verifyGitHubInstallationBinding, +} from '@/lib/oauth/github-installation' +import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' + +interface InstallationInput { + organizationId: string + signal?: AbortSignal +} + +interface ConnectInstallationInput extends InstallationInput { + installationId: string +} + +/** Selects only the acting person's live, organization-bound GitHub connection. */ +async function findReaderCredential(executor: DbOrTx, organizationId: string, userId: string) { + const policy = await getCredentialGroupProviderAdapter('github-repositories').getPolicy( + undefined, + { organizationId } + ) + const rows = await executor + .select({ + id: credential.id, + authorizationAppId: credential.authorizationAppId, + groupId: credentialGroup.id, + subjectId: credential.providerSubjectId, + }) + .from(credential) + .innerJoin( + credentialGroupEnrollment, + eq(credentialGroupEnrollment.id, credential.credentialGroupEnrollmentId) + ) + .innerJoin(credentialGroup, eq(credentialGroup.id, credentialGroupEnrollment.credentialGroupId)) + .where( + and( + eq(credential.organizationId, organizationId), + eq(credentialGroup.organizationId, organizationId), + eq(credentialGroupEnrollment.userId, userId), + eq(credential.type, 'managed_oauth'), + eq(credential.providerId, 'github-repositories'), + eq(credential.managedOauthStatus, 'active'), + eq(credential.authorizationAppId, policy.authorizationAppId), + eq(credential.managedOauthScopeVersion, policy.scopeVersion), + isNull(credential.revokedAt), + eq(credentialGroup.status, 'active'), + inArray(credentialGroupEnrollment.status, [...LIVE_ENROLLMENT_STATUSES]), + isNull(credentialGroupEnrollment.revokedAt), + sql`EXISTS (SELECT 1 FROM jsonb_array_elements(${credentialGroup.options}) AS option + WHERE option->>'id' = ${credential.credentialGroupOptionId} + AND option->>'provider' = 'github-repositories' AND option->>'status' = 'active')` + ) + ) + .limit(2) + if (rows.length > 1) + throw new OrchestrationError( + 'conflict', + 'Connect one GitHub account for this organization before choosing an installation' + ) + return rows[0] ?? null +} + +async function readerToken(organizationId: string, credentialId: string) { + return resolveManagedOAuthToken({ + credentialId, + organizationId, + expectedProviderId: 'github-repositories', + requiredScopes: [], + }) +} + +export const listGitHubSearchInstallations = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.listGitHubInstallations, + resolveContext: ({ input }: { input: InstallationInput }) => + resolveKnowledgeOrganizationContext(input), + async execute({ principal, input, context }) { + await requireOrganizationSearchAvailable(context.organizationId) + const configuration = getGitHubInstallationConfiguration() + const reader = configuration.configured + ? await findReaderCredential(db, context.organizationId, principal.userId) + : null + const installations = reader + ? await listUserAdminGitHubInstallations( + (await readerToken(context.organizationId, reader.id)).accessToken, + { signal: input.signal } + ) + : [] + return { + available: configuration.configured, + installUrl: configuration.installUrl, + needsUserConnection: configuration.configured && !reader, + installations, + } + }, +}) + +export const connectGitHubSearchInstallation = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.connectGitHubInstallation, + resolveContext: ({ input }: { input: ConnectInstallationInput }) => + resolveKnowledgeOrganizationContext(input), + async execute({ principal, input, context }) { + await requireOrganizationSearchAvailable(context.organizationId) + if (!getGitHubInstallationConfiguration().configured) + throw new OrchestrationError( + 'validation', + 'GitHub App installation indexing is not configured for this environment' + ) + const reader = await findReaderCredential(db, context.organizationId, principal.userId) + if (!reader) + throw new OrchestrationError( + 'validation', + 'Connect your GitHub account before choosing an installation' + ) + const { accessToken } = await readerToken(context.organizationId, reader.id) + const binding = await verifyGitHubInstallationBinding(accessToken, input.installationId, { + signal: input.signal, + }) + const { encrypted } = await encryptSecret(JSON.stringify(binding)) + return db.transaction(async (tx) => { + await tx.execute( + sql`SELECT pg_advisory_xact_lock(hashtextextended(${`github-search:${context.organizationId}:${binding.installationId}`}, 0))` + ) + const [admin] = await tx + .select({ id: member.id }) + .from(member) + .where( + and( + eq(member.organizationId, context.organizationId), + eq(member.userId, principal.userId), + inArray(member.role, ['admin', 'owner']) + ) + ) + .for('update') + .limit(1) + if (!admin) + throw new OrchestrationError('forbidden', 'Organization administrator access is required') + /** Lock the group before rechecking enrollment, as account configuration and revocation do. */ + await tx + .select({ id: credentialGroup.id }) + .from(credentialGroup) + .where( + and( + eq(credentialGroup.id, reader.groupId), + eq(credentialGroup.organizationId, context.organizationId) + ) + ) + .for('update') + .limit(1) + const current = await findReaderCredential(tx, context.organizationId, principal.userId) + if ( + current?.id !== reader.id || + current.authorizationAppId !== reader.authorizationAppId || + current.subjectId !== reader.subjectId + ) + throw new OrchestrationError( + 'conflict', + 'Your GitHub connection changed during setup. Try again.' + ) + const [existing] = await tx + .select({ id: credential.id }) + .from(credential) + .where( + and( + eq(credential.organizationId, context.organizationId), + eq(credential.type, 'service_account'), + eq(credential.providerId, GITHUB_INSTALLATION_PROVIDER_ID), + eq(credential.providerSubjectId, binding.installationId), + eq(credential.authorizationAppId, reader.authorizationAppId!) + ) + ) + .for('update') + .limit(1) + const id = existing?.id ?? generateId() + const now = new Date() + const displayName = `GitHub App · ${binding.accountLogin}` + const values = { + displayName, + encryptedServiceAccountKey: encrypted, + providerTenantId: binding.accountId, + revokedAt: null, + updatedAt: now, + } + if (existing) await tx.update(credential).set(values).where(eq(credential.id, id)) + else + await tx.insert(credential).values({ + id, + organizationId: context.organizationId, + workspaceId: null, + type: 'service_account', + providerId: GITHUB_INSTALLATION_PROVIDER_ID, + providerSubjectId: binding.installationId, + authorizationAppId: reader.authorizationAppId, + createdBy: principal.userId, + ...values, + }) + await tx + .insert(credentialMember) + .values({ + id: generateId(), + credentialId: id, + userId: principal.userId, + role: 'admin', + status: 'active', + joinedAt: now, + }) + .onConflictDoUpdate({ + target: [credentialMember.credentialId, credentialMember.userId], + set: { role: 'admin', status: 'active', joinedAt: now, updatedAt: now }, + }) + return { credential: { id, displayName }, created: !existing } + }) + }, + projectAudit: ({ result }) => ({ + action: result.created ? AuditAction.CREDENTIAL_CREATED : AuditAction.CREDENTIAL_UPDATED, + resourceType: AuditResourceType.CREDENTIAL, + resourceId: result.credential.id, + resourceName: result.credential.displayName, + description: 'Connected a GitHub App installation for Search indexing', + }), +}) diff --git a/apps/sim/lib/knowledge/application/operations.ts b/apps/sim/lib/knowledge/application/operations.ts index b72dadcf115..0e7ae360969 100644 --- a/apps/sim/lib/knowledge/application/operations.ts +++ b/apps/sim/lib/knowledge/application/operations.ts @@ -102,6 +102,24 @@ const HUMAN_COPILOT_AND_EXECUTOR_PRINCIPAL_POLICY = { } as const export const knowledgeOperations = { + listGitHubInstallations: defineKnowledgeOperation( + defineWorkspaceOperation({ + id: 'knowledge.github.installations.list', + minimumRole: 'admin', + workspaceApiKey: 'deny', + capability: 'knowledge.use', + principalKinds: ['session'], + }) + ), + connectGitHubInstallation: defineKnowledgeOperation( + defineWorkspaceOperation({ + id: 'knowledge.github.installations.connect', + minimumRole: 'admin', + workspaceApiKey: 'deny', + capability: 'knowledge.use', + principalKinds: ['session'], + }) + ), prepareSlackInstallation: defineKnowledgeOperation( defineWorkspaceOperation({ id: 'knowledge.slack.prepare', diff --git a/apps/sim/lib/knowledge/application/search.ts b/apps/sim/lib/knowledge/application/search.ts index 946688eb782..84d25e239b7 100644 --- a/apps/sim/lib/knowledge/application/search.ts +++ b/apps/sim/lib/knowledge/application/search.ts @@ -229,7 +229,11 @@ async function resolveKnowledgeSearchContext( return { ...context, knowledgeBases: knowledgeBases as ActiveKnowledgeBaseReference[], - access: createKnowledgeAccessProvider(principal, context), + access: createKnowledgeAccessProvider(principal, { + ...context, + knowledgeBaseIds: knowledgeBases.map((base) => base!.id), + signal: input.signal, + }), } } if (!canonicalWorkspaceId) { @@ -241,7 +245,11 @@ async function resolveKnowledgeSearchContext( return { ...workspaceContext, knowledgeBases: knowledgeBases as ActiveKnowledgeBaseReference[], - access: createKnowledgeAccessProvider(principal, { workspaceId: canonicalWorkspaceId }), + access: createKnowledgeAccessProvider(principal, { + workspaceId: canonicalWorkspaceId, + knowledgeBaseIds: knowledgeBases.map((base) => base!.id), + signal: input.signal, + }), } } diff --git a/apps/sim/lib/knowledge/chunks/keyset-sql.test.ts b/apps/sim/lib/knowledge/chunks/keyset-sql.test.ts index 71ece9c4b13..ee1ccd8e2cf 100644 --- a/apps/sim/lib/knowledge/chunks/keyset-sql.test.ts +++ b/apps/sim/lib/knowledge/chunks/keyset-sql.test.ts @@ -108,7 +108,7 @@ describe('chunk list generated SQL', () => { expect(where.sql).toContain('"document"."acl" && ARRAY[$2, $3]::text[]') expect(where.sql).toContain('required_clause.tokens ?| ARRAY[$4, $5]::text[]') expect(where.sql).toContain( - '"knowledge_connector_member"."subject_token" = ANY(ARRAY[$7, $8]::text[])' + '"knowledge_connector_member"."subject_token" = ANY(ARRAY[$8, $9]::text[])' ) expect(where.params).toEqual([ 'document-1', @@ -116,6 +116,7 @@ describe('chunk list generated SQL', () => { 'ws', 'pub', 'ws', + 'github-app-installation', SOURCE_ACL_MAX_AGE_MS, 'pub', 'ws', diff --git a/apps/sim/lib/knowledge/connectors/access-token.test.ts b/apps/sim/lib/knowledge/connectors/access-token.test.ts index 8e0d59c34dc..4da693c4d03 100644 --- a/apps/sim/lib/knowledge/connectors/access-token.test.ts +++ b/apps/sim/lib/knowledge/connectors/access-token.test.ts @@ -100,6 +100,24 @@ describe('resolveConnectorAccessToken', () => { expect(mockDecryptApiKey).not.toHaveBeenCalled() }) + it('passes the immutable repository scope to installation token resolution', async () => { + await resolveConnectorAccessToken({ + auth: { mode: 'oauth', provider: 'github-repositories' }, + connector: credentialConnector('installation-credential'), + userId: 'actor', + requestId: 'request', + sourceConfig: { repository: 'team/repo', githubRepositoryId: '101' }, + }) + expect(mockResolveTokenBundle).toHaveBeenCalledWith( + 'installation-credential', + 'actor', + 'request', + undefined, + undefined, + { githubRepositoryScope: { repository: 'team/repo', repositoryId: '101' } } + ) + }) + it('does not accept an undeclared key alternative on other OAuth connectors', async () => { await expect( resolveConnectorAccessToken({ diff --git a/apps/sim/lib/knowledge/connectors/access-token.ts b/apps/sim/lib/knowledge/connectors/access-token.ts index 2a46a10d323..8fc29e5b98b 100644 --- a/apps/sim/lib/knowledge/connectors/access-token.ts +++ b/apps/sim/lib/knowledge/connectors/access-token.ts @@ -101,12 +101,26 @@ export async function resolveConnectorAccessToken(params: { } const subject = connectorServiceAccountSubject(auth, params.sourceConfig) + const githubRepositoryScope = + auth.mode === 'oauth' && auth.provider === 'github-repositories' + ? { + repositoryId: + typeof params.sourceConfig.githubRepositoryId === 'string' + ? params.sourceConfig.githubRepositoryId + : undefined, + repository: + typeof params.sourceConfig.repository === 'string' + ? params.sourceConfig.repository + : undefined, + } + : undefined const bundle = await resolveCredentialTokenBundle( connector.credentialId, userId, requestId, connectorServiceAccountScopes(auth), - subject + subject, + ...(githubRepositoryScope ? [{ githubRepositoryScope }] : []) ) if (!bundle?.accessToken) return null diff --git a/apps/sim/lib/knowledge/orchestration/connector-access.ts b/apps/sim/lib/knowledge/orchestration/connector-access.ts index 0612398729d..bf4aab69bfc 100644 --- a/apps/sim/lib/knowledge/orchestration/connector-access.ts +++ b/apps/sim/lib/knowledge/orchestration/connector-access.ts @@ -363,6 +363,16 @@ export async function performUpdateKnowledgeConnectorAccess( credentialGroupId: target.binding.credentialGroupId, credentialGroupOptionId: target.binding.credentialGroupOptionId, }) + /** A new repository identity cannot inherit observations collected before it was verified. */ + if ( + existing.connectorType === 'github' && + target.binding.sourceConfig.githubRepositoryId !== + (existing.sourceConfig as Record).githubRepositoryId + ) { + await tx + .delete(knowledgeConnectorMember) + .where(eq(knowledgeConnectorMember.connectorId, connectorId)) + } const [row] = await tx .update(knowledgeConnector) .set({ diff --git a/apps/sim/lib/knowledge/orchestration/connectors.ts b/apps/sim/lib/knowledge/orchestration/connectors.ts index 5765d5b8604..8c8d44d4087 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.ts @@ -650,6 +650,11 @@ export interface PerformUpdateKnowledgeConnectorParams extends KnowledgeOperatio } /** Resolves the payer only when a source change will queue synchronization. */ resolveBillingAttribution: () => Promise + /** Canonicalizes provider identities through the authorized application caller. */ + prepareSourceConfig?: ( + connector: KnowledgeConnectorRow, + sourceConfig: Record + ) => Promise> /** * Validates a replacement `sourceConfig` against the live source. Supplied by * the caller because resolving the connector's token needs the requesting @@ -793,7 +798,9 @@ export async function performUpdateKnowledgeConnector( let sourceConfigToStore = updates.sourceConfig if (updates.sourceConfig !== undefined) { const accessMode = existing.accessMode as ConnectorAccessMode - let nextSourceConfig = updates.sourceConfig + let nextSourceConfig = params.prepareSourceConfig + ? await params.prepareSourceConfig(existing, updates.sourceConfig) + : updates.sourceConfig if (aclIsDerived(accessMode)) { /** A derived-ACL mode has no listing cap; a save may refuse one, never store one. */ const { CONNECTOR_REGISTRY } = await import('@/connectors/registry.server') @@ -909,6 +916,8 @@ export async function performUpdateKnowledgeConnector( isNull(knowledgeConnector.deletedAt), ] updateConditions.push(eq(knowledgeConnector.status, existing.status)) + if (sourceConfigToStore !== undefined) + updateConditions.push(eq(knowledgeConnector.updatedAt, existing.updatedAt)) if (syncsPerMember) { updateConditions.push(eq(knowledgeConnector.memberSyncStatus, existing.memberSyncStatus)) } diff --git a/apps/sim/lib/oauth/credential-service.ts b/apps/sim/lib/oauth/credential-service.ts index 292fd4921a9..bd68fa4f4cb 100644 --- a/apps/sim/lib/oauth/credential-service.ts +++ b/apps/sim/lib/oauth/credential-service.ts @@ -21,6 +21,14 @@ import { parseTokenServiceAccountSecretBlob, type TokenServiceAccountSecretBlob, } from '@/lib/credentials/token-service-accounts/server' +import { + parseGitHubInstallationBinding, + resolveGitHubInstallationAccessToken, +} from '@/lib/oauth/github-installation' +import { + GITHUB_INSTALLATION_PROVIDER_ID, + type GitHubInstallationRepositoryScope, +} from '@/lib/oauth/github-installation-types' import { isInstagramProvider, shouldProactivelyRefreshInstagramToken } from '@/lib/oauth/instagram' import { getMicrosoftRefreshTokenExpiry, @@ -63,6 +71,8 @@ export interface CredentialTokenResolutionOptions { * mode so selector and ordinary calls share the same locks and dead flags. */ privacyMode?: 'selector' + /** GitHub installation content tokens may only address one connector repository. */ + githubRepositoryScope?: GitHubInstallationRepositoryScope } function privateCredentialIdentity(namespace: string, value: string): string { @@ -635,6 +645,7 @@ interface ServiceAccountTokenOptions { scopes?: string[] impersonateEmail?: string privacyMode?: 'selector' + githubRepositoryScope?: GitHubInstallationRepositoryScope } type ServiceAccountTokenResolver = ( @@ -648,6 +659,40 @@ type ServiceAccountTokenResolver = ( * generically: the stored token IS the access token. */ const SERVICE_ACCOUNT_TOKEN_RESOLVERS: Record = { + [GITHUB_INSTALLATION_PROVIDER_ID]: async (credentialId, { githubRepositoryScope }) => { + if (!githubRepositoryScope) + throw new Error('GitHub installation tokens require a source repository') + const [row] = await db + .select({ + type: credential.type, + providerId: credential.providerId, + encryptedServiceAccountKey: credential.encryptedServiceAccountKey, + providerSubjectId: credential.providerSubjectId, + providerTenantId: credential.providerTenantId, + revokedAt: credential.revokedAt, + }) + .from(credential) + .where(eq(credential.id, credentialId)) + .limit(1) + if ( + row?.type !== 'service_account' || + row.providerId !== GITHUB_INSTALLATION_PROVIDER_ID || + row.revokedAt || + !row.encryptedServiceAccountKey || + row.encryptedServiceAccountKey.length > 16_384 + ) { + throw new Error('GitHub installation credential is unavailable') + } + const { decrypted } = await decryptSecret(row.encryptedServiceAccountKey) + const binding = parseGitHubInstallationBinding(JSON.parse(decrypted)) + if ( + row.providerSubjectId !== binding.installationId || + row.providerTenantId !== binding.accountId + ) { + throw new Error('GitHub installation credential identity does not match its binding') + } + return resolveGitHubInstallationAccessToken(binding, githubRepositoryScope) + }, [ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID]: async (credentialId) => { const secret = await getAtlassianServiceAccountSecret(credentialId) return { accessToken: secret.apiToken, cloudId: secret.cloudId, domain: secret.domain } diff --git a/apps/sim/lib/oauth/github-installation-credential.test.ts b/apps/sim/lib/oauth/github-installation-credential.test.ts new file mode 100644 index 00000000000..29657563997 --- /dev/null +++ b/apps/sim/lib/oauth/github-installation-credential.test.ts @@ -0,0 +1,99 @@ +/** @vitest-environment node */ +import { credential } from '@sim/db/schema' +import { queueTableRows, resetDbChainMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + decryptSecret: vi.fn(), + parseBinding: vi.fn(), + resolveToken: vi.fn(), +})) +vi.mock('@/lib/core/security/encryption', () => ({ decryptSecret: mocks.decryptSecret })) +vi.mock('@/lib/oauth/github-installation', () => ({ + parseGitHubInstallationBinding: mocks.parseBinding, + resolveGitHubInstallationAccessToken: mocks.resolveToken, +})) +vi.mock('@/lib/oauth/oauth', () => ({ OAUTH_PROVIDERS: {}, refreshOAuthToken: vi.fn() })) + +import { resolveServiceAccountToken } from '@/lib/oauth/credential-service' +import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' + +const row = { + type: 'service_account', + providerId: GITHUB_INSTALLATION_PROVIDER_ID, + providerSubjectId: '21', + providerTenantId: '11', + encryptedServiceAccountKey: 'encrypted', +} +const binding = { installationId: '21', accountId: '11' } + +beforeEach(() => { + resetDbChainMock() + vi.clearAllMocks() + mocks.decryptSecret.mockResolvedValue({ decrypted: JSON.stringify(binding) }) + mocks.parseBinding.mockReturnValue(binding) + mocks.resolveToken.mockResolvedValue({ accessToken: 'ghs_contents' }) +}) + +describe('installation credential token dispatch', () => { + it('requires a repository scope before reading or decrypting a credential', async () => { + await expect( + resolveServiceAccountToken('credential-1', GITHUB_INSTALLATION_PROVIDER_ID) + ).rejects.toThrow('require a source repository') + expect(mocks.decryptSecret).not.toHaveBeenCalled() + }) + + it('dispatches only the validated installation credential and exact repository scope', async () => { + queueTableRows(credential, [row]) + const scope = { repositoryId: '101', repository: 'team/repo' } + expect( + await resolveServiceAccountToken( + 'credential-1', + GITHUB_INSTALLATION_PROVIDER_ID, + undefined, + undefined, + { githubRepositoryScope: scope } + ) + ).toEqual({ accessToken: 'ghs_contents' }) + expect(mocks.resolveToken).toHaveBeenCalledWith(binding, scope) + }) + + it.each([ + { ...row, type: 'oauth' }, + { ...row, revokedAt: new Date() }, + { ...row, providerId: 'google-service-account' }, + { ...row, encryptedServiceAccountKey: 'x'.repeat(16_385) }, + ])('rejects incompatible or oversized credential rows before decrypting', async (invalidRow) => { + queueTableRows(credential, [invalidRow]) + await expect( + resolveServiceAccountToken( + 'credential-1', + GITHUB_INSTALLATION_PROVIDER_ID, + undefined, + undefined, + { githubRepositoryScope: { repositoryId: '101' } } + ) + ).rejects.toThrow('unavailable') + expect(mocks.decryptSecret).not.toHaveBeenCalled() + }) + + it.each([ + { ...row, providerSubjectId: '22' }, + { ...row, providerTenantId: '12' }, + ])( + 'refuses a credential whose stored columns disagree with the verified binding', + async (invalidRow) => { + queueTableRows(credential, [invalidRow]) + await expect( + resolveServiceAccountToken( + 'credential-1', + GITHUB_INSTALLATION_PROVIDER_ID, + undefined, + undefined, + { githubRepositoryScope: { repositoryId: '101' } } + ) + ).rejects.toThrow('does not match its binding') + expect(mocks.resolveToken).not.toHaveBeenCalled() + } + ) +}) diff --git a/apps/sim/lib/oauth/github-installation-types.ts b/apps/sim/lib/oauth/github-installation-types.ts new file mode 100644 index 00000000000..1aae8c76752 --- /dev/null +++ b/apps/sim/lib/oauth/github-installation-types.ts @@ -0,0 +1,23 @@ +/** Installation credentials supply repository content, never a person's access grants. */ +export const GITHUB_INSTALLATION_PROVIDER_ID = 'github-app-installation' as const + +export interface GitHubInstallationSummary { + appId: string + appClientId: string + installationId: string + accountId: string + accountType: 'User' | 'Organization' + accountLogin: string + repositorySelection: 'all' | 'selected' +} + +/** Provider-verified installation identity; the application's signing key stays server-owned. */ +export interface GitHubInstallationBinding extends GitHubInstallationSummary { + type: 'github_app_installation' + version: 1 +} + +export interface GitHubInstallationRepositoryScope { + repositoryId?: string + repository?: string +} diff --git a/apps/sim/lib/oauth/github-installation.test.ts b/apps/sim/lib/oauth/github-installation.test.ts new file mode 100644 index 00000000000..4267804d545 --- /dev/null +++ b/apps/sim/lib/oauth/github-installation.test.ts @@ -0,0 +1,331 @@ +/** @vitest-environment node */ +import { generateKeyPairSync, verify } from 'node:crypto' +import { resetEnvMock, setEnv } from '@sim/testing' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + assertGitHubInstallationActive, + assertGitHubInstallationRepositoryActive, + getGitHubInstallationConfiguration, + listUserAdminGitHubInstallations, + parseGitHubInstallationBinding, + resolveGitHubInstallationAccessToken, + resolveGitHubInstallationRepository, + verifyGitHubInstallationBinding, +} from '@/lib/oauth/github-installation' +import type { GitHubInstallationBinding } from '@/lib/oauth/github-installation-types' + +const { privateKey, publicKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }) +const privateKeyPem = privateKey.export({ type: 'pkcs8', format: 'pem' }).toString() +const user = { id: 9, type: 'User' } +const installation = { + id: 21, + app_id: 1, + client_id: 'Iv123', + account: { id: 11, login: 'team', type: 'Organization' }, + repository_selection: 'selected', + permissions: { contents: 'read', metadata: 'read' }, + suspended_at: null, +} +const membership = { state: 'active', role: 'admin', organization: { id: 11 }, user: { id: 9 } } +const binding: GitHubInstallationBinding = { + type: 'github_app_installation', + version: 1, + appId: '1', + appClientId: 'Iv123', + installationId: '21', + accountId: '11', + accountType: 'Organization', + accountLogin: 'team', + repositorySelection: 'selected', +} +let now = Date.UTC(2026, 8, 9) +const fetchMock = vi.fn() + +function json(value: unknown, status = 200) { + return new Response(JSON.stringify(value), { status }) +} + +function tokenResponse(repositoryId = 101, contents = true) { + return { + token: contents ? 'ghs_contents' : 'ghs_metadata', + expires_at: new Date(Date.now() + 60 * 60_000).toISOString(), + permissions: { metadata: 'read', ...(contents ? { contents: 'read' } : {}) }, + repositories: [{ id: repositoryId }], + } +} + +beforeEach(() => { + vi.useFakeTimers() + now += 86_400_000 + vi.setSystemTime(now) + setEnv({ + GITHUB_APP_ID: '1', + GITHUB_APP_CLIENT_ID: 'Iv123', + GITHUB_APP_CLIENT_SECRET: 'secret', + GITHUB_APP_PRIVATE_KEY: privateKeyPem, + GITHUB_APP_SLUG: 'sim-search', + }) + fetchMock.mockReset() + vi.stubGlobal('fetch', fetchMock) +}) + +afterEach(() => { + resetEnvMock() + vi.unstubAllGlobals() + vi.useRealTimers() +}) + +function mockDiscovery( + installs: unknown[] = [installation], + memberships: unknown[] = [membership] +) { + fetchMock.mockImplementation(async (input) => { + const path = new URL(String(input)).pathname + if (path === '/user') return json(user) + if (path === '/user/memberships/orgs') return json(memberships) + if (path === '/user/installations') + return json({ total_count: installs.length, installations: installs }) + if (path === '/app/installations/21') return json(installation) + throw new Error(`Unexpected request: ${path}`) + }) +} + +describe('GitHub installation setup', () => { + it('rechecks repository installation selection without caching a previous success', async () => { + fetchMock + .mockResolvedValueOnce(json(installation)) + .mockResolvedValueOnce(json({ message: 'Not Found' }, 404)) + await expect( + assertGitHubInstallationRepositoryActive(binding, 'team/repo') + ).resolves.toBeUndefined() + await expect(assertGitHubInstallationRepositoryActive(binding, 'team/repo')).rejects.toThrow() + expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([ + 'https://api.github.com/repos/team/repo/installation', + 'https://api.github.com/repos/team/repo/installation', + ]) + const headers = new Headers(fetchMock.mock.calls[0][1]?.headers) + expect(headers.get('authorization')?.split('.')).toHaveLength(3) + }) + + it.each([ + { ...installation, id: 22 }, + { ...installation, app_id: 2 }, + { ...installation, client_id: 'another-app' }, + { ...installation, account: { ...installation.account, id: 12 } }, + { ...installation, suspended_at: '2026-01-01T00:00:00Z' }, + { ...installation, permissions: { metadata: 'read' } }, + ])('rejects a removed, suspended, or rebound repository installation %#', async (current) => { + fetchMock.mockResolvedValueOnce(json(current)) + await expect(assertGitHubInstallationRepositoryActive(binding, 'team/repo')).rejects.toThrow( + 'unavailable or its account binding changed' + ) + }) + + it('requires a valid RSA app key and returns no credentials in readiness metadata', () => { + expect(getGitHubInstallationConfiguration()).toEqual({ + configured: true, + installUrl: 'https://github.com/apps/sim-search/installations/new', + }) + setEnv({ GITHUB_APP_PRIVATE_KEY: 'invalid' }) + expect(getGitHubInstallationConfiguration()).toEqual({ configured: false, installUrl: null }) + }) + + it('lists only owned personal accounts and active organization-owner installations', async () => { + mockDiscovery([ + installation, + { ...installation, id: 22, account: { id: 9, login: 'me', type: 'User' } }, + { ...installation, id: 23, account: { id: 10, login: 'other', type: 'User' } }, + { + ...installation, + id: 24, + account: { id: 12, login: 'read-only-org', type: 'Organization' }, + }, + { ...installation, id: 25, suspended_at: '2026-01-01T00:00:00Z' }, + { ...installation, id: 26, app_id: 2 }, + ]) + expect( + (await listUserAdminGitHubInstallations('ghu_user')).map((entry) => entry.installationId) + ).toEqual(['21', '22']) + expect(fetchMock.mock.calls.some(([url]) => String(url).includes('/app/installations'))).toBe( + false + ) + }) + + it('never treats read-visible installation membership as authority to bind', async () => { + mockDiscovery([installation], [{ ...membership, role: 'member' }]) + await expect(verifyGitHubInstallationBinding('ghu_user', '21')).rejects.toThrow( + 'Only the GitHub account owner' + ) + }) + + it('rejects pending owners and provider identity mismatches', async () => { + mockDiscovery([installation], [{ ...membership, state: 'pending' }]) + expect(await listUserAdminGitHubInstallations('ghu_user')).toEqual([]) + mockDiscovery([installation], [{ ...membership, user: { id: 99 } }]) + await expect(listUserAdminGitHubInstallations('ghu_user')).rejects.toThrow( + 'identity does not match' + ) + }) + + it('revalidates the chosen installation using a short-lived signed app JWT', async () => { + mockDiscovery() + expect(await verifyGitHubInstallationBinding('ghu_user', '21')).toEqual(binding) + const appRequest = fetchMock.mock.calls.find(([url]) => + String(url).endsWith('/app/installations/21') + ) + const token = new Headers(appRequest?.[1]?.headers).get('Authorization')?.slice(7) ?? '' + const [header, payload, signature] = token.split('.') + expect( + verify( + 'RSA-SHA256', + Buffer.from(`${header}.${payload}`), + publicKey, + Buffer.from(signature, 'base64url') + ) + ).toBe(true) + expect(JSON.parse(Buffer.from(payload, 'base64url').toString())).toEqual({ + iat: now / 1000 - 60, + exp: now / 1000 + 540, + iss: 'Iv123', + }) + expect(appRequest?.[1]?.redirect).toBe('error') + }) + + it('accepts GitHub responses omitting optional client_id, using verified app_id', async () => { + const { client_id: _clientId, ...withoutClientId } = installation + mockDiscovery([withoutClientId]) + expect((await listUserAdminGitHubInstallations('ghu_user'))[0].appClientId).toBe('Iv123') + }) + + it('fails closed when a listing reaches the explicit page cap', async () => { + fetchMock.mockImplementation(async (input) => { + const path = new URL(String(input)).pathname + return path === '/user' ? json(user) : json(Array.from({ length: 100 }, () => membership)) + }) + await expect(listUserAdminGitHubInstallations('ghu_user')).rejects.toThrow('listing exceeds') + expect(fetchMock).toHaveBeenCalledTimes(11) + }) + + it('rejects unsafe identifiers, unknown binding fields, and non-user tokens before network access', async () => { + expect(() => parseGitHubInstallationBinding({ ...binding, privateKey: 'untrusted' })).toThrow( + 'invalid' + ) + await expect(verifyGitHubInstallationBinding('ghu_user', '../21')).rejects.toThrow( + 'ID is invalid' + ) + await expect(listUserAdminGitHubInstallations('ghs_installation')).rejects.toThrow( + 'Connect your GitHub account' + ) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) + +describe('GitHub installation content tokens', () => { + it.each([ + { ...installation, suspended_at: '2026-01-01T00:00:00Z' }, + { ...installation, account: { ...installation.account, id: 99 } }, + { ...installation, app_id: 2 }, + { ...installation, client_id: 'wrong' }, + { ...installation, permissions: { metadata: 'read' } }, + ])('denies a suspended, moved, or incompatible installation', async (providerInstallation) => { + fetchMock.mockResolvedValue(json(providerInstallation)) + await expect(assertGitHubInstallationActive(binding)).rejects.toThrow( + 'unavailable or its account binding changed' + ) + }) + + it('refuses cached bindings after changing the configured app', async () => { + setEnv({ GITHUB_APP_ID: '2' }) + await expect(assertGitHubInstallationActive(binding)).rejects.toThrow( + 'different configured app' + ) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('mints a token narrowed to one immutable repository with only read permissions', async () => { + fetchMock.mockResolvedValueOnce(json(installation)).mockResolvedValueOnce(json(tokenResponse())) + expect(await resolveGitHubInstallationAccessToken(binding, { repositoryId: '101' })).toEqual({ + accessToken: 'ghs_contents', + }) + expect(JSON.parse(String(fetchMock.mock.calls[1][1]?.body))).toEqual({ + permissions: { contents: 'read', metadata: 'read' }, + repository_ids: [101], + }) + }) + + it('rechecks suspension before returning a cached token', async () => { + fetchMock + .mockResolvedValueOnce(json(installation)) + .mockResolvedValueOnce(json(tokenResponse())) + .mockResolvedValueOnce(json({ ...installation, suspended_at: '2026-01-01T00:00:00Z' })) + await resolveGitHubInstallationAccessToken(binding, { repositoryId: '101' }) + await expect( + resolveGitHubInstallationAccessToken(binding, { repositoryId: '101' }) + ).rejects.toThrow('unavailable') + expect(fetchMock).toHaveBeenCalledTimes(3) + }) + + it('refuses unscoped token resolution and widened provider token responses', async () => { + await expect(resolveGitHubInstallationAccessToken(binding, {})).rejects.toThrow( + 'require a source repository' + ) + expect(fetchMock).not.toHaveBeenCalled() + fetchMock + .mockResolvedValueOnce(json(installation)) + .mockResolvedValueOnce( + json({ ...tokenResponse(), permissions: { contents: 'write', metadata: 'read' } }) + ) + await expect( + resolveGitHubInstallationAccessToken(binding, { repositoryId: '101' }) + ).rejects.toThrow('invalid installation token scope') + }) + + it.each(['team/repo', ' https://github.com/team/repo.git/ '])( + 'resolves %s using repository-scoped metadata access and verifies its owner ID', + async (repository) => { + fetchMock + .mockResolvedValueOnce(json(installation)) + .mockResolvedValueOnce(json(tokenResponse(101, false))) + .mockResolvedValueOnce( + json({ id: 101, full_name: 'team/repo', owner: { id: 11 }, default_branch: 'main' }) + ) + expect(await resolveGitHubInstallationRepository(binding, repository)).toEqual({ + id: '101', + fullName: 'team/repo', + defaultBranch: 'main', + }) + expect(JSON.parse(String(fetchMock.mock.calls[1][1]?.body))).toEqual({ + permissions: { metadata: 'read' }, + repositories: ['repo'], + }) + expect(fetchMock.mock.calls[2][0]).toBe('https://api.github.com/repos/team/repo') + fetchMock + .mockResolvedValueOnce(json(installation)) + .mockResolvedValueOnce( + json({ id: 101, full_name: 'team/repo', owner: { id: 99 }, default_branch: 'main' }) + ) + await expect(resolveGitHubInstallationRepository(binding, 'team/repo')).rejects.toThrow( + 'another GitHub installation account' + ) + } + ) + + it.each([ + 'https://github.com@evil.example/team/repo', + 'https://github.com.evil.example/team/repo', + 'team/../repo', + 'team/repo?redirect=https://example.com', + ])('rejects unsafe repository %s before minting a token', async (repository) => { + await expect(resolveGitHubInstallationRepository(binding, repository)).rejects.toThrow( + 'owner/repo format' + ) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('rejects oversized provider payloads before parsing', async () => { + fetchMock.mockResolvedValue( + new Response('x', { headers: { 'content-length': String(3 * 1024 * 1024) } }) + ) + await expect(assertGitHubInstallationActive(binding)).rejects.toThrow() + }) +}) diff --git a/apps/sim/lib/oauth/github-installation.ts b/apps/sim/lib/oauth/github-installation.ts new file mode 100644 index 00000000000..1e3c7dfe720 --- /dev/null +++ b/apps/sim/lib/oauth/github-installation.ts @@ -0,0 +1,481 @@ +import { createHash, createPrivateKey, createSign } from 'node:crypto' +import { z } from 'zod' +import { env } from '@/lib/core/config/env' +import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import type { + GitHubInstallationBinding, + GitHubInstallationRepositoryScope, + GitHubInstallationSummary, +} from '@/lib/oauth/github-installation-types' +import { parseGitHubRepository } from '@/lib/oauth/github-repository' + +const API_URL = 'https://api.github.com' +const PAGE_SIZE = 100 +const MAX_PAGES = 10 +const MAX_RESPONSE_BYTES = 2 * 1024 * 1024 +const REQUEST_TIMEOUT_MS = 10_000 +const OPERATION_TIMEOUT_MS = 30_000 +const TOKEN_HEADROOM_MS = 5 * 60_000 +const MAX_CACHED_TOKENS = 128 + +const idSchema = z + .string() + .regex(/^[1-9]\d{0,15}$/) + .refine((id) => Number.isSafeInteger(Number(id))) +const apiIdSchema = z.number().int().positive().max(Number.MAX_SAFE_INTEGER) +const loginSchema = z.string().regex(/^[a-z\d](?:[a-z\d-]{0,37}[a-z\d])?$/i) +const permissionsSchema = z.object({ + contents: z.enum(['read', 'write']).optional(), + metadata: z.literal('read').optional(), +}) +const installationSchema = z.object({ + id: apiIdSchema, + app_id: apiIdSchema, + client_id: z.string().min(1).max(200).optional(), + account: z.object({ + id: apiIdSchema, + login: loginSchema, + type: z.enum(['User', 'Organization']), + }), + repository_selection: z.enum(['all', 'selected']), + permissions: permissionsSchema, + suspended_at: z.string().nullable(), +}) +const bindingSchema = z + .object({ + type: z.literal('github_app_installation'), + version: z.literal(1), + appId: idSchema, + appClientId: z.string().min(1).max(200), + installationId: idSchema, + accountId: idSchema, + accountType: z.enum(['User', 'Organization']), + accountLogin: loginSchema, + repositorySelection: z.enum(['all', 'selected']), + }) + .strict() +const repositorySchema = z.object({ + id: apiIdSchema, + full_name: z.string().min(1).max(200), + owner: z.object({ id: apiIdSchema }), + default_branch: z.string().min(1).max(1024), +}) + +export class GitHubInstallationError extends Error { + constructor( + message: string, + readonly status?: number + ) { + super(message) + this.name = 'GitHubInstallationError' + } +} + +function readConfiguration() { + const appId = env.GITHUB_APP_ID?.trim() + const clientId = env.GITHUB_APP_CLIENT_ID?.trim() + const clientSecretConfigured = Boolean(env.GITHUB_APP_CLIENT_SECRET?.trim()) + const privateKey = env.GITHUB_APP_PRIVATE_KEY?.replace(/\\n/g, '\n').trim() + const slug = env.GITHUB_APP_SLUG?.trim() + if ( + !appId || + !idSchema.safeParse(appId).success || + !clientId || + !clientSecretConfigured || + !privateKey || + !slug || + !/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(slug) + ) + return null + try { + const key = createPrivateKey(privateKey) + if (key.asymmetricKeyType !== 'rsa') return null + return { + appId, + clientId, + key, + slug, + keyRevision: createHash('sha256').update(privateKey).digest('hex'), + } + } catch { + return null + } +} + +/** Exposes readiness and the provider installation URL without returning signing material. */ +export function getGitHubInstallationConfiguration() { + const configuration = readConfiguration() + return { + configured: configuration !== null, + installUrl: configuration + ? `https://github.com/apps/${configuration.slug}/installations/new` + : null, + } +} + +function requireConfiguration() { + const configuration = readConfiguration() + if (!configuration) + throw new GitHubInstallationError('GitHub App installation setup is not configured') + return configuration +} + +function createAppJwt(configuration: NonNullable>) { + const now = Math.floor(Date.now() / 1000) + const header = Buffer.from(JSON.stringify({ alg: 'RS256', typ: 'JWT' })).toString('base64url') + const payload = Buffer.from( + JSON.stringify({ iat: now - 60, exp: now + 540, iss: configuration.clientId }) + ).toString('base64url') + const message = `${header}.${payload}` + return `${message}.${createSign('RSA-SHA256').update(message).sign(configuration.key).toString('base64url')}` +} + +interface RequestOptions { + signal?: AbortSignal +} + +function operationSignal(options: RequestOptions): AbortSignal { + const timeout = AbortSignal.timeout(OPERATION_TIMEOUT_MS) + return options.signal ? AbortSignal.any([options.signal, timeout]) : timeout +} + +async function request( + path: string, + token: string, + signal: AbortSignal, + body?: unknown +): Promise { + const requestSignal = AbortSignal.any([signal, AbortSignal.timeout(REQUEST_TIMEOUT_MS)]) + const response = await fetch(`${API_URL}${path}`, { + method: body === undefined ? 'GET' : 'POST', + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'Sim', + ...(body === undefined ? {} : { 'Content-Type': 'application/json' }), + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + redirect: 'error', + signal: requestSignal, + }) + if (!response.ok) { + await response.body?.cancel() + throw new GitHubInstallationError( + `GitHub installation request failed with HTTP ${response.status}`, + response.status + ) + } + return readResponseJsonWithLimit(response, { + maxBytes: MAX_RESPONSE_BYTES, + signal: requestSignal, + label: 'GitHub installation response', + }) +} + +function summary( + installation: z.output, + clientId: string +): GitHubInstallationSummary { + return { + appId: String(installation.app_id), + appClientId: clientId, + installationId: String(installation.id), + accountId: String(installation.account.id), + accountType: installation.account.type, + accountLogin: installation.account.login, + repositorySelection: installation.repository_selection, + } +} + +function installationIsReady( + installation: z.output, + configuration: { appId: string; clientId: string } +) { + return ( + String(installation.app_id) === configuration.appId && + (installation.client_id === undefined || installation.client_id === configuration.clientId) && + installation.suspended_at === null && + Boolean(installation.permissions.contents) && + installation.permissions.metadata === 'read' + ) +} + +/** Rejects malformed encrypted blobs without interpreting an installation as a human identity. */ +export function parseGitHubInstallationBinding(value: unknown): GitHubInstallationBinding { + const parsed = bindingSchema.safeParse(value) + if (!parsed.success) + throw new GitHubInstallationError('Stored GitHub installation binding is invalid') + return parsed.data +} + +async function adminAccountIds(userAccessToken: string, signal: AbortSignal) { + if (!userAccessToken.startsWith('ghu_')) + throw new GitHubInstallationError('Connect your GitHub account before choosing an installation') + const user = z + .object({ id: apiIdSchema, type: z.literal('User') }) + .parse(await request('/user', userAccessToken, signal)) + const organizations = new Set() + const membershipsSchema = z + .array( + z.object({ + state: z.enum(['active', 'pending']), + role: z.enum(['admin', 'member', 'billing_manager']), + organization: z.object({ id: apiIdSchema }), + user: z.object({ id: apiIdSchema }), + }) + ) + .max(PAGE_SIZE) + for (let page = 1; page <= MAX_PAGES; page++) { + const memberships = membershipsSchema.parse( + await request( + `/user/memberships/orgs?state=active&per_page=${PAGE_SIZE}&page=${page}`, + userAccessToken, + signal + ) + ) + for (const membership of memberships) { + if (membership.user.id !== user.id) + throw new GitHubInstallationError( + 'GitHub membership identity does not match the connected account' + ) + if (membership.state === 'active' && membership.role === 'admin') + organizations.add(String(membership.organization.id)) + } + if (memberships.length < PAGE_SIZE) return { userId: String(user.id), organizations } + } + throw new GitHubInstallationError( + 'GitHub organization membership listing exceeds the supported limit' + ) +} + +/** User-visible installations are filtered by actual account ownership, not mere repository access. */ +export async function listUserAdminGitHubInstallations( + userAccessToken: string, + options: RequestOptions = {} +): Promise { + const configuration = requireConfiguration() + const signal = operationSignal(options) + const accounts = await adminAccountIds(userAccessToken, signal) + const result: GitHubInstallationSummary[] = [] + const pageSchema = z.object({ + total_count: z + .number() + .int() + .min(0) + .max(PAGE_SIZE * MAX_PAGES), + installations: z.array(installationSchema).max(PAGE_SIZE), + }) + for (let page = 1; page <= MAX_PAGES; page++) { + const data = pageSchema.parse( + await request( + `/user/installations?per_page=${PAGE_SIZE}&page=${page}`, + userAccessToken, + signal + ) + ) + for (const installation of data.installations) { + const ownsAccount = + installation.account.type === 'User' + ? String(installation.account.id) === accounts.userId + : accounts.organizations.has(String(installation.account.id)) + if (ownsAccount && installationIsReady(installation, configuration)) + result.push(summary(installation, configuration.clientId)) + } + if (data.installations.length < PAGE_SIZE) return result + } + throw new GitHubInstallationError('GitHub installation listing exceeds the supported limit') +} + +async function readBoundInstallation( + binding: GitHubInstallationBinding, + path: string, + options: RequestOptions = {} +) { + const verified = parseGitHubInstallationBinding(binding) + const configuration = requireConfiguration() + if (verified.appId !== configuration.appId || verified.appClientId !== configuration.clientId) + throw new GitHubInstallationError('GitHub installation belongs to a different configured app') + const installation = installationSchema.parse( + await request(path, createAppJwt(configuration), operationSignal(options)) + ) + if ( + !installationIsReady(installation, configuration) || + String(installation.id) !== verified.installationId || + String(installation.account.id) !== verified.accountId || + installation.account.type !== verified.accountType + ) + throw new GitHubInstallationError( + 'GitHub installation is unavailable or its account binding changed' + ) + return summary(installation, configuration.clientId) +} + +/** Rechecks current provider state so cached content tokens never hide app suspension or rebinding. */ +export async function assertGitHubInstallationActive( + binding: GitHubInstallationBinding, + options: RequestOptions = {} +) { + return readBoundInstallation(binding, `/app/installations/${binding.installationId}`, options) +} + +/** Rechecks the repository's current installation even when its public content remains readable. */ +export async function assertGitHubInstallationRepositoryActive( + binding: GitHubInstallationBinding, + repository: string, + options: RequestOptions = {} +): Promise { + const { owner, repo } = parseGitHubRepository(repository) + await readBoundInstallation( + binding, + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/installation`, + options + ) +} + +/** Requires both the initiating GitHub user's account authority and the server's current app identity. */ +export async function verifyGitHubInstallationBinding( + userAccessToken: string, + installationId: string, + options: RequestOptions = {} +): Promise { + if (!idSchema.safeParse(installationId).success) + throw new GitHubInstallationError('GitHub installation ID is invalid') + const signal = operationSignal(options) + const installations = await listUserAdminGitHubInstallations(userAccessToken, { signal }) + const installation = installations.find( + (candidate) => candidate.installationId === installationId + ) + if (!installation) + throw new GitHubInstallationError( + 'Only the GitHub account owner or an organization owner can connect this installation' + ) + const binding: GitHubInstallationBinding = { + type: 'github_app_installation', + version: 1, + ...installation, + } + return { + type: 'github_app_installation', + version: 1, + ...(await assertGitHubInstallationActive(binding, { signal })), + } +} + +interface CachedToken { + accessToken: string + expiresAt: number +} +const tokenCache = new Map() + +async function mintToken( + binding: GitHubInstallationBinding, + signal: AbortSignal, + repositoryId?: string, + repositoryName?: string +) { + const configuration = requireConfiguration() + const key = [ + configuration.keyRevision, + binding.appClientId, + binding.installationId, + binding.accountId, + repositoryId ?? `metadata:${repositoryName}`, + ].join(':') + for (const [cachedKey, entry] of tokenCache) + if (entry.expiresAt <= Date.now() + TOKEN_HEADROOM_MS) tokenCache.delete(cachedKey) + const cached = tokenCache.get(key) + if (cached) return cached.accessToken + const permissions = repositoryId ? { contents: 'read', metadata: 'read' } : { metadata: 'read' } + const response = z + .object({ + token: z.string().min(1).max(1024), + expires_at: z.iso.datetime(), + permissions: permissionsSchema.strict(), + repositories: z + .array(z.object({ id: apiIdSchema })) + .max(1) + .optional(), + }) + .parse( + await request( + `/app/installations/${binding.installationId}/access_tokens`, + createAppJwt(configuration), + signal, + { + permissions, + ...(repositoryId + ? { repository_ids: [Number(repositoryId)] } + : { repositories: [repositoryName] }), + } + ) + ) + const expiresAt = Date.parse(response.expires_at) + if ( + expiresAt <= Date.now() + TOKEN_HEADROOM_MS || + expiresAt > Date.now() + 65 * 60_000 || + response.permissions.metadata !== 'read' || + response.permissions.contents !== (repositoryId ? 'read' : undefined) || + !response.repositories || + response.repositories.length !== 1 || + (repositoryId && String(response.repositories[0].id) !== repositoryId) + ) { + throw new GitHubInstallationError( + 'GitHub returned an invalid installation token scope or expiration' + ) + } + if (tokenCache.size >= MAX_CACHED_TOKENS) { + const oldest = tokenCache.keys().next().value + if (oldest !== undefined) tokenCache.delete(oldest) + } + tokenCache.set(key, { accessToken: response.token, expiresAt }) + return response.token +} + +/** Resolves a mutable repository name to its immutable identity inside the bound GitHub account. */ +export async function resolveGitHubInstallationRepository( + binding: GitHubInstallationBinding, + repository: string, + options: RequestOptions = {} +) { + let parsed: ReturnType + try { + parsed = parseGitHubRepository(repository) + } catch { + throw new GitHubInstallationError('Use a GitHub repository in owner/repo format') + } + const { owner, repo } = parsed + const signal = operationSignal(options) + await assertGitHubInstallationActive(binding, { signal }) + const token = await mintToken(binding, signal, undefined, repo) + const resolved = repositorySchema.parse( + await request(`/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`, token, signal) + ) + if (String(resolved.owner.id) !== binding.accountId) + throw new GitHubInstallationError('Repository belongs to another GitHub installation account') + return { + id: String(resolved.id), + fullName: resolved.full_name, + defaultBranch: resolved.default_branch, + } +} + +/** Mints contents access for exactly one source repository; generic unscoped token reads are refused. */ +export async function resolveGitHubInstallationAccessToken( + binding: GitHubInstallationBinding, + scope: GitHubInstallationRepositoryScope, + options: RequestOptions = {} +) { + const signal = operationSignal(options) + let repositoryId = scope.repositoryId + if (repositoryId && !idSchema.safeParse(repositoryId).success) + throw new GitHubInstallationError('GitHub repository ID is invalid') + if (!repositoryId && scope.repository) + repositoryId = ( + await resolveGitHubInstallationRepository(binding, scope.repository, { signal }) + ).id + if (!repositoryId) + throw new GitHubInstallationError('GitHub installation tokens require a source repository') + await assertGitHubInstallationActive(binding, { signal }) + return { accessToken: await mintToken(binding, signal, repositoryId) } +} diff --git a/apps/sim/lib/oauth/github-repository.test.ts b/apps/sim/lib/oauth/github-repository.test.ts new file mode 100644 index 00000000000..2dfb2a25d01 --- /dev/null +++ b/apps/sim/lib/oauth/github-repository.test.ts @@ -0,0 +1,42 @@ +/** @vitest-environment node */ +import { describe, expect, it } from 'vitest' +import { parseGitHubRepository } from '@/lib/oauth/github-repository' + +describe('parseGitHubRepository', () => { + it.each([ + 'owner/repo', + ' owner/repo ', + 'owner/repo.git', + 'owner/repo.git/', + 'https://github.com/owner/repo', + 'https://github.com/owner/repo.git/', + 'HTTP://GITHUB.COM/owner/repo/', + ])('normalizes %s to the same repository', (repository) => { + expect(parseGitHubRepository(repository)).toEqual({ owner: 'owner', repo: 'repo' }) + }) + + it.each([ + '', + 'owner', + 'owner/repo/extra', + 'owner/.', + 'owner/..', + 'owner/../repo', + 'owner/%2e%2e', + 'owner/repo%2fextra', + 'owner/repo?redirect=https://example.com', + 'owner/repo#fragment', + 'owner\\repo', + '//github.com/owner/repo', + 'https://github.com.evil.example/owner/repo', + 'https://github.com@evil.example/owner/repo', + 'https://evil.example@github.com/owner/repo', + 'https://github.com:443/owner/repo', + 'https://127.0.0.1/owner/repo', + 'https://github.com/owner/repo/../../app', + 'https://github.com/owner/repo.git//', + 'git@github.com:owner/repo.git', + ])('rejects invalid or unsafe repository reference %s', (repository) => { + expect(() => parseGitHubRepository(repository)).toThrow('Invalid repository format') + }) +}) diff --git a/apps/sim/lib/oauth/github-repository.ts b/apps/sim/lib/oauth/github-repository.ts new file mode 100644 index 00000000000..087f5858470 --- /dev/null +++ b/apps/sim/lib/oauth/github-repository.ts @@ -0,0 +1,19 @@ +/** Accepts GitHub repository names and web URLs without allowing provider-path traversal. */ +export function parseGitHubRepository(repository: string): { owner: string; repo: string } { + const cleaned = repository + .trim() + .replace(/^https?:\/\/github\.com\//i, '') + .replace(/\/$/, '') + .replace(/\.git$/, '') + const parts = cleaned.split('/') + if ( + parts.length !== 2 || + !/^[a-z\d](?:[a-z\d-]*[a-z\d])?$/i.test(parts[0] ?? '') || + !/^[a-z\d_.-]+$/i.test(parts[1] ?? '') || + parts[1] === '.' || + parts[1] === '..' + ) { + throw new Error(`Invalid repository format: "${repository}". Use "owner/repo".`) + } + return { owner: parts[0], repo: parts[1] } +} diff --git a/apps/sim/lib/oauth/oauth.ts b/apps/sim/lib/oauth/oauth.ts index 5e1dda3ce1e..1d6891b9d6c 100644 --- a/apps/sim/lib/oauth/oauth.ts +++ b/apps/sim/lib/oauth/oauth.ts @@ -80,6 +80,7 @@ import { readResponseTextWithLimit, } from '@/lib/core/utils/stream-limits' import { getDocusignOAuthUrl } from '@/lib/oauth/docusign' +import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' import { GITHUB_TOKEN_URL, parseGitHubRepositoriesTokenResponse, @@ -119,6 +120,7 @@ export const OAUTH_PROVIDERS: Record = { name: 'GitHub', description: 'Search repository files through your GitHub App access.', providerId: 'github-repositories', + serviceAccountProviderId: GITHUB_INSTALLATION_PROVIDER_ID, icon: GithubIcon, baseProviderIcon: GithubIcon, scopes: [], diff --git a/scripts/check-tool-registry-boundary.baseline.json b/scripts/check-tool-registry-boundary.baseline.json index c5367f11976..bf2e2980b4d 100644 --- a/scripts/check-tool-registry-boundary.baseline.json +++ b/scripts/check-tool-registry-boundary.baseline.json @@ -402,16 +402,16 @@ "gateways": {} }, "app/workspace/[workspaceId]/settings/[section]/page.tsx": { - "modules": 2242, + "modules": 2291, "gateways": { - "apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx": 676, + "apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx": 707, "apps/sim/triggers/registry.ts": 485, + "apps/sim/lib/auth/index.ts": 365, "apps/sim/blocks/registry.ts": 354, - "apps/sim/lib/auth/index.ts": 351, - "apps/sim/lib/webhooks/providers/index.ts": 117, - "apps/sim/lib/webhooks/providers/registry.ts": 115, - "apps/sim/ee/access-control/components/access-control.tsx": 74, - "apps/sim/ee/access-control/components/group-detail.tsx": 72 + "apps/sim/lib/webhooks/providers/index.ts": 118, + "apps/sim/lib/webhooks/providers/registry.ts": 116, + "apps/sim/ee/access-control/components/access-control.tsx": 75, + "apps/sim/ee/access-control/components/group-detail.tsx": 73 } }, "app/workspace/[workspaceId]/settings/billing/credit-usage/layout.tsx": { From 10f5c42fb3ee50b9a7637dee66b9e4ed6568c30a Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Wed, 9 Sep 2026 16:31:52 -0700 Subject: [PATCH 2/3] fix(search): scope GitHub access checks to requested results --- .../integrations/credential-display.test.ts | 1 + .../github-member.integration.ts | 190 +++++++++++- .../access/github-installation.test.ts | 72 ++++- .../knowledge/access/github-installation.ts | 200 ++++++------ apps/sim/lib/knowledge/access/predicate.ts | 23 +- apps/sim/lib/knowledge/access/scope.test.ts | 6 +- apps/sim/lib/knowledge/access/scope.ts | 110 +++++-- apps/sim/lib/knowledge/access/types.ts | 11 + .../knowledge/application/contexts.test.ts | 24 ++ .../sim/lib/knowledge/application/contexts.ts | 64 +++- .../knowledge/application/documents.test.ts | 6 +- .../lib/knowledge/application/documents.ts | 53 ++-- .../knowledge/application/operations.test.ts | 2 + .../application/read-indexed-document.ts | 34 +- .../application/search-source-overview.ts | 123 ++++---- .../application/search-source-progress.ts | 4 +- .../application/search-sources.test.ts | 7 +- .../knowledge/application/search-sources.ts | 6 +- .../lib/knowledge/application/search.test.ts | 7 +- apps/sim/lib/knowledge/application/search.ts | 6 +- .../application/slack-search/source-status.ts | 55 ++-- apps/sim/lib/knowledge/application/tags.ts | 4 +- .../lib/knowledge/documents/service.test.ts | 54 ++++ apps/sim/lib/knowledge/documents/service.ts | 265 ++++++++++------ apps/sim/lib/knowledge/read-access.test.ts | 75 +++++ apps/sim/lib/knowledge/read-access.ts | 65 ++++ apps/sim/lib/knowledge/search/queries.test.ts | 157 +++++++++- apps/sim/lib/knowledge/search/queries.ts | 291 +++++++++++++++++- apps/sim/lib/knowledge/tags/service.ts | 95 +++--- 29 files changed, 1580 insertions(+), 430 deletions(-) create mode 100644 apps/sim/lib/knowledge/documents/service.test.ts create mode 100644 apps/sim/lib/knowledge/read-access.test.ts create mode 100644 apps/sim/lib/knowledge/read-access.ts diff --git a/apps/sim/lib/integrations/credential-display.test.ts b/apps/sim/lib/integrations/credential-display.test.ts index 470b293640a..456332bb029 100644 --- a/apps/sim/lib/integrations/credential-display.test.ts +++ b/apps/sim/lib/integrations/credential-display.test.ts @@ -49,6 +49,7 @@ const EXPECTED_COVERAGE: Record = { 'calcom-service-account': ['cal-com'], 'claude-platform-service-account': [], 'clickup-service-account': ['clickup'], + 'github-app-installation': ['github'], 'google-service-account': [ 'gmail', 'google-bigquery', diff --git a/apps/sim/lib/knowledge/__integration__/github-member.integration.ts b/apps/sim/lib/knowledge/__integration__/github-member.integration.ts index f75c1919953..31cd2ddc32e 100644 --- a/apps/sim/lib/knowledge/__integration__/github-member.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/github-member.integration.ts @@ -72,11 +72,15 @@ import { seedKnowledgeAclFixture, seedKnowledgeMemberFixture, } from '@/lib/knowledge/__integration__/seed-source-access-fixture' +import { GITHUB_READ_SOURCE_TIMEOUT_MS } from '@/lib/knowledge/access/github-installation' +import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' import { subjectToken } from '@/lib/knowledge/access/tokens' import { KnowledgeDocumentNotReadyError } from '@/lib/knowledge/application/chunk-errors' import { listKnowledgeChunks } from '@/lib/knowledge/application/chunks' import { readKnowledgeDocument } from '@/lib/knowledge/application/documents' +import { readIndexedKnowledgeDocument } from '@/lib/knowledge/application/read-indexed-document' import { searchKnowledge } from '@/lib/knowledge/application/search' +import { readSearchSourceOverview } from '@/lib/knowledge/application/search-source-overview' import { listSearchSources } from '@/lib/knowledge/application/search-sources' import { grantKnowledgeConnectorCredentialAccess } from '@/lib/knowledge/connectors/member-access' import { executeMemberSync } from '@/lib/knowledge/connectors/member-sync-engine' @@ -84,8 +88,11 @@ import { MEMBER_SUSPENDED_PURGE_DAYS, MEMBER_TOMBSTONE_PURGE_DAYS, } from '@/lib/knowledge/connectors/sync-limits' +import { getDocuments } from '@/lib/knowledge/documents/service' +import { getTagUsageStats } from '@/lib/knowledge/tags/service' import { deleteFile } from '@/lib/uploads/core/storage-service' import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server' +import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' const redisUrl = process.env.KNOWLEDGE_ACL_TEST_REDIS_URL if (redisUrl) { @@ -105,6 +112,7 @@ interface RepositoryFixture { id: number public: boolean installed: boolean + stallRef: boolean readers: Set defaultBranch: string files: Map @@ -133,6 +141,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { const { privateKey, publicKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }) let organizationSource = false let installationSuspended = false + let referenceObserved: ((repository: string) => void) | undefined const installation = () => ({ id: 42, app_id: 1, @@ -164,6 +173,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { id: 9001 + repositories.size, public: false, installed: true, + stallRef: false, readers: new Set(readers), defaultBranch: 'trunk', files: new Map([ @@ -240,30 +250,33 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { if (url.pathname === '/app/installations/42/access_tokens') { expect(request.method).toBe('POST') const body = await request.json() - expect(body).toEqual({ + expect(body).toMatchObject({ permissions: { contents: 'read', metadata: 'read' }, - repository_ids: [9001], }) + expect(body.repository_ids).toHaveLength(1) + const repositoryId = body.repository_ids[0] + expect( + [...repositories.values()].some( + (repository) => repository.id === repositoryId && repository.installed + ) + ).toBe(true) return Response.json({ - token: 'ghs_fixture_installation', + token: `ghs_fixture_installation_${repositoryId}`, expires_at: new Date(Date.now() + 60 * 60_000).toISOString(), permissions: body.permissions, - repositories: [{ id: 9001 }], + repositories: [{ id: repositoryId }], }) } expect(request.method).toBe('GET') - if ( - url.pathname === '/repos/fixture/shared/installation' && - !repositories.get('shared')?.installed - ) + const repositoryInstallation = url.pathname.match(/^\/repos\/fixture\/([^/]+)\/installation$/) + if (repositoryInstallation && !repositories.get(repositoryInstallation[1])?.installed) return Response.json({ message: 'Not Found' }, { status: 404 }) - expect(['/app/installations/42', '/repos/fixture/shared/installation']).toContain( - url.pathname - ) + expect(url.pathname === '/app/installations/42' || Boolean(repositoryInstallation)).toBe(true) return Response.json(installation()) } if (request.method !== 'GET') throw new Error(`Unexpected GitHub method: ${request.method}`) - const installationToken = bearer === 'ghs_fixture_installation' + const installationRepository = bearer.match(/^ghs_fixture_installation_(\d+)$/)?.[1] + const installationToken = Boolean(installationRepository) const member = enrolled.members.find((candidate) => [tokenFor(candidate.userId), `${tokenFor(candidate.userId)}_refreshed`].some( (token) => request.headers.get('authorization') === `Bearer ${token}` @@ -295,6 +308,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { if (!match) throw new Error(`Unexpected GitHub endpoint: ${url.pathname}`) const source = repositories.get(match[1]) if (!source) throw new Error('Unexpected GitHub repository') + if (installationToken) expect(installationRepository).toBe(String(source.id)) if (source.throttledReaders.has(actingId)) return Response.json( { message: 'You have exceeded a secondary rate limit.' }, @@ -317,6 +331,13 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { default_branch: source.defaultBranch, }) if (match[2].startsWith('/git/ref/heads/')) { + referenceObserved?.(match[1]) + if (source.stallRef) + return new Promise((_resolve, reject) => { + request.signal.addEventListener('abort', () => reject(request.signal.reason), { + once: true, + }) + }) const ref = decodeURIComponent(match[2].slice('/git/ref/heads/'.length)) return ref === source.defaultBranch ? Response.json({ ref: `refs/heads/${ref}`, object: { type: 'commit', sha: shaFor(ref) } }) @@ -399,6 +420,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { refreshedUsers.clear() organizationSource = false installationSuspended = false + referenceObserved = undefined oauthStateKey = undefined oauthVerification = undefined Object.assign(env, { @@ -585,7 +607,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { .orderBy(document.externalId) } - async function search(principal: Principal) { + async function search(principal: Principal, searchMode: 'hybrid' | 'vector' = 'hybrid') { const result = await searchKnowledge.execute({ principal, input: { @@ -594,7 +616,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { : { workspaceId: ids.workspaceId }), knowledgeBaseIds: [ids.knowledgeBaseId], query: 'Orion', - searchMode: 'hybrid', + searchMode, topK: 20, }, }) @@ -697,11 +719,79 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { actorUserId: ids.aliceId, organizationId: ids.organizationId, }) + const unrelatedSources = Array.from({ length: 105 }, () => generateId()) + await db.insert(knowledgeConnector).values( + unrelatedSources.map((id) => ({ + id, + knowledgeBaseId: ids.knowledgeBaseId, + connectorType: 'github', + accessMode: 'members', + credentialId: installationCredentialId, + credentialGroupId: enrolled.groupId, + credentialGroupOptionId: enrolled.optionId, + sourceConfig: { repository: 'fixture/shared', githubRepositoryId: '9001' }, + })) + ) + await db.insert(knowledgeConnectorMember).values( + unrelatedSources.map((connectorId) => ({ + id: generateId(), + organizationId: ids.organizationId, + connectorId, + credentialId: enrolled.members[0].credentialId, + subjectToken: enrolled.members[0].subjectToken, + })) + ) const result = await sync() expect(result.error).toBeUndefined() expect(result.docsHydratedOnce).toBe(1) const [indexed] = await rows() expect(indexed).toBeDefined() + const provider = (userId: string) => + createKnowledgeAccessProvider(actor(userId), { + organizationId: ids.organizationId, + knowledgeBaseIds: [ids.knowledgeBaseId], + }) + const page = (userId: string, offset = 0) => + getDocuments( + ids.knowledgeBaseId, + { limit: 1, offset, sortBy: 'filename', sortOrder: 'asc' }, + 'github-candidate-regression', + provider(userId) + ) + expect(await page(ids.aliceId)).toMatchObject({ + documents: [{ id: indexed.id }], + pagination: { total: 1 }, + }) + expect( + ( + await readSearchSourceOverview.execute({ + principal: actor(ids.aliceId), + input: { organizationId: ids.organizationId }, + }) + ).hasSearchableDocuments + ).toBe(true) + await db.update(document).set({ tag1: 'fixture' }).where(eq(document.id, indexed.id)) + await db.update(embedding).set({ tag1: 'fixture' }).where(eq(embedding.documentId, indexed.id)) + expect( + await getTagUsageStats(ids.knowledgeBaseId, provider(ids.aliceId), 'github-tag-regression') + ).toEqual( + expect.arrayContaining([ + expect.objectContaining({ tagSlot: 'tag1', documentCount: 1, chunkCount: 1 }), + ]) + ) + expect( + ( + await readIndexedKnowledgeDocument.execute({ + principal: actor(ids.aliceId), + input: { + organizationId: ids.organizationId, + target: { kind: 'url', url: indexed.sourceUrl! }, + limit: 1, + resultSecretRegistry: new ResolvedSecretTraceRegistry(), + }, + }) + ).documentId + ).toBe(indexed.id) expect( requests.filter((entry) => entry.path.includes('/git/blobs/')).map((entry) => entry.userId) ).toEqual(['installation']) @@ -710,6 +800,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { await assertAccess(actor(ids.bobId), indexed, true) const source = repositories.get('shared')! source.readers.delete(ids.bobId) + expect(await page(ids.bobId)).toMatchObject({ documents: [], pagination: { total: 0 } }) expect(await search(actor(ids.bobId))).toEqual([]) await assertAccess(actor(ids.bobId), indexed, false) expect(await search(actor(ids.aliceId))).toEqual([indexed.id]) @@ -729,6 +820,77 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { expect(await search(actor(ids.aliceId))).toEqual([]) installationSuspended = false expect(await search(actor(ids.aliceId))).toEqual([indexed.id]) + const slowRepository = repository('slow') + const slowSourceId = generateId() + await db.insert(knowledgeConnector).values({ + id: slowSourceId, + knowledgeBaseId: ids.knowledgeBaseId, + connectorType: 'github', + accessMode: 'members', + credentialId: installationCredentialId, + credentialGroupId: enrolled.groupId, + credentialGroupOptionId: enrolled.optionId, + sourceConfig: { repository: 'fixture/slow', githubRepositoryId: String(slowRepository.id) }, + }) + expect((await sync(slowSourceId)).error).toBeUndefined() + const [slowDocument] = await rows(slowSourceId) + expect(slowDocument).toBeDefined() + const deniedRepository = repository('denied-paging', [ids.aliceId]) + const deniedSourceId = generateId() + await db.insert(knowledgeConnector).values({ + id: deniedSourceId, + knowledgeBaseId: ids.knowledgeBaseId, + connectorType: 'github', + accessMode: 'members', + credentialId: installationCredentialId, + credentialGroupId: enrolled.groupId, + credentialGroupOptionId: enrolled.optionId, + sourceConfig: { + repository: 'fixture/denied-paging', + githubRepositoryId: String(deniedRepository.id), + }, + }) + expect((await sync(deniedSourceId)).error).toBeUndefined() + const [deniedDocument] = await rows(deniedSourceId) + deniedRepository.readers.delete(ids.aliceId) + for (const [id, filename] of [ + [indexed.id, 'alpha'], + [deniedDocument.id, 'beta'], + [slowDocument.id, 'gamma'], + ]) + await db.update(document).set({ filename }).where(eq(document.id, id)) + expect(await page(ids.aliceId, 1)).toMatchObject({ + documents: [{ id: slowDocument.id }], + pagination: { total: 2, offset: 1 }, + }) + slowRepository.stallRef = true + const sourceTimers: AbortController[] = [] + const nativeTimeout = AbortSignal.timeout.bind(AbortSignal) + const timerSpy = vi.spyOn(AbortSignal, 'timeout').mockImplementation((duration) => { + if (duration !== GITHUB_READ_SOURCE_TIMEOUT_MS) return nativeTimeout(duration) + const controller = new AbortController() + sourceTimers.push(controller) + return controller.signal + }) + try { + const observed = new Set() + const candidatesStarted = new Promise((resolve) => { + referenceObserved = (name) => { + observed.add(name) + if (observed.has('shared') && observed.has('slow')) resolve() + } + }) + const pending = search(actor(ids.aliceId), 'vector') + await candidatesStarted + /** Complete the fast response's microtasks before expiring the stalled candidate. */ + for (let turn = 0; turn < 20; turn++) await Promise.resolve() + for (const timer of sourceTimers) timer.abort(new Error('fixture source timeout')) + expect(await pending).toEqual([indexed.id]) + } finally { + timerSpy.mockRestore() + referenceObserved = undefined + slowRepository.stallRef = false + } await db .delete(member) .where(and(eq(member.organizationId, ids.organizationId), eq(member.userId, ids.bobId))) diff --git a/apps/sim/lib/knowledge/access/github-installation.test.ts b/apps/sim/lib/knowledge/access/github-installation.test.ts index 22d5eb560ff..1e14159bfc8 100644 --- a/apps/sim/lib/knowledge/access/github-installation.test.ts +++ b/apps/sim/lib/knowledge/access/github-installation.test.ts @@ -1,12 +1,14 @@ /** @vitest-environment node */ import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { GITHUB_READ_CONCURRENCY, GITHUB_READ_RESPONSE_MAX_BYTES, - GITHUB_READ_SOURCE_LIMIT, + GITHUB_READ_SOURCE_TIMEOUT_MS, + GITHUB_READ_TIMEOUT_MS, resolveGitHubInstallationReadGrants, } from '@/lib/knowledge/access/github-installation' +import { MAX_KNOWLEDGE_ACCESS_CANDIDATES } from '@/lib/knowledge/access/types' const mocks = vi.hoisted(() => ({ token: vi.fn(), @@ -27,6 +29,7 @@ const input = { scope: { kind: 'organization' as const, organizationId: 'org-1' }, readers: [{ credentialId: 'alice-credential', subjectToken: 's:github-repositories:-:alice' }], knowledgeBaseIds: ['index-1'], + connectorIds: ['source-1'], } const source = { connectorId: 'source-1', @@ -55,6 +58,7 @@ const metadata = { id: 123, owner: { id: 90 }, default_branch: 'main' } const reference = { ref: 'refs/heads/main', object: { type: 'commit', sha: 'a'.repeat(40) } } function queueSources(rows: (typeof source)[] = [source]) { + input.connectorIds = rows.map((row) => row.connectorId) queueTableRows(schemaMock.knowledgeConnector, rows) queueTableRows(schemaMock.credential, [contentCredential]) } @@ -71,6 +75,7 @@ beforeEach(() => { Response.json(url.includes('/git/ref/') ? reference : metadata) ) }) +afterEach(() => vi.restoreAllMocks()) describe('live GitHub installation reader access', () => { it('requires both current installation and personal Contents access for the immutable repository', async () => { @@ -194,18 +199,71 @@ describe('live GitHub installation reader access', () => { expect(mocks.fetch).toHaveBeenCalledTimes(2) }) - it('bounds sources before any provider request', async () => { + it('authorizes a candidate batch beyond the old 100-source cliff', async () => { queueSources( - Array.from({ length: GITHUB_READ_SOURCE_LIMIT + 1 }, (_, index) => ({ + Array.from({ length: 101 }, (_, index) => ({ ...source, connectorId: `source-${index}`, })) ) - await expect(resolveGitHubInstallationReadGrants(input)).resolves.toEqual([]) - expect(dbChainMockFns.limit).toHaveBeenCalledWith(GITHUB_READ_SOURCE_LIMIT + 1) + await expect(resolveGitHubInstallationReadGrants(input)).resolves.toHaveLength(101) + expect(dbChainMockFns.limit).toHaveBeenCalledWith(MAX_KNOWLEDGE_ACCESS_CANDIDATES) + expect(mocks.fetch).toHaveBeenCalledTimes(2) + }) + + it('bounds one candidate batch without enumerating all organization sources', async () => { + await expect( + resolveGitHubInstallationReadGrants({ + ...input, + connectorIds: Array.from( + { length: MAX_KNOWLEDGE_ACCESS_CANDIDATES + 1 }, + (_, index) => `source-${index}` + ), + }) + ).rejects.toThrow('bounded pages') expect(mocks.installation).not.toHaveBeenCalled() }) + it.each(['source', 'admission'])( + 'retains completed proofs and advances other workers while a %s deadline expires', + async (deadline) => { + const overall = new AbortController() + const sourceTimers: AbortController[] = [] + vi.spyOn(AbortSignal, 'timeout').mockImplementation((duration) => { + if (duration === GITHUB_READ_TIMEOUT_MS) return overall.signal + expect(duration).toBe(GITHUB_READ_SOURCE_TIMEOUT_MS) + const timer = new AbortController() + sourceTimers.push(timer) + return timer.signal + }) + queueSources( + Array.from({ length: 6 }, (_, index) => ({ + ...source, + connectorId: `source-${index}`, + repository: `company/repo-${index}`, + })) + ) + let lastFastCheck: (() => void) | undefined + const allFastChecks = new Promise((resolve) => { + lastFastCheck = resolve + }) + mocks.fetch.mockImplementation(async (url: string) => { + if (url.includes('/repo-0')) return new Promise(() => {}) + if (url.includes('/repo-5/git/ref/')) lastFastCheck?.() + return Response.json(url.includes('/git/ref/') ? reference : metadata) + }) + const pending = resolveGitHubInstallationReadGrants(input) + await allFastChecks + /** Let the response proof finish before expiring the unrelated stalled request. */ + for (let turn = 0; turn < 20; turn++) await Promise.resolve() + ;(deadline === 'source' ? sourceTimers[0] : overall).abort(new Error('deadline')) + const grants = await pending + expect(grants).toHaveLength(5) + expect(grants.map((entry) => entry.connectorId)).not.toContain('source-0') + expect(grants.map((entry) => entry.connectorId)).toContain('source-5') + } + ) + it('bounds concurrent source checks and never buffers unbounded response bytes', async () => { queueSources( Array.from({ length: GITHUB_READ_CONCURRENCY + 2 }, (_, index) => ({ @@ -241,7 +299,7 @@ describe('live GitHub installation reader access', () => { }) await expect( resolveGitHubInstallationReadGrants({ ...input, signal: controller.signal }) - ).resolves.toEqual([]) + ).rejects.toThrow('cancelled') expect(mocks.fetch).not.toHaveBeenCalled() }) }) diff --git a/apps/sim/lib/knowledge/access/github-installation.ts b/apps/sim/lib/knowledge/access/github-installation.ts index ecd411bc912..0e70615154c 100644 --- a/apps/sim/lib/knowledge/access/github-installation.ts +++ b/apps/sim/lib/knowledge/access/github-installation.ts @@ -1,6 +1,7 @@ import { db } from '@sim/db' import { credential, + credentialGroupEnrollment, knowledgeBase, knowledgeConnector, knowledgeConnectorMember, @@ -13,7 +14,10 @@ import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { decryptSecret } from '@/lib/core/security/encryption' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { resolveManagedOAuthToken } from '@/lib/credentials/managed-oauth' -import type { GitHubInstallationReadGrant } from '@/lib/knowledge/access/types' +import { + type GitHubInstallationReadGrant, + MAX_KNOWLEDGE_ACCESS_CANDIDATES, +} from '@/lib/knowledge/access/types' import { assertGitHubInstallationActive, assertGitHubInstallationRepositoryActive, @@ -25,9 +29,9 @@ import { } from '@/lib/oauth/github-installation-types' const logger = createLogger('GitHubInstallationReadAccess') -export const GITHUB_READ_SOURCE_LIMIT = 100 export const GITHUB_READ_CONCURRENCY = 4 export const GITHUB_READ_TIMEOUT_MS = 8000 +export const GITHUB_READ_SOURCE_TIMEOUT_MS = 4000 export const GITHUB_READ_RESPONSE_MAX_BYTES = 64 * 1024 const INSTALLATION_BINDING_MAX_BYTES = 16 * 1024 @@ -135,16 +139,14 @@ async function verifyRepository( export async function resolveGitHubInstallationReadGrants(input: { scope: ResourceScope readers: readonly GitHubReaderCredential[] + connectorIds: readonly string[] knowledgeBaseIds?: readonly string[] signal?: AbortSignal }): Promise { - if ( - !input.readers.length || - input.readers.length > GITHUB_READ_SOURCE_LIMIT || - (input.knowledgeBaseIds && - (input.knowledgeBaseIds.length === 0 || - input.knowledgeBaseIds.length > GITHUB_READ_SOURCE_LIMIT)) - ) + input.signal?.throwIfAborted() + if (input.connectorIds.length > MAX_KNOWLEDGE_ACCESS_CANDIDATES) + throw new Error('Knowledge access candidates must be authorized in bounded pages') + if (!input.readers.length || !input.connectorIds.length || input.knowledgeBaseIds?.length === 0) return [] const readers = new Map(input.readers.map((reader) => [reader.credentialId, reader.subjectToken])) const sources: GitHubReadSource[] = await db @@ -165,9 +167,24 @@ export async function resolveGitHubInstallationReadGrants(input: { knowledgeConnectorMember, eq(knowledgeConnectorMember.connectorId, knowledgeConnector.id) ) + .innerJoin( + credential, + and( + eq(credential.id, knowledgeConnectorMember.credentialId), + eq(credential.credentialGroupOptionId, knowledgeConnector.credentialGroupOptionId) + ) + ) + .innerJoin( + credentialGroupEnrollment, + and( + eq(credentialGroupEnrollment.id, credential.credentialGroupEnrollmentId), + eq(credentialGroupEnrollment.credentialGroupId, knowledgeConnector.credentialGroupId) + ) + ) .where( and( resourceScopeCondition(knowledgeBase, input.scope), + inArray(knowledgeConnector.id, [...new Set(input.connectorIds)]), input.knowledgeBaseIds ? inArray(knowledgeBase.id, [...input.knowledgeBaseIds]) : undefined, isNull(knowledgeBase.deletedAt), eq(knowledgeConnector.connectorType, 'github'), @@ -180,8 +197,8 @@ export async function resolveGitHubInstallationReadGrants(input: { ) ) .orderBy(asc(knowledgeConnector.id), asc(knowledgeConnectorMember.id)) - .limit(GITHUB_READ_SOURCE_LIMIT + 1) - if (sources.length > GITHUB_READ_SOURCE_LIMIT || !sources.length) return [] + .limit(MAX_KNOWLEDGE_ACCESS_CANDIDATES) + if (!sources.length) return [] const contentCredentialIds = [ ...new Set( sources.flatMap((source) => (source.contentCredentialId ? [source.contentCredentialId] : [])) @@ -206,92 +223,95 @@ export async function resolveGitHubInstallationReadGrants(input: { sql`octet_length(${credential.encryptedServiceAccountKey}) <= ${INSTALLATION_BINDING_MAX_BYTES}` ) ) - .limit(GITHUB_READ_SOURCE_LIMIT) + .limit(MAX_KNOWLEDGE_ACCESS_CANDIDATES) const contentById = new Map(credentials.map((entry) => [entry.id, entry])) const timeout = AbortSignal.timeout(GITHUB_READ_TIMEOUT_MS) - const signal = input.signal ? AbortSignal.any([input.signal, timeout]) : timeout + const admissionSignal = input.signal ? AbortSignal.any([input.signal, timeout]) : timeout const installations = new Map>() const tokens = new Map>() const proofs = new Map>() const grants = new Map() - for ( - let offset = 0; - offset < sources.length && !signal.aborted; - offset += GITHUB_READ_CONCURRENCY - ) { - await Promise.all( - sources.slice(offset, offset + GITHUB_READ_CONCURRENCY).map(async (source) => { - if (readers.get(source.memberCredentialId) !== source.subjectToken) return - const content = source.contentCredentialId - ? contentById.get(source.contentCredentialId) - : undefined - if (!content?.key || !source.repositoryId) return - try { - let installation = installations.get(content.id) - if (!installation) { - installation = (async () => { - const { decrypted } = await decryptSecret(content.key!) - const binding = parseGitHubInstallationBinding(JSON.parse(decrypted)) - if ( - binding.installationId !== content.installationId || - binding.accountId !== content.accountId - ) - throw new Error('GitHub installation credential identity mismatch') - await assertGitHubInstallationActive(binding, { signal }) - return binding - })() - installations.set(content.id, installation) - } - const binding = await installation - signal.throwIfAborted() - let token = tokens.get(source.memberCredentialId) - if (!token) { - token = resolveManagedOAuthToken({ - credentialId: source.memberCredentialId, - ...resourceScopeFields(input.scope), - expectedProviderId: 'github-repositories', - requiredScopes: [], - }).then(({ accessToken }) => { - if (!accessToken.startsWith('ghu_')) - throw new Error('A GitHub App user token is required') - return accessToken - }) - tokens.set(source.memberCredentialId, token) - } - const accessToken = await withinAdmission(token, signal) - signal.throwIfAborted() - const key = JSON.stringify([ - content.id, - source.memberCredentialId, - source.repositoryId, - source.repository, - source.branch, - ]) - let proof = proofs.get(key) - if (!proof) { - proof = (async () => { - if (!source.repository) return false - await assertGitHubInstallationRepositoryActive(binding, source.repository, { signal }) - return verifyRepository(source, binding.accountId, accessToken, signal) - })() - proofs.set(key, proof) - } - if (await proof) - grants.set(source.connectorId, { - connectorId: source.connectorId, - contentCredentialId: content.id, - readerCredentialId: source.memberCredentialId, - readerSubjectToken: source.subjectToken, - repositoryId: source.repositoryId, - }) - } catch { - logger.warn('GitHub did not confirm current Search access', { - connectorId: source.connectorId, + let nextSource = 0 + const worker = async () => { + while (nextSource < sources.length && !admissionSignal.aborted) { + const source = sources[nextSource++] + const signal = AbortSignal.any([ + admissionSignal, + AbortSignal.timeout(GITHUB_READ_SOURCE_TIMEOUT_MS), + ]) + if (readers.get(source.memberCredentialId) !== source.subjectToken) continue + const content = source.contentCredentialId + ? contentById.get(source.contentCredentialId) + : undefined + if (!content?.key || !source.repositoryId) continue + try { + let installation = installations.get(content.id) + if (!installation) { + installation = (async () => { + const { decrypted } = await decryptSecret(content.key!) + const binding = parseGitHubInstallationBinding(JSON.parse(decrypted)) + if ( + binding.installationId !== content.installationId || + binding.accountId !== content.accountId + ) + throw new Error('GitHub installation credential identity mismatch') + await assertGitHubInstallationActive(binding, { signal: admissionSignal }) + return binding + })() + installations.set(content.id, installation) + } + const binding = await withinAdmission(installation, signal) + signal.throwIfAborted() + let token = tokens.get(source.memberCredentialId) + if (!token) { + token = resolveManagedOAuthToken({ + credentialId: source.memberCredentialId, + ...resourceScopeFields(input.scope), + expectedProviderId: 'github-repositories', + requiredScopes: [], + }).then(({ accessToken }) => { + if (!accessToken.startsWith('ghu_')) + throw new Error('A GitHub App user token is required') + return accessToken }) + tokens.set(source.memberCredentialId, token) } - }) - ) + const accessToken = await withinAdmission(token, signal) + signal.throwIfAborted() + const key = JSON.stringify([ + content.id, + source.memberCredentialId, + source.repositoryId, + source.repository, + source.branch, + ]) + let proof = proofs.get(key) + if (!proof) { + proof = (async () => { + if (!source.repository) return false + await assertGitHubInstallationRepositoryActive(binding, source.repository, { signal }) + return verifyRepository(source, binding.accountId, accessToken, signal) + })() + proofs.set(key, proof) + } + if (await withinAdmission(proof, signal)) + grants.set(source.connectorId, { + connectorId: source.connectorId, + contentCredentialId: content.id, + readerCredentialId: source.memberCredentialId, + readerSubjectToken: source.subjectToken, + repositoryId: source.repositoryId, + }) + } catch { + logger.warn('GitHub did not confirm current Search access', { + connectorId: source.connectorId, + }) + } + } } - /** Once admission expires, none of its partial proofs may authorize a later content query. */ - return signal.aborted ? [] : [...grants.values()] + await Promise.all( + Array.from({ length: Math.min(GITHUB_READ_CONCURRENCY, sources.length) }, worker) + ) + input.signal?.throwIfAborted() + return [...grants.values()] } diff --git a/apps/sim/lib/knowledge/access/predicate.ts b/apps/sim/lib/knowledge/access/predicate.ts index 1f734782142..fb46dd48f86 100644 --- a/apps/sim/lib/knowledge/access/predicate.ts +++ b/apps/sim/lib/knowledge/access/predicate.ts @@ -97,6 +97,27 @@ function githubInstallationAccessCondition(scope: KnowledgeAccessScope): SQL { * partial listings confirm only the documents actually observed. */ export function knowledgeAccessCondition(scope: KnowledgeAccessScope | SystemAccessScope): SQL { + return storedKnowledgeAccessCondition( + scope, + scope.kind === 'system' ? sql`true` : githubInstallationAccessCondition(scope) + ) +} + +/** + * Stored access for fixed identifier/rank candidate projections only. Candidate identities + * must pass live source authorization and knowledgeAccessCondition before content, names, + * tags, counts, provenance, or model input are selected or returned. + */ +export function knowledgeMetadataCandidateAccessCondition( + scope: KnowledgeAccessScope | SystemAccessScope +): SQL { + return storedKnowledgeAccessCondition(scope, sql`true`) +} + +function storedKnowledgeAccessCondition( + scope: KnowledgeAccessScope | SystemAccessScope, + liveSourceAccess: SQL +): SQL { if (scope.kind === 'system') return sql`true` if (scope.tokens.length === 0) return sql`false` const tokens = textArrayLiteral(scope.tokens) @@ -113,7 +134,7 @@ export function knowledgeAccessCondition(scope: KnowledgeAccessScope | SystemAcc SELECT 1 FROM ${knowledgeConnector} WHERE ${knowledgeConnector.id} = ${document.connectorId} AND ${searchIntegrationAccessCondition()} - AND ${githubInstallationAccessCondition(scope)} + AND ${liveSourceAccess} AND ( (${knowledgeConnector.accessMode} = 'workspace' AND ${document.acl} = ARRAY['ws']::text[]) OR (${document.acl} <> ARRAY['ws']::text[] AND ( diff --git a/apps/sim/lib/knowledge/access/scope.test.ts b/apps/sim/lib/knowledge/access/scope.test.ts index 440cdb3ec5d..d1f936e2004 100644 --- a/apps/sim/lib/knowledge/access/scope.test.ts +++ b/apps/sim/lib/knowledge/access/scope.test.ts @@ -468,14 +468,18 @@ describe('organization document ACL scope', () => { providerTenantId: null, }, ]) - const scope = await resolveKnowledgeAccessScope(SESSION, { + const provider = createKnowledgeAccessProvider(SESSION, { ...organization, knowledgeBaseIds: ['index-1'], }) + expect(await provider.get()).not.toHaveProperty('githubInstallationGrants') + expect(mockGitHubReadGrants).not.toHaveBeenCalled() + const scope = await provider.getForConnectors(['source-after-100']) expect(mockGitHubReadGrants).toHaveBeenCalledWith({ scope: { kind: 'organization', organizationId: 'org-1' }, readers: [{ credentialId: 'personal-github', subjectToken: 's:github-repositories:-:42' }], knowledgeBaseIds: ['index-1'], + connectorIds: ['source-after-100'], signal: undefined, }) expect(scope).toMatchObject({ githubInstallationGrants: [] }) diff --git a/apps/sim/lib/knowledge/access/scope.ts b/apps/sim/lib/knowledge/access/scope.ts index 508696b4ff7..63ad6d25dd3 100644 --- a/apps/sim/lib/knowledge/access/scope.ts +++ b/apps/sim/lib/knowledge/access/scope.ts @@ -4,7 +4,9 @@ import { credential, credentialGroup, credentialGroupEnrollment, + document, foldedEmail, + knowledgeBase, knowledgeExternalGroup, knowledgeExternalGroupMember, member, @@ -12,7 +14,7 @@ import { } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { and, eq, gte, inArray, sql } from 'drizzle-orm' +import { and, eq, gte, inArray, isNull, sql } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' import { type ResourceScope, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' @@ -27,6 +29,7 @@ import { type GitHubReaderCredential, resolveGitHubInstallationReadGrants, } from '@/lib/knowledge/access/github-installation' +import { knowledgeMetadataCandidateAccessCondition } from '@/lib/knowledge/access/predicate' import { groupToken, sortAccessTokens, @@ -36,8 +39,8 @@ import { import { type KnowledgeAccessProvider, type KnowledgeAccessScope, + MAX_KNOWLEDGE_ACCESS_CANDIDATES, ORGANIZATION_ACCESS_TOKENS, - type UserAccessScope, WORKSPACE_ACCESS_TOKENS, type WorkspaceAccessScope, } from '@/lib/knowledge/access/types' @@ -142,7 +145,7 @@ export interface KnowledgeAccessScopeContext { async function loadUserAccess( userId: string, context: KnowledgeAccessScopeContext -): Promise> { +): Promise<{ tokens: readonly string[]; githubReaders?: GitHubReaderCredential[] }> { const { workspaceId, organizationId } = context const scope = resourceScopeFromOwner(context) const baseline = organizationId ? ORGANIZATION_ACCESS_TOKENS : WORKSPACE_ACCESS_TOKENS @@ -271,16 +274,7 @@ async function loadUserAccess( return { tokens: sortAccessTokens(new Set([...baseline, ...identityTokens])), - ...(githubReaders.length - ? { - githubInstallationGrants: await resolveGitHubInstallationReadGrants({ - scope, - readers: githubReaders, - knowledgeBaseIds: context.knowledgeBaseIds, - signal: context.signal, - }), - } - : {}), + githubReaders, } } @@ -295,6 +289,13 @@ export async function resolveKnowledgeAccessScope( principal: Principal, context: KnowledgeAccessScopeContext ): Promise { + return (await resolveKnowledgeIdentity(principal, context)).access +} + +async function resolveKnowledgeIdentity( + principal: Principal, + context: KnowledgeAccessScopeContext +): Promise<{ access: KnowledgeAccessScope; githubReaders: readonly GitHubReaderCredential[] }> { if (principal.kind === 'credential_group_enrollment') { throw new OrchestrationError( 'forbidden', @@ -306,12 +307,12 @@ export async function resolveKnowledgeAccessScope( if (subject?.kind !== 'sim_user') { if (context.organizationId) throw new OrchestrationError('forbidden', 'Organization search requires a user subject') - return WORKSPACE_ACCESS_SCOPE + return { access: WORKSPACE_ACCESS_SCOPE, githubReaders: [] } } + const { tokens, githubReaders = [] } = await loadUserAccess(subject.userId, context) return { - kind: 'user', - userId: subject.userId, - ...(await loadUserAccess(subject.userId, context)), + access: { kind: 'user', userId: subject.userId, tokens }, + githubReaders, } } @@ -325,7 +326,7 @@ export async function resolveUserKnowledgeAccessScope( userId: string, workspaceId: string | undefined ): Promise { - return { kind: 'user', userId, ...(await loadUserAccess(userId, { workspaceId })) } + return { kind: 'user', userId, tokens: (await loadUserAccess(userId, { workspaceId })).tokens } } /** Memoises {@link resolveKnowledgeAccessScope} for one operation; a failed lookup is retried on the next call. */ @@ -333,14 +334,71 @@ export function createKnowledgeAccessProvider( principal: Principal, context: KnowledgeAccessScopeContext ): KnowledgeAccessProvider { - let pending: Promise | undefined - return { - get() { - pending ??= resolveKnowledgeAccessScope(principal, context).catch((error: unknown) => { - pending = undefined - throw error - }) - return pending + let pending: ReturnType | undefined + const identity = () => { + pending ??= resolveKnowledgeIdentity(principal, context).catch((error: unknown) => { + pending = undefined + throw error + }) + return pending + } + const boundedIds = (ids: readonly string[]) => { + if (ids.length > MAX_KNOWLEDGE_ACCESS_CANDIDATES) + throw new Error('Knowledge access candidates must be authorized in bounded pages') + return [...new Set(ids)] + } + const provider: KnowledgeAccessProvider = { + async get() { + return (await identity()).access + }, + async getForConnectors(connectorIds, signal) { + const ids = boundedIds(connectorIds) + const cancellation = + context.signal && signal + ? AbortSignal.any([context.signal, signal]) + : (signal ?? context.signal) + cancellation?.throwIfAborted() + const { access, githubReaders } = await identity() + cancellation?.throwIfAborted() + if (access.kind !== 'user' || !githubReaders.length || !ids.length) return access + return { + ...access, + githubInstallationGrants: await resolveGitHubInstallationReadGrants({ + scope: resourceScopeFromOwner(context), + readers: githubReaders, + knowledgeBaseIds: context.knowledgeBaseIds, + connectorIds: ids, + signal: cancellation, + }), + } + }, + async getForDocuments(documentIds, signal) { + const ids = boundedIds(documentIds) + signal?.throwIfAborted() + context.signal?.throwIfAborted() + const { access, githubReaders } = await identity() + if (access.kind !== 'user' || !githubReaders.length || !ids.length) return access + const candidates = await db + .select({ connectorId: document.connectorId }) + .from(document) + .innerJoin(knowledgeBase, eq(knowledgeBase.id, document.knowledgeBaseId)) + .where( + and( + inArray(document.id, ids), + resourceScopeCondition(knowledgeBase, resourceScopeFromOwner(context)), + context.knowledgeBaseIds + ? inArray(knowledgeBase.id, [...context.knowledgeBaseIds]) + : undefined, + isNull(knowledgeBase.deletedAt), + knowledgeMetadataCandidateAccessCondition(access) + ) + ) + .limit(MAX_KNOWLEDGE_ACCESS_CANDIDATES) + return provider.getForConnectors( + candidates.flatMap((candidate) => (candidate.connectorId ? [candidate.connectorId] : [])), + signal + ) }, } + return provider } diff --git a/apps/sim/lib/knowledge/access/types.ts b/apps/sim/lib/knowledge/access/types.ts index d09dfabe3ed..400486649a9 100644 --- a/apps/sim/lib/knowledge/access/types.ts +++ b/apps/sim/lib/knowledge/access/types.ts @@ -74,8 +74,19 @@ export type MirroredDocumentAcl = readonly string[] | SourceDocumentAcl */ export interface KnowledgeAccessProvider { get(): Promise + getForConnectors( + connectorIds: readonly string[], + signal?: AbortSignal + ): Promise + getForDocuments( + documentIds: readonly string[], + signal?: AbortSignal + ): Promise } +/** Two existing search legs each contribute at most 200 candidates to one authorization batch. */ +export const MAX_KNOWLEDGE_ACCESS_CANDIDATES = 400 + declare const systemAccessScopeBrand: unique symbol /** diff --git a/apps/sim/lib/knowledge/application/contexts.test.ts b/apps/sim/lib/knowledge/application/contexts.test.ts index 04eb468265e..f7752fcd4c1 100644 --- a/apps/sim/lib/knowledge/application/contexts.test.ts +++ b/apps/sim/lib/knowledge/application/contexts.test.ts @@ -2,6 +2,7 @@ * @vitest-environment node */ +import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ @@ -15,6 +16,8 @@ const mocks = vi.hoisted(() => ({ loadWorkspaceIncludingArchived: vi.fn(), createAccessProvider: vi.fn(() => ({ get: async () => ({ kind: 'workspace', tokens: ['pub', 'ws'] }), + getForDocuments: vi.fn(async () => ({ kind: 'workspace', tokens: ['pub', 'ws'] })), + getForConnectors: vi.fn(async () => ({ kind: 'workspace', tokens: ['pub', 'ws'] })), })), })) @@ -48,6 +51,7 @@ vi.mock('@/lib/workspaces/application/workspace-context', () => ({ import { loadKnowledgeWorkspaceAuthorizationContext, resolveActiveKnowledgeBaseContext, + resolveActiveKnowledgeChunkContext, resolveActiveKnowledgeConnectorContext, resolveActiveKnowledgeResourceContext, resolveActiveKnowledgeTagContext, @@ -67,6 +71,7 @@ const principal = { kind: 'session' as const, userId: 'user-1', sessionId: 'sess describe('knowledge application contexts', () => { beforeEach(() => { vi.clearAllMocks() + resetDbChainMock() mocks.getKnowledgeBase.mockResolvedValue(knowledgeBase) mocks.getKnowledgeBaseWithCounts.mockResolvedValue({ ...knowledgeBase, @@ -77,6 +82,24 @@ describe('knowledge application contexts', () => { mocks.loadWorkspaceIncludingArchived.mockResolvedValue(workspace) }) + it('selects only chunk identity before document authorization and never hydrates a denied chunk', async () => { + queueTableRows(schemaMock.embedding, [ + { id: 'chunk-1', documentId: 'document-1', knowledgeBaseId: 'knowledge-1' }, + ]) + mocks.getDocumentById.mockResolvedValueOnce(null) + await expect( + resolveActiveKnowledgeChunkContext( + { chunkId: 'chunk-1', documentId: 'document-1', knowledgeBaseId: 'knowledge-1' }, + principal + ) + ).rejects.toMatchObject({ code: 'not_found' }) + expect(dbChainMockFns.select).toHaveBeenCalledExactlyOnceWith({ + id: schemaMock.embedding.id, + documentId: schemaMock.embedding.documentId, + knowledgeBaseId: schemaMock.embedding.knowledgeBaseId, + }) + }) + it('resolves child-resource context without loading display counts', async () => { const context = await resolveActiveKnowledgeResourceContext( { knowledgeBaseId: 'knowledge-1', assertedWorkspaceId: 'workspace-1' }, @@ -87,6 +110,7 @@ describe('knowledge application contexts', () => { expect(mocks.getKnowledgeBaseWithCounts).not.toHaveBeenCalled() expect(mocks.createAccessProvider).toHaveBeenCalledWith(principal, { workspaceId: 'workspace-1', + knowledgeBaseIds: ['knowledge-1'], }) }) diff --git a/apps/sim/lib/knowledge/application/contexts.ts b/apps/sim/lib/knowledge/application/contexts.ts index 0347e8fe312..813571ece43 100644 --- a/apps/sim/lib/knowledge/application/contexts.ts +++ b/apps/sim/lib/knowledge/application/contexts.ts @@ -1,11 +1,12 @@ import type { Principal } from '@sim/auth/principal' import { db } from '@sim/db' -import { embedding, organization } from '@sim/db/schema' -import { and, eq } from 'drizzle-orm' +import { document as documentTable, embedding, organization } from '@sim/db/schema' +import { and, eq, getTableColumns } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' import { type ResourceOwner, resourceScopeFromOwner } from '@/lib/core/resource-scope' +import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' -import type { KnowledgeAccessProvider } from '@/lib/knowledge/access/types' +import type { KnowledgeAccessProvider, KnowledgeAccessScope } from '@/lib/knowledge/access/types' import type { KnowledgeAuthorizationContext, KnowledgeOrganizationAuthorizationContext, @@ -65,6 +66,24 @@ interface KnowledgeAccessBearingContext { access: KnowledgeAccessProvider } +/** A canonical child reuses only its own bounded source admission throughout the operation. */ +function narrowKnowledgeAccessProvider( + provider: KnowledgeAccessProvider, + resolve: () => Promise +): KnowledgeAccessProvider { + let pending: Promise | undefined + return { + ...provider, + get() { + pending ??= resolve().catch((error: unknown) => { + pending = undefined + throw error + }) + return pending + }, + } +} + export interface ActiveKnowledgeBaseContext extends KnowledgeWorkspaceContext, KnowledgeAccessBearingContext { @@ -289,14 +308,18 @@ export async function resolveActiveKnowledgeDocumentContext( principal: Principal ): Promise { const context = await resolveActiveKnowledgeResourceContext(input, principal) + const access = narrowKnowledgeAccessProvider(context.access, () => + context.access.getForDocuments([input.documentId]) + ) const document = await getKnowledgeDocument( context.knowledgeBaseId, input.documentId, - await context.access.get() + await access.get() ) if (!document) throw new OrchestrationError('not_found', 'Document not found') return { ...context, + access, documentId: document.id, document, } @@ -319,12 +342,16 @@ export async function resolveCanonicalActiveKnowledgeDocumentContext( principal: Principal ): Promise { const context = await resolveActiveKnowledgeResourceContext(input, principal) - const document = await getKnowledgeDocumentById(input.documentId, await context.access.get()) + const access = narrowKnowledgeAccessProvider(context.access, () => + context.access.getForDocuments([input.documentId]) + ) + const document = await getKnowledgeDocumentById(input.documentId, await access.get()) if (!document || document.knowledgeBaseId !== context.knowledgeBaseId) { throw new OrchestrationError('not_found', 'Document not found') } return { ...context, + access, documentId: document.id, document, } @@ -340,15 +367,32 @@ export async function resolveActiveKnowledgeChunkContext( }, principal: Principal ): Promise { - const [chunk] = await db - .select() + const [reference] = await db + .select({ + id: embedding.id, + documentId: embedding.documentId, + knowledgeBaseId: embedding.knowledgeBaseId, + }) .from(embedding) .where(and(eq(embedding.id, input.chunkId), eq(embedding.documentId, input.documentId))) .limit(1) - if (!chunk || chunk.knowledgeBaseId !== input.knowledgeBaseId) { + if (!reference || reference.knowledgeBaseId !== input.knowledgeBaseId) { throw new OrchestrationError('not_found', 'Chunk not found') } const context = await resolveCanonicalActiveKnowledgeDocumentContext(input, principal) + const [chunk] = await db + .select(getTableColumns(embedding)) + .from(embedding) + .innerJoin(documentTable, eq(documentTable.id, embedding.documentId)) + .where( + and( + eq(embedding.id, reference.id), + eq(embedding.documentId, context.documentId), + knowledgeAccessCondition(await context.access.get()) + ) + ) + .limit(1) + if (!chunk) throw new OrchestrationError('not_found', 'Chunk not found') return { ...context, chunkId: chunk.id, @@ -406,11 +450,15 @@ export async function resolveActiveKnowledgeConnectorContext( { knowledgeBaseId: connector.knowledgeBaseId, assertedWorkspaceId: input.assertedWorkspaceId, + assertedOrganizationId: input.assertedOrganizationId, }, principal ) return { ...context, + access: narrowKnowledgeAccessProvider(context.access, () => + context.access.getForConnectors([connector.id]) + ), connectorId: connector.id, connector, } diff --git a/apps/sim/lib/knowledge/application/documents.test.ts b/apps/sim/lib/knowledge/application/documents.test.ts index b55db469cfa..751ea4c7426 100644 --- a/apps/sim/lib/knowledge/application/documents.test.ts +++ b/apps/sim/lib/knowledge/application/documents.test.ts @@ -115,7 +115,11 @@ import { } from '@/lib/knowledge/application/documents' /** Every mocked context carries the workspace read scope the resolvers would attach. */ -const knowledgeAccess = { get: async () => WORKSPACE_ACCESS_SCOPE } +const knowledgeAccess = { + get: async () => WORKSPACE_ACCESS_SCOPE, + getForDocuments: async () => WORKSPACE_ACCESS_SCOPE, + getForConnectors: async () => WORKSPACE_ACCESS_SCOPE, +} const context = { access: knowledgeAccess, diff --git a/apps/sim/lib/knowledge/application/documents.ts b/apps/sim/lib/knowledge/application/documents.ts index 508f6e6eacf..90e9e5f04cd 100644 --- a/apps/sim/lib/knowledge/application/documents.ts +++ b/apps/sim/lib/knowledge/application/documents.ts @@ -11,7 +11,6 @@ import { import { authorizeWorkspaceOperation } from '@/lib/core/application' import { asOrchestrationError, OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' -import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' import { knowledgeDelegationPolicy } from '@/lib/knowledge/application/authorization' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { @@ -61,6 +60,7 @@ import { performUploadKnowledgeDocument, performUploadKnowledgeDocuments, } from '@/lib/knowledge/orchestration/documents' +import { knowledgeReadAccessBatches } from '@/lib/knowledge/read-access' import type { KnowledgeDocumentWriteSecretProvenance } from '@/lib/knowledge/secret-provenance' import { type KnowledgeTagNameFilter, @@ -329,7 +329,7 @@ export const listKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ tagFilters: tagFilters.length > 0 ? tagFilters : undefined, }, generateRequestId(), - await context.access.get() + context.organizationId ? context.access : await context.access.get() ) return { ...result, @@ -673,36 +673,27 @@ export const upsertKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ * Only a document the caller may read counts as the one being replaced: * a restricted document is neither confirmed to exist nor replaced. */ - const access = await context.access.get() + const lookupConditions = [ + eq(documentTable.knowledgeBaseId, context.knowledgeBaseId), + isNull(documentTable.deletedAt), + input.documentId + ? eq(documentTable.id, input.documentId) + : eq(documentTable.filename, input.filename), + ] let existingDocumentId: string | null = null - if (input.documentId) { - const [existing] = await db - .select({ id: documentTable.id }) - .from(documentTable) - .where( - and( - eq(documentTable.id, input.documentId), - eq(documentTable.knowledgeBaseId, context.knowledgeBaseId), - isNull(documentTable.deletedAt), - knowledgeAccessCondition(access) - ) - ) - .limit(1) - existingDocumentId = existing?.id ?? null - } else { + for await (const accessCondition of knowledgeReadAccessBatches( + context.organizationId ? context.access : await context.access.get(), + lookupConditions + )) { const [existing] = await db .select({ id: documentTable.id }) .from(documentTable) - .where( - and( - eq(documentTable.filename, input.filename), - eq(documentTable.knowledgeBaseId, context.knowledgeBaseId), - isNull(documentTable.deletedAt), - knowledgeAccessCondition(access) - ) - ) + .where(and(...lookupConditions, accessCondition)) .limit(1) - existingDocumentId = existing?.id ?? null + if (existing) { + existingDocumentId = existing.id + break + } } const requestId = generateRequestId() const createdDocuments = await createDocumentRecords( @@ -728,7 +719,7 @@ export const upsertKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ context.knowledgeBaseId, existingDocumentId, requestId, - access + await context.access.getForDocuments([existingDocumentId]) ) } catch (error) { /** @@ -889,7 +880,7 @@ export const bulkDeleteKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ canonical.knowledgeBaseId, canonical.documentId, generateRequestId(), - await context.access.get() + await canonical.access.get() ) deletedDocuments.push({ id: canonical.documentId, @@ -1036,7 +1027,7 @@ export const bulkUpdateKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ context.knowledgeBaseId, input.operation, input.enabledFilter, - await context.access.get(), + context.organizationId ? context.access : await context.access.get(), generateRequestId() ) : input.documentIds?.length @@ -1044,7 +1035,7 @@ export const bulkUpdateKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ context.knowledgeBaseId, input.operation, input.documentIds, - await context.access.get(), + context.organizationId ? context.access : await context.access.get(), generateRequestId() ) : null diff --git a/apps/sim/lib/knowledge/application/operations.test.ts b/apps/sim/lib/knowledge/application/operations.test.ts index d5732c64c53..246bcd4e307 100644 --- a/apps/sim/lib/knowledge/application/operations.test.ts +++ b/apps/sim/lib/knowledge/application/operations.test.ts @@ -16,6 +16,8 @@ describe('knowledge operation registry', () => { it('defines unique stable semantic operation IDs', () => { const ids = Object.values(knowledgeOperations).map((operation) => operation.id) expect(ids).toEqual([ + 'knowledge.github.installations.list', + 'knowledge.github.installations.connect', 'knowledge.slack.prepare', 'knowledge.slack.oauth.start', 'knowledge.slack.oauth.complete', diff --git a/apps/sim/lib/knowledge/application/read-indexed-document.ts b/apps/sim/lib/knowledge/application/read-indexed-document.ts index f611d2f19aa..37acf6579b2 100644 --- a/apps/sim/lib/knowledge/application/read-indexed-document.ts +++ b/apps/sim/lib/knowledge/application/read-indexed-document.ts @@ -14,6 +14,7 @@ import { import { readKnowledgeDocument } from '@/lib/knowledge/application/documents' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import type { ChunkQueryResult } from '@/lib/knowledge/chunks/types' +import { knowledgeReadAccessBatches } from '@/lib/knowledge/read-access' import { isKnowledgeSourceUrl } from '@/lib/knowledge/search/citation' import { findSearchIndex } from '@/lib/knowledge/search/search-index' import { @@ -48,14 +49,14 @@ export interface ReadIndexedKnowledgeDocumentResult { pagination?: ChunkQueryResult['pagination'] } -function activeDocumentConditions(knowledgeBaseId: string, access: KnowledgeAccessScope) { +function activeDocumentConditions(knowledgeBaseId: string, access?: KnowledgeAccessScope) { return [ eq(document.knowledgeBaseId, knowledgeBaseId), eq(document.enabled, true), eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), - knowledgeAccessCondition(access), + access ? knowledgeAccessCondition(access) : undefined, ] } @@ -112,21 +113,29 @@ export const readIndexedKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ { knowledgeBaseId, ...assertions }, principal ) - const access = await knowledgeContext.access.get() let documentId: string if (input.target.kind === 'id') { documentId = input.target.documentId } else { - const matches = await db - .select({ id: document.id }) - .from(document) - .where( - and( - ...activeDocumentConditions(knowledgeBaseId, access), - eq(document.sourceUrl, input.target.url.trim()) - ) + const conditions = [ + ...activeDocumentConditions(knowledgeBaseId), + eq(document.sourceUrl, input.target.url.trim()), + ] + const matches: { id: string }[] = [] + for await (const accessCondition of knowledgeReadAccessBatches( + knowledgeContext.access, + conditions, + input.signal + )) { + matches.push( + ...(await db + .select({ id: document.id }) + .from(document) + .where(and(...conditions, accessCondition)) + .limit(2 - matches.length)) ) - .limit(2) + if (matches.length > 1) break + } if (!matches.length) throw new OrchestrationError('not_found', 'Document not found') if (matches.length > 1) { throw new OrchestrationError( @@ -136,6 +145,7 @@ export const readIndexedKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ } documentId = matches[0].id } + const access = await knowledgeContext.access.getForDocuments([documentId], input.signal) input.signal?.throwIfAborted() const { document: doc } = await readKnowledgeDocument.execute({ principal, diff --git a/apps/sim/lib/knowledge/application/search-source-overview.ts b/apps/sim/lib/knowledge/application/search-source-overview.ts index fbf1b56f503..b75d5a1611d 100644 --- a/apps/sim/lib/knowledge/application/search-source-overview.ts +++ b/apps/sim/lib/knowledge/application/search-source-overview.ts @@ -5,12 +5,12 @@ import type { SearchSourceOverview } from '@/lib/api/contracts/knowledge/connect import { type ResourceOwner, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { resolveKnowledgeAccessAvailability } from '@/lib/knowledge/access/availability' -import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { resolveKnowledgeOwnerContext } from '@/lib/knowledge/application/contexts' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { MAX_SEARCH_SOURCE_PROVIDER_TYPES } from '@/lib/knowledge/constants' +import { knowledgeReadAccessBatches } from '@/lib/knowledge/read-access' import { searchIntegrationAccessCondition } from '@/lib/knowledge/search/integration-policy' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' @@ -19,10 +19,8 @@ export const readSearchSourceOverview = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.readSearchSourceOverview, resolveContext: ({ input }: { input: ResourceOwner }) => resolveKnowledgeOwnerContext(input), async execute({ principal, context }): Promise { - const [availability, access] = await Promise.all([ - resolveKnowledgeAccessAvailability(context), - createKnowledgeAccessProvider(principal, context).get(), - ]) + const availability = await resolveKnowledgeAccessAvailability(context) + const access = createKnowledgeAccessProvider(principal, context) const providerTypes = Object.keys(CONNECTOR_META_REGISTRY) if (providerTypes.length > MAX_SEARCH_SOURCE_PROVIDER_TYPES) { throw new Error('Search provider catalog exceeds the overview bound') @@ -60,80 +58,93 @@ export const readSearchSourceOverview = defineAuthorizedKnowledgeUseCase({ notInArray(knowledgeConnector.memberSyncStatus, ['disabled']) ) ) - const readableDocument = and( + const documentConditions = and( eq(document.connectorId, knowledgeConnector.id), eq(document.knowledgeBaseId, knowledgeConnector.knowledgeBaseId), eq(document.enabled, true), eq(document.userExcluded, false), isNull(document.archivedAt), - isNull(document.deletedAt), - knowledgeAccessCondition(access) + isNull(document.deletedAt) ) const providersQuery = () => db .selectDistinct({ connectorType: knowledgeConnector.connectorType }) .from(knowledgeConnector) .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) - const [providers, indexing, searchable] = await Promise.all([ - providersQuery().where(configured).limit(MAX_SEARCH_SOURCE_PROVIDER_TYPES), - availability.memberScoped || availability.sourceMirrored - ? providersQuery() - .where( - and( - configured, - syncingEnabled, - or( - inArray(knowledgeConnector.status, ['pending', 'syncing']), - and( - eq(knowledgeConnector.accessMode, 'members'), - inArray(knowledgeConnector.memberSyncStatus, ['pending', 'running']) - ), + const providers = await providersQuery() + .where(configured) + .limit(MAX_SEARCH_SOURCE_PROVIDER_TYPES) + const indexingTypes = new Set() + let hasSearchableDocuments = false + for await (const accessCondition of knowledgeReadAccessBatches(access, [ + configured, + available, + documentConditions, + ])) { + const readableDocument = and(documentConditions, accessCondition) + const [indexing, searchable] = await Promise.all([ + availability.memberScoped || availability.sourceMirrored + ? providersQuery() + .where( + and( + configured, + syncingEnabled, + or( + inArray(knowledgeConnector.status, ['pending', 'syncing']), + and( + eq(knowledgeConnector.accessMode, 'members'), + inArray(knowledgeConnector.memberSyncStatus, ['pending', 'running']) + ), + exists( + db + .select({ id: document.id }) + .from(document) + .where( + and( + readableDocument, + inArray(document.processingStatus, ['pending', 'processing']) + ) + ) + ) + ) + ) + ) + .limit(MAX_SEARCH_SOURCE_PROVIDER_TYPES) + : [], + availability.memberScoped || availability.sourceMirrored + ? db + .select({ id: document.id }) + .from(document) + .innerJoin(knowledgeConnector, eq(knowledgeConnector.id, document.connectorId)) + .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) + .where( + and( + configured, + available, + readableDocument, + eq(document.processingStatus, 'completed'), exists( db - .select({ id: document.id }) - .from(document) + .select({ id: embedding.id }) + .from(embedding) .where( - and( - readableDocument, - inArray(document.processingStatus, ['pending', 'processing']) - ) + and(eq(embedding.documentId, document.id), eq(embedding.enabled, true)) ) ) ) ) - ) - .limit(MAX_SEARCH_SOURCE_PROVIDER_TYPES) - : [], - availability.memberScoped || availability.sourceMirrored - ? db - .select({ id: document.id }) - .from(document) - .innerJoin(knowledgeConnector, eq(knowledgeConnector.id, document.connectorId)) - .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) - .where( - and( - configured, - available, - readableDocument, - eq(document.processingStatus, 'completed'), - exists( - db - .select({ id: embedding.id }) - .from(embedding) - .where(and(eq(embedding.documentId, document.id), eq(embedding.enabled, true))) - ) - ) - ) - .limit(1) - : [], - ]) - const indexingTypes = new Set(indexing.map((provider) => provider.connectorType)) + .limit(1) + : [], + ]) + for (const provider of indexing) indexingTypes.add(provider.connectorType) + hasSearchableDocuments ||= searchable.length > 0 + } return { providers: providers.map(({ connectorType }) => ({ connectorType, isSyncing: indexingTypes.has(connectorType), })), - hasSearchableDocuments: searchable.length > 0, + hasSearchableDocuments, } }, }) diff --git a/apps/sim/lib/knowledge/application/search-source-progress.ts b/apps/sim/lib/knowledge/application/search-source-progress.ts index dbabc0f8b00..5543c3ec974 100644 --- a/apps/sim/lib/knowledge/application/search-source-progress.ts +++ b/apps/sim/lib/knowledge/application/search-source-progress.ts @@ -31,7 +31,9 @@ export const readSearchSourceProgress = defineAuthorizedKnowledgeUseCase({ `Provide between 1 and ${MAX_SEARCH_SOURCE_PROGRESS_ITEMS} sources` ) } - const access = await createKnowledgeAccessProvider(principal, context).get() + const access = await createKnowledgeAccessProvider(principal, context).getForConnectors( + input.connectorIds + ) const hasDocumentsInState = (statuses: string[]) => sql`${exists( db diff --git a/apps/sim/lib/knowledge/application/search-sources.test.ts b/apps/sim/lib/knowledge/application/search-sources.test.ts index 036aed5ce73..7780bf13f9b 100644 --- a/apps/sim/lib/knowledge/application/search-sources.test.ts +++ b/apps/sim/lib/knowledge/application/search-sources.test.ts @@ -41,6 +41,7 @@ vi.mock('@/lib/knowledge/access/scope', () => ({ })) vi.mock('@/lib/knowledge/access/predicate', () => ({ knowledgeAccessCondition: mocks.predicate, + knowledgeMetadataCandidateAccessCondition: mocks.predicate, })) vi.mock('@/connectors/registry', () => { const registry = { @@ -113,7 +114,11 @@ beforeEach(() => { mocks.permission.mockResolvedValue('read') mocks.availability.mockResolvedValue({ sourceMirrored: true, memberScoped: true }) mocks.memberships.mockResolvedValue(new Map()) - mocks.access.mockReturnValue({ get: async () => access }) + mocks.access.mockReturnValue({ + get: async () => access, + getForConnectors: async () => access, + getForDocuments: async () => access, + }) mocks.predicate.mockReturnValue(ACL) }) diff --git a/apps/sim/lib/knowledge/application/search-sources.ts b/apps/sim/lib/knowledge/application/search-sources.ts index 8e1599057be..a639708d49d 100644 --- a/apps/sim/lib/knowledge/application/search-sources.ts +++ b/apps/sim/lib/knowledge/application/search-sources.ts @@ -106,7 +106,7 @@ export const listSearchSources = defineAuthorizedKnowledgeUseCase({ if (candidates.length === 0) return { sources: [], nextCursor: null } const scanned = candidates.slice(0, SEARCH_SOURCE_CANDIDATE_PAGE_SIZE) - const [availability, memberships, viewers, access, approvals] = await Promise.all([ + const [availability, memberships, viewers, approvals] = await Promise.all([ resolveKnowledgeAccessAvailability(context), resolveViewerConnectorMemberships({ userId: principal.userId, @@ -119,7 +119,6 @@ export const listSearchSources = defineAuthorizedKnowledgeUseCase({ .from(user) .where(eq(user.id, principal.userId)) .limit(1), - createKnowledgeAccessProvider(principal, context).get(), context.organizationId ? listOrganizationSearchApprovals(context.organizationId) : null, ]) /** Filtering uses the same safe display labels and verified membership as the source rows. */ @@ -145,6 +144,9 @@ export const listSearchSources = defineAuthorizedKnowledgeUseCase({ ).toString('base64url') : null if (rows.length === 0) return { sources: [], nextCursor } + const access = await createKnowledgeAccessProvider(principal, context).getForConnectors( + rows.map((row) => row.id) + ) const documentStates = await db .select({ connectorId: document.connectorId, diff --git a/apps/sim/lib/knowledge/application/search.test.ts b/apps/sim/lib/knowledge/application/search.test.ts index ce1a911aed1..185e4d64760 100644 --- a/apps/sim/lib/knowledge/application/search.test.ts +++ b/apps/sim/lib/knowledge/application/search.test.ts @@ -626,7 +626,12 @@ describe('knowledge search application use case', () => { }, }) - expect(mocks.getDocumentMetadata).toHaveBeenCalledWith(['document-1'], expect.anything()) + expect(mocks.getDocumentMetadata).toHaveBeenCalledWith( + ['document-1'], + expect.anything(), + undefined, + undefined + ) expect(result.results[0]).toMatchObject({ documentName: 'guide.pdf', sourceUrl: 'https://example.com/guide', diff --git a/apps/sim/lib/knowledge/application/search.ts b/apps/sim/lib/knowledge/application/search.ts index 84d25e239b7..a47c6f7141c 100644 --- a/apps/sim/lib/knowledge/application/search.ts +++ b/apps/sim/lib/knowledge/application/search.ts @@ -374,6 +374,8 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ topK: candidateTopK, filters: input.filters, access, + accessProvider: context.organizationId ? context.access : undefined, + signal: input.signal, searchMode: searchDefaults.searchMode, boostRecency: searchDefaults.boostRecency, query: input.query, @@ -575,7 +577,9 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ */ const basicDocumentMetadata = await getDocumentMetadataByIds( rows.map((row) => row.documentId), - access + access, + context.organizationId ? context.access : undefined, + input.signal ) const results = rows .filter((row) => basicDocumentMetadata[row.documentId]) diff --git a/apps/sim/lib/knowledge/application/slack-search/source-status.ts b/apps/sim/lib/knowledge/application/slack-search/source-status.ts index 817299c61bc..d9a5e764f39 100644 --- a/apps/sim/lib/knowledge/application/slack-search/source-status.ts +++ b/apps/sim/lib/knowledge/application/slack-search/source-status.ts @@ -4,8 +4,8 @@ import { and, eq, exists, isNull } from 'drizzle-orm' import type { OperationUseCase } from '@/lib/core/application/operation' import { authorizeOrganizationOperation } from '@/lib/core/application/organization-authorization' import { defineOrganizationOperation } from '@/lib/core/application/organization-operation' -import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' +import { knowledgeReadAccessBatches } from '@/lib/knowledge/read-access' const operation = defineOrganizationOperation({ id: 'knowledge.slack.sources.status', @@ -25,31 +25,32 @@ export const getSlackSearchSourceStatus: OperationUseCase< operation, async execute({ principal, input }) { await authorizeOrganizationOperation(principal, operation, input) - const access = await createKnowledgeAccessProvider(principal, input).get() - const [visible] = await db - .select({ id: document.id }) - .from(document) - .innerJoin(knowledgeBase, eq(knowledgeBase.id, document.knowledgeBaseId)) - .where( - and( - eq(knowledgeBase.organizationId, input.organizationId), - eq(knowledgeBase.isSearchIndex, true), - isNull(knowledgeBase.deletedAt), - eq(document.processingStatus, 'completed'), - eq(document.enabled, true), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - knowledgeAccessCondition(access), - exists( - db - .select({ id: embedding.id }) - .from(embedding) - .where(and(eq(embedding.documentId, document.id), eq(embedding.enabled, true))) - ) - ) - ) - .limit(1) - return { hasSearchableDocuments: Boolean(visible) } + const access = createKnowledgeAccessProvider(principal, input) + const conditions = [ + eq(knowledgeBase.organizationId, input.organizationId), + eq(knowledgeBase.isSearchIndex, true), + isNull(knowledgeBase.deletedAt), + eq(document.processingStatus, 'completed'), + eq(document.enabled, true), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt), + exists( + db + .select({ id: embedding.id }) + .from(embedding) + .where(and(eq(embedding.documentId, document.id), eq(embedding.enabled, true))) + ), + ] + for await (const accessCondition of knowledgeReadAccessBatches(access, conditions)) { + const [visible] = await db + .select({ id: document.id }) + .from(document) + .innerJoin(knowledgeBase, eq(knowledgeBase.id, document.knowledgeBaseId)) + .where(and(...conditions, accessCondition)) + .limit(1) + if (visible) return { hasSearchableDocuments: true } + } + return { hasSearchableDocuments: false } }, } diff --git a/apps/sim/lib/knowledge/application/tags.ts b/apps/sim/lib/knowledge/application/tags.ts index c2e836f1075..dd8b9a56c8f 100644 --- a/apps/sim/lib/knowledge/application/tags.ts +++ b/apps/sim/lib/knowledge/application/tags.ts @@ -357,7 +357,7 @@ export const readKnowledgeTagUsage = defineAuthorizedKnowledgeUseCase({ return { usage: await getTagUsageStats( context.knowledgeBaseId, - await context.access.get(), + context.organizationId ? context.access : await context.access.get(), generateRequestId() ), } @@ -373,7 +373,7 @@ export const readDetailedKnowledgeTagUsage = defineAuthorizedKnowledgeUseCase({ usage: await getTagUsage( context.knowledgeBaseId, generateRequestId(), - await context.access.get() + context.organizationId ? context.access : await context.access.get() ), } }, diff --git a/apps/sim/lib/knowledge/documents/service.test.ts b/apps/sim/lib/knowledge/documents/service.test.ts new file mode 100644 index 00000000000..974a5e3fdd1 --- /dev/null +++ b/apps/sim/lib/knowledge/documents/service.test.ts @@ -0,0 +1,54 @@ +/** @vitest-environment node */ +import { document } from '@sim/db/schema' +import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { WORKSPACE_ACCESS_SCOPE } from '@/lib/knowledge/access/scope' +import type { KnowledgeAccessProvider, KnowledgeAccessScope } from '@/lib/knowledge/access/types' +import { getDocuments } from '@/lib/knowledge/documents/service' + +beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() +}) + +describe('getDocuments pagination', () => { + it('keeps static scopes on the direct count and offset queries', async () => { + queueTableRows(document, [{ count: 25 }]) + queueTableRows(document, []) + + const result = await getDocuments( + 'knowledge-1', + { limit: 5, offset: 20 }, + 'request-1', + WORKSPACE_ACCESS_SCOPE + ) + + expect(result.pagination).toEqual({ total: 25, limit: 5, offset: 20, hasMore: false }) + expect(dbChainMockFns.select).toHaveBeenCalledTimes(2) + expect(dbChainMockFns.selectDistinct).not.toHaveBeenCalled() + expect(dbChainMockFns.as).not.toHaveBeenCalled() + expect(dbChainMockFns.offset).toHaveBeenCalledWith(20) + expect(dbChainMockFns.limit).toHaveBeenCalledWith(5) + }) + + it('reads a newly admitted candidate even when the earlier count admitted no documents', async () => { + const identity: KnowledgeAccessScope = { kind: 'user', userId: 'reader', tokens: [] } + const resolve = vi.fn(async () => identity) + const access: KnowledgeAccessProvider = { + get: async () => identity, + getForConnectors: async () => identity, + getForDocuments: resolve, + } + queueTableRows(document, [{ count: 0 }]) + dbChainMockFns.limit + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([{ id: 'newly-readable', rank: 1 }]) + .mockResolvedValueOnce([{ id: 'newly-readable', rank: 1 }]) + .mockResolvedValueOnce([{ id: 'newly-readable', filename: 'Visible file' }]) + + const result = await getDocuments('knowledge-1', { limit: 1 }, 'request-1', access) + + expect(resolve).toHaveBeenCalledWith(['newly-readable']) + expect(result.documents).toMatchObject([{ id: 'newly-readable', filename: 'Visible file' }]) + }) +}) diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index ca05d231cc3..8a555fb1834 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -70,9 +70,13 @@ import { EXACT_EMPTY_DURABLE_SECRET_PROVENANCE, mergeDurableSecretProvenance, } from '@/lib/execution/durable-secret-provenance' -import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' +import { + knowledgeAccessCondition, + knowledgeMetadataCandidateAccessCondition, +} from '@/lib/knowledge/access/predicate' import { type KnowledgeAccessScope, + MAX_KNOWLEDGE_ACCESS_CANDIDATES, SYSTEM_ACCESS_SCOPE, type SystemAccessScope, } from '@/lib/knowledge/access/types' @@ -138,6 +142,7 @@ import { } from '@/lib/knowledge/embedding-models' import { generateEmbeddings, type KbEmbeddingTarget } from '@/lib/knowledge/embeddings' import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance' +import { type KnowledgeReadAccess, knowledgeReadAccessBatches } from '@/lib/knowledge/read-access' import { bindKnowledgeDocumentFieldSecretProvenance, createKnowledgeDocumentSourceValue, @@ -2456,7 +2461,7 @@ export async function getDocuments( tagFilters?: TagFilterCondition[] }, requestId: string, - access: KnowledgeAccessScope | SystemAccessScope + access: KnowledgeReadAccess ): Promise<{ documents: Array<{ id: string @@ -2517,7 +2522,6 @@ export async function getDocuments( eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), - knowledgeAccessCondition(access), ] if (enabledFilter === 'enabled') { @@ -2538,14 +2542,6 @@ export async function getDocuments( } } - const totalResult = await db - .select({ count: sql`COUNT(*)` }) - .from(document) - .where(and(...whereConditions)) - - const total = Number(totalResult[0]?.count ?? 0) - const hasMore = offset + limit < total - const getOrderByColumn = () => { switch (sortBy) { case 'filename': @@ -2571,50 +2567,117 @@ export async function getDocuments( const secondaryOrderBy = sortBy === 'filename' ? desc(document.uploadedAt) : asc(document.filename) - const documents = await db - .select({ - id: document.id, - knowledgeBaseId: document.knowledgeBaseId, - filename: document.filename, - fileUrl: document.fileUrl, - fileSize: document.fileSize, - mimeType: document.mimeType, - chunkCount: document.chunkCount, - tokenCount: document.tokenCount, - characterCount: document.characterCount, - processingStatus: document.processingStatus, - processingStartedAt: document.processingStartedAt, - processingCompletedAt: document.processingCompletedAt, - processingError: document.processingError, - enabled: document.enabled, - uploadedAt: document.uploadedAt, - tag1: document.tag1, - tag2: document.tag2, - tag3: document.tag3, - tag4: document.tag4, - tag5: document.tag5, - tag6: document.tag6, - tag7: document.tag7, - number1: document.number1, - number2: document.number2, - number3: document.number3, - number4: document.number4, - number5: document.number5, - date1: document.date1, - date2: document.date2, - boolean1: document.boolean1, - boolean2: document.boolean2, - boolean3: document.boolean3, - connectorId: document.connectorId, - connectorType: knowledgeConnector.connectorType, - sourceUrl: document.sourceUrl, - }) - .from(document) - .leftJoin(knowledgeConnector, eq(document.connectorId, knowledgeConnector.id)) - .where(and(...whereConditions)) - .orderBy(primaryOrderBy, secondaryOrderBy) - .limit(limit) - .offset(offset) + const readDocuments = () => + db + .select({ + id: document.id, + knowledgeBaseId: document.knowledgeBaseId, + filename: document.filename, + fileUrl: document.fileUrl, + fileSize: document.fileSize, + mimeType: document.mimeType, + chunkCount: document.chunkCount, + tokenCount: document.tokenCount, + characterCount: document.characterCount, + processingStatus: document.processingStatus, + processingStartedAt: document.processingStartedAt, + processingCompletedAt: document.processingCompletedAt, + processingError: document.processingError, + enabled: document.enabled, + uploadedAt: document.uploadedAt, + tag1: document.tag1, + tag2: document.tag2, + tag3: document.tag3, + tag4: document.tag4, + tag5: document.tag5, + tag6: document.tag6, + tag7: document.tag7, + number1: document.number1, + number2: document.number2, + number3: document.number3, + number4: document.number4, + number5: document.number5, + date1: document.date1, + date2: document.date2, + boolean1: document.boolean1, + boolean2: document.boolean2, + boolean3: document.boolean3, + connectorId: document.connectorId, + connectorType: knowledgeConnector.connectorType, + sourceUrl: document.sourceUrl, + }) + .from(document) + .leftJoin(knowledgeConnector, eq(document.connectorId, knowledgeConnector.id)) + + let total = 0 + for await (const accessCondition of knowledgeReadAccessBatches(access, whereConditions)) { + const [counts] = await db + .select({ count: sql`COUNT(*)` }) + .from(document) + .where(and(...whereConditions, accessCondition)) + total += Number(counts?.count ?? 0) + } + + let documents: Awaited> = [] + if (!('get' in access)) { + documents = await readDocuments() + .where(and(...whereConditions, knowledgeAccessCondition(access))) + .orderBy(primaryOrderBy, secondaryOrderBy) + .limit(limit) + .offset(offset) + } else { + const identity = await access.get() + const rankedCandidates = db + .select({ + id: document.id, + rank: sql`row_number() over (order by ${primaryOrderBy}, ${secondaryOrderBy}, ${asc(document.id)})` + .mapWith(Number) + .as('read_rank'), + }) + .from(document) + .where(and(...whereConditions, knowledgeMetadataCandidateAccessCondition(identity))) + .as('knowledge_document_candidates') + let remainingOffset = offset + let lastRank = 0 + while (documents.length < limit) { + const candidates = await db + .select({ id: rankedCandidates.id, rank: rankedCandidates.rank }) + .from(rankedCandidates) + .where(sql`${rankedCandidates.rank} > ${lastRank}`) + .orderBy(asc(rankedCandidates.rank)) + .limit(MAX_KNOWLEDGE_ACCESS_CANDIDATES) + if (candidates.length === 0) break + const candidateIds = candidates.map((candidate) => candidate.id) + const scope = await access.getForDocuments(candidateIds) + const accessCondition = knowledgeAccessCondition(scope) + const visible = await db + .select({ id: document.id, rank: rankedCandidates.rank }) + .from(document) + .innerJoin(rankedCandidates, eq(document.id, rankedCandidates.id)) + .where(and(...whereConditions, inArray(document.id, candidateIds), accessCondition)) + .orderBy(asc(rankedCandidates.rank)) + .limit(MAX_KNOWLEDGE_ACCESS_CANDIDATES) + if (remainingOffset >= visible.length) remainingOffset -= visible.length + else { + const pageIds = visible + .slice(remainingOffset, remainingOffset + limit - documents.length) + .map((row) => row.id) + remainingOffset = 0 + if (pageIds.length) { + documents.push( + ...(await readDocuments() + .innerJoin(rankedCandidates, eq(document.id, rankedCandidates.id)) + .where(and(...whereConditions, accessCondition, inArray(document.id, pageIds))) + .orderBy(asc(rankedCandidates.rank)) + .limit(limit)) + ) + } + } + if (candidates.length < MAX_KNOWLEDGE_ACCESS_CANDIDATES) break + lastRank = candidates[candidates.length - 1].rank + } + } + const hasMore = offset + limit < total logger.info( `[${requestId}] Retrieved ${documents.length} documents (${offset}-${offset + documents.length} of ${total}) for knowledge base ${knowledgeBaseId}` @@ -3053,7 +3116,7 @@ export async function bulkDocumentOperation( knowledgeBaseId: string, operation: 'enable' | 'disable' | 'delete', documentIds: string[], - access: KnowledgeAccessScope, + access: KnowledgeReadAccess, requestId: string ): Promise<{ success: boolean @@ -3069,22 +3132,31 @@ export async function bulkDocumentOperation( `[${requestId}] Starting bulk ${operation} operation on ${documentIds.length} documents in knowledge base ${knowledgeBaseId}` ) - const documentsToUpdate = await db - .select({ - id: document.id, - enabled: document.enabled, - }) - .from(document) - .where( - and( - eq(document.knowledgeBaseId, knowledgeBaseId), - inArray(document.id, documentIds), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - knowledgeAccessCondition(access) - ) + const candidateConditions = [ + eq(document.knowledgeBaseId, knowledgeBaseId), + inArray(document.id, documentIds), + ] + const documentsToUpdate: { id: string; enabled: boolean }[] = [] + for await (const accessCondition of knowledgeReadAccessBatches(access, candidateConditions)) { + documentsToUpdate.push( + ...(await db + .select({ + id: document.id, + enabled: document.enabled, + }) + .from(document) + .where( + and( + eq(document.knowledgeBaseId, knowledgeBaseId), + inArray(document.id, documentIds), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt), + accessCondition + ) + )) ) + } if (documentsToUpdate.length === 0) { throw new OrchestrationError('not_found', 'No valid documents found to update') @@ -3147,7 +3219,7 @@ export async function bulkDocumentOperationByFilter( knowledgeBaseId: string, operation: 'enable' | 'disable' | 'delete', enabledFilter: 'all' | 'enabled' | 'disabled' | undefined, - access: KnowledgeAccessScope, + access: KnowledgeReadAccess, requestId: string ): Promise<{ success: boolean @@ -3167,8 +3239,6 @@ export async function bulkDocumentOperationByFilter( eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), - /** "Every document" means every document the caller can see. */ - knowledgeAccessCondition(access), ] if (enabledFilter === 'enabled') { @@ -3177,33 +3247,36 @@ export async function bulkDocumentOperationByFilter( whereConditions.push(eq(document.enabled, false)) } - let updateResult: Array<{ + const updateResult: Array<{ id: string enabled?: boolean deletedAt?: Date | null - }> - - if (operation === 'delete') { - const matchingDocs = await db - .select({ id: document.id }) - .from(document) - .where(and(...whereConditions)) - - const deletedIds = matchingDocs.map((doc) => doc.id) - const deletedCount = await deleteDocumentsByLifecyclePolicy(deletedIds, requestId) - updateResult = deletedIds.slice(0, deletedCount).map((id) => ({ id })) - } else { - const enabled = operation === 'enable' + }> = [] + + for await (const accessCondition of knowledgeReadAccessBatches(access, whereConditions)) { + if (operation === 'delete') { + const matchingDocs = await db + .select({ id: document.id }) + .from(document) + .where(and(...whereConditions, accessCondition)) + + const deletedIds = matchingDocs.map((doc) => doc.id) + const deletedCount = await deleteDocumentsByLifecyclePolicy(deletedIds, requestId) + updateResult.push(...deletedIds.slice(0, deletedCount).map((id) => ({ id }))) + } else { + const enabled = operation === 'enable' - updateResult = await db - .update(document) - .set({ - enabled, - }) - .where(and(...whereConditions)) - .returning({ id: document.id, enabled: document.enabled }) + updateResult.push( + ...(await db + .update(document) + .set({ + enabled, + }) + .where(and(...whereConditions, accessCondition)) + .returning({ id: document.id, enabled: document.enabled })) + ) + } } - const successCount = updateResult.length logger.info( diff --git a/apps/sim/lib/knowledge/read-access.test.ts b/apps/sim/lib/knowledge/read-access.test.ts new file mode 100644 index 00000000000..9d0d1179fab --- /dev/null +++ b/apps/sim/lib/knowledge/read-access.test.ts @@ -0,0 +1,75 @@ +/** @vitest-environment node */ +import { document } from '@sim/db/schema' +import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' +import { eq, gt } from 'drizzle-orm' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + type KnowledgeAccessProvider, + type KnowledgeAccessScope, + MAX_KNOWLEDGE_ACCESS_CANDIDATES, +} from '@/lib/knowledge/access/types' +import { knowledgeReadAccessBatches } from '@/lib/knowledge/read-access' + +const identity: KnowledgeAccessScope = { kind: 'user', userId: 'reader', tokens: ['org'] } + +beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() +}) + +describe('knowledgeReadAccessBatches', () => { + it('continues after a full denied candidate page using fixed source IDs only', async () => { + const first = Array.from({ length: MAX_KNOWLEDGE_ACCESS_CANDIDATES }, (_, index) => ({ + connectorId: `source-${String(index).padStart(4, '0')}`, + })) + queueTableRows(document, first) + queueTableRows(document, [{ connectorId: 'source-last' }]) + const resolve = vi.fn(async () => identity) + const provider: KnowledgeAccessProvider = { + get: async () => identity, + getForConnectors: resolve, + getForDocuments: async () => identity, + } + const filter = eq(document.knowledgeBaseId, 'one-index') + const batches = [] + for await (const predicate of knowledgeReadAccessBatches(provider, [filter])) + batches.push(predicate) + expect(batches).toHaveLength(3) + expect(resolve.mock.calls).toHaveLength(2) + expect(resolve).toHaveBeenNthCalledWith( + 1, + first.map((row) => row.connectorId), + undefined + ) + expect(resolve).toHaveBeenNthCalledWith(2, ['source-last'], undefined) + expect(dbChainMockFns.selectDistinct).toHaveBeenCalledWith({ + connectorId: document.connectorId, + }) + expect(dbChainMockFns.limit).toHaveBeenCalledWith(MAX_KNOWLEDGE_ACCESS_CANDIDATES) + expect(gt).toHaveBeenCalledWith(document.connectorId, first.at(-1)!.connectorId) + }) + + it('does not enumerate sources after a satisfied ordinary existence probe', async () => { + const resolve = vi.fn(async () => identity) + const provider: KnowledgeAccessProvider = { + get: async () => identity, + getForConnectors: resolve, + getForDocuments: async () => identity, + } + for await (const predicate of knowledgeReadAccessBatches(provider, [])) { + expect(predicate).toBeDefined() + break + } + expect(dbChainMockFns.selectDistinct).not.toHaveBeenCalled() + expect(resolve).not.toHaveBeenCalled() + }) + + it('honors cancellation before returning any predicate', async () => { + const controller = new AbortController() + controller.abort(new Error('cancelled')) + await expect( + knowledgeReadAccessBatches(identity, [], controller.signal).next() + ).rejects.toThrow('cancelled') + expect(dbChainMockFns.selectDistinct).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/read-access.ts b/apps/sim/lib/knowledge/read-access.ts new file mode 100644 index 00000000000..e1942b698a2 --- /dev/null +++ b/apps/sim/lib/knowledge/read-access.ts @@ -0,0 +1,65 @@ +import { db } from '@sim/db' +import { document, knowledgeBase, knowledgeConnector } from '@sim/db/schema' +import { and, asc, eq, gt, inArray, isNotNull, not, type SQL } from 'drizzle-orm' +import { + knowledgeAccessCondition, + knowledgeMetadataCandidateAccessCondition, +} from '@/lib/knowledge/access/predicate' +import { + type KnowledgeAccessProvider, + type KnowledgeAccessScope, + MAX_KNOWLEDGE_ACCESS_CANDIDATES, + type SystemAccessScope, +} from '@/lib/knowledge/access/types' + +export type KnowledgeReadAccess = KnowledgeAccessScope | SystemAccessScope | KnowledgeAccessProvider + +/** + * Streams disjoint, fully authorized document predicates for an existing reader's filters. + * Only connector IDs are selected before live proof; totals and metadata use the yielded + * full predicate. Paging avoids making unrelated sources a prerequisite for any one batch. + */ +export async function* knowledgeReadAccessBatches( + access: KnowledgeReadAccess, + conditions: readonly (SQL | undefined)[], + signal?: AbortSignal +): AsyncGenerator { + signal?.throwIfAborted() + const provider = 'get' in access ? access : undefined + const scope = 'get' in access ? await access.get() : access + const ordinary = knowledgeAccessCondition(scope) + yield ordinary + if (!provider || scope.kind !== 'user') return + + let cursor: string | undefined + while (true) { + signal?.throwIfAborted() + const rows = await db + .selectDistinct({ connectorId: document.connectorId }) + .from(document) + .innerJoin(knowledgeConnector, eq(document.connectorId, knowledgeConnector.id)) + .innerJoin(knowledgeBase, eq(document.knowledgeBaseId, knowledgeBase.id)) + .where( + and( + ...conditions, + knowledgeMetadataCandidateAccessCondition(scope), + not(ordinary), + isNotNull(document.connectorId), + cursor ? gt(document.connectorId, cursor) : undefined + ) + ) + .orderBy(asc(document.connectorId)) + .limit(MAX_KNOWLEDGE_ACCESS_CANDIDATES) + if (rows.length === 0) return + const connectorIds = rows.flatMap(({ connectorId }) => (connectorId ? [connectorId] : [])) + if (connectorIds.length === 0) return + const proof = await provider.getForConnectors(connectorIds, signal) + yield and( + not(ordinary), + inArray(document.connectorId, connectorIds), + knowledgeAccessCondition(proof) + )! + if (rows.length < MAX_KNOWLEDGE_ACCESS_CANDIDATES) return + cursor = connectorIds[connectorIds.length - 1] + } +} diff --git a/apps/sim/lib/knowledge/search/queries.test.ts b/apps/sim/lib/knowledge/search/queries.test.ts index 72c65866d91..e8edea4491d 100644 --- a/apps/sim/lib/knowledge/search/queries.test.ts +++ b/apps/sim/lib/knowledge/search/queries.test.ts @@ -9,7 +9,11 @@ import { schemaMock, } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { WORKSPACE_ACCESS_TOKENS } from '@/lib/knowledge/access/types' +import { + type KnowledgeAccessProvider, + type UserAccessScope, + WORKSPACE_ACCESS_TOKENS, +} from '@/lib/knowledge/access/types' import { buildTagFilterCondition } from '@/lib/knowledge/documents/tag-filter' import { executeKeywordSearch, @@ -470,3 +474,154 @@ describe('workspace search filters before ranking', () => { expectScopeOnEveryQuery() }) }) + +describe('live repository authorization follows ranked candidates', () => { + const identity: UserAccessScope = { + kind: 'user', + userId: 'reader', + tokens: ['org', 's:github-repositories:-:42'], + } + const allowed: UserAccessScope = { + ...identity, + githubInstallationGrants: [ + { + connectorId: 'allowed-source', + contentCredentialId: 'installation-credential', + readerCredentialId: 'reader-credential', + repositoryId: '101', + readerSubjectToken: 's:github-repositories:-:42', + }, + ], + } + const candidate = (id: string, connectorId: string) => ({ + id, + documentId: `doc-${id}`, + connectorId, + installationSource: true, + distance: 0.1, + }) + const getForConnectors = vi.fn() + const provider: KnowledgeAccessProvider = { + get: async () => identity, + getForConnectors, + getForDocuments: async () => allowed, + } + const params: SearchParams = { + knowledgeBaseIds: ['org-index'], + topK: 1, + access: identity, + accessProvider: provider, + queryVector: { vector: '[0.1,0.2]', dimensions: 1536 }, + distanceThreshold: 0.8, + structuredFilters: [{ tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'release' }], + } + + beforeEach(() => { + resetDbChainMock() + getForConnectors.mockReset().mockResolvedValue(allowed) + }) + + afterEach(() => vi.useRealTimers()) + + it.each(['vector', 'tag-vector', 'tags', 'keyword'] as const)( + '%s ranks identifiers before verification and loads content under the full predicate', + async (mode) => { + queueTableRows(schemaMock.embedding, [candidate('selected', 'allowed-source')]) + queueTableRows(schemaMock.embedding, [{ id: 'selected', content: 'verified result' }]) + const rows = + mode === 'vector' + ? await handleVectorOnlySearch(params) + : mode === 'tag-vector' + ? await handleTagAndVectorSearch(params) + : mode === 'tags' + ? await handleTagOnlySearch(params) + : await executeKeywordSearch({ + ...params, + query: 'release', + queryVector: params.queryVector!, + }) + expect(rows).toEqual([{ id: 'selected', content: 'verified result' }]) + expect(getForConnectors).toHaveBeenCalledWith(['allowed-source'], undefined) + expect(Object.keys(dbChainMockFns.select.mock.calls[0][0]).sort()).toEqual( + [ + 'id', + 'documentId', + 'connectorId', + 'installationSource', + ...(mode === 'keyword' ? ['keywordRank'] : mode === 'tags' ? [] : ['distance']), + ].sort() + ) + expect(dbChainMockFns.select.mock.invocationCallOrder[0]).toBeLessThan( + getForConnectors.mock.invocationCallOrder[0] + ) + expect(getForConnectors.mock.invocationCallOrder[0]).toBeLessThan( + dbChainMockFns.select.mock.invocationCallOrder[1] + ) + const fullPredicate = dbChainMockFns.where.mock.calls[1][0] + const serializedPredicate = JSON.stringify(fullPredicate) + expect(serializedPredicate).toContain('github_read_grant') + expect(serializedPredicate).toContain('allowed-source') + expect(serializedPredicate).toContain('reader-credential') + expect( + hasMockCondition( + fullPredicate, + (node) => + node.type === 'inArray' && + node.column === schemaMock.embedding.id && + Array.isArray(node.values) && + node.values.length === 1 && + node.values[0] === 'selected' + ) + ).toBe(true) + } + ) + + it('refills after a denied repository instead of letting its matches consume the result limit', async () => { + getForConnectors.mockResolvedValueOnce(identity) + queueTableRows(schemaMock.embedding, [candidate('denied', 'revoked-source')]) + queueTableRows(schemaMock.embedding, []) + queueTableRows(schemaMock.embedding, [candidate('selected', 'allowed-source')]) + queueTableRows(schemaMock.embedding, [{ id: 'selected', content: 'verified result' }]) + const rows = await handleTagOnlySearch(params) + expect(rows).toEqual([{ id: 'selected', content: 'verified result' }]) + expect(getForConnectors.mock.calls.map(([ids]) => ids)).toEqual([ + ['revoked-source'], + ['allowed-source'], + ]) + expect(dbChainMockFns.offset.mock.calls).toEqual([[0], [0]]) + const refillPredicate = JSON.stringify(dbChainMockFns.where.mock.calls[2][0]) + expect(refillPredicate).toContain('NOT') + expect(refillPredicate).toContain('revoked-source') + }) + + it('retains a completed authorized result when the next candidate page exhausts its deadline', async () => { + vi.useFakeTimers() + vi.setSystemTime(new Date(10000)) + getForConnectors.mockImplementation(async () => { + vi.setSystemTime(new Date(19000)) + return allowed + }) + queueTableRows(schemaMock.embedding, [ + candidate('selected', 'allowed-source'), + candidate('slow', 'slow-source'), + ]) + queueTableRows(schemaMock.embedding, [{ id: 'selected', content: 'verified result' }]) + expect(await handleTagOnlySearch({ ...params, topK: 2 })).toEqual([ + { id: 'selected', content: 'verified result' }, + ]) + expect(getForConnectors).toHaveBeenCalledOnce() + }) + + it('propagates caller cancellation before content hydration', async () => { + const cancellation = new AbortController() + getForConnectors.mockImplementation(async () => { + cancellation.abort(new Error('Search cancelled')) + return allowed + }) + queueTableRows(schemaMock.embedding, [candidate('selected', 'allowed-source')]) + await expect(handleTagOnlySearch({ ...params, signal: cancellation.signal })).rejects.toThrow( + 'Search cancelled' + ) + expect(dbChainMockFns.select).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/knowledge/search/queries.ts b/apps/sim/lib/knowledge/search/queries.ts index 715f9a8aaca..dbe5d64735b 100644 --- a/apps/sim/lib/knowledge/search/queries.ts +++ b/apps/sim/lib/knowledge/search/queries.ts @@ -3,8 +3,11 @@ import { document, embedding, knowledgeConnector } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { getErrorMessage, getPostgresErrorCode } from '@sim/utils/errors' import { and, eq, inArray, isNull, type SQL, sql } from 'drizzle-orm' -import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' -import type { KnowledgeAccessScope } from '@/lib/knowledge/access/types' +import { + knowledgeAccessCondition, + knowledgeMetadataCandidateAccessCondition, +} from '@/lib/knowledge/access/predicate' +import type { KnowledgeAccessProvider, KnowledgeAccessScope } from '@/lib/knowledge/access/types' import type { KbEmbeddingDimensions } from '@/lib/knowledge/embedding-models' import { workspaceSearchFilterConditions } from '@/lib/knowledge/search/filter-conditions' import type { WorkspaceSearchFilters } from '@/lib/knowledge/search/filters' @@ -79,13 +82,18 @@ export interface DocumentMetadata { */ export async function getDocumentMetadataByIds( documentIds: string[], - access: KnowledgeAccessScope + access: KnowledgeAccessScope, + accessProvider?: KnowledgeAccessProvider, + signal?: AbortSignal ): Promise> { if (documentIds.length === 0) { return {} } const uniqueIds = [...new Set(documentIds)] + const authorizedAccess = accessProvider + ? await accessProvider.getForDocuments(uniqueIds, signal) + : access const documents = await db .select({ id: document.id, @@ -102,7 +110,7 @@ export async function getDocumentMetadataByIds( eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), - knowledgeAccessCondition(access) + knowledgeAccessCondition(authorizedAccess) ) ) @@ -168,6 +176,8 @@ export interface SearchParams { topK: number /** What the caller may read; every leg applies it. Required so no leg can be written without it. */ access: KnowledgeAccessScope + accessProvider?: KnowledgeAccessProvider + signal?: AbortSignal structuredFilters?: StructuredFilter[] filters?: WorkspaceSearchFilters queryVector?: KnowledgeQueryVector @@ -397,7 +407,11 @@ const FTS_CONFIG = 'english' * overlaps the caller's tokens. Every leg spreads this helper rather than * listing the predicates itself, so no leg can drift from the others. */ -function getVisibilityConditions(access: KnowledgeAccessScope, filters?: WorkspaceSearchFilters) { +function getVisibilityConditions( + access: KnowledgeAccessScope, + filters?: WorkspaceSearchFilters, + accessCondition: SQL = knowledgeAccessCondition(access) +) { return [ eq(embedding.enabled, true), eq(document.enabled, true), @@ -405,11 +419,128 @@ function getVisibilityConditions(access: KnowledgeAccessScope, filters?: Workspa eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), - knowledgeAccessCondition(access), + accessCondition, ...workspaceSearchFilterConditions(filters), ] } +interface SearchReadCandidate { + id: string + documentId: string + connectorId: string | null + installationSource: boolean +} + +/** Only opaque identifiers leave candidate ranking; content stays behind the full read predicate. */ +const SEARCH_READ_CANDIDATE_FIELDS = { + id: embedding.id, + documentId: document.id, + connectorId: document.connectorId, + installationSource: sql`EXISTS ( + SELECT 1 FROM ${knowledgeConnector} + WHERE ${knowledgeConnector.id} = ${document.connectorId} + AND ${knowledgeConnector.connectorType} = 'github' + AND ${knowledgeConnector.sourceConfig}::jsonb ? 'githubRepositoryId' + )`, +} + +const LIVE_SEARCH_PAGE_SIZE = 200 +const LIVE_SEARCH_BUDGET_MS = 8000 + +/** + * Verification follows ranked candidates, never the organization's source order. Denied + * repositories are excluded on refill, so many matches from one revoked source cannot + * consume every result slot. The existing vector tuple budget also bounds candidate work. + */ +async function selectAuthorizedSearchResults(input: { + accessProvider: KnowledgeAccessProvider + signal?: AbortSignal + topK: number + selectPage: ( + limit: number, + offset: number, + excludedSources: readonly string[] + ) => Promise + hydrate: (ids: string[], access: KnowledgeAccessScope) => Promise +}): Promise { + const deadline = Date.now() + LIVE_SEARCH_BUDGET_MS + const pageSize = Math.min(LIVE_SEARCH_PAGE_SIZE, Math.max(input.topK, 20)) + const results = new Map() + const excludedSources = new Set() + let scanned = 0 + let offset = 0 + while ( + results.size < input.topK && + scanned < Number(HNSW_MAX_SCAN_TUPLES) && + Date.now() < deadline + ) { + input.signal?.throwIfAborted() + const candidates = await input.selectPage(pageSize, offset, [...excludedSources]) + if (!candidates.length) break + scanned += candidates.length + const connectorIds = [ + ...new Set( + candidates.flatMap((candidate) => (candidate.connectorId ? [candidate.connectorId] : [])) + ), + ] + const access = await input.accessProvider.getForConnectors(connectorIds, input.signal) + input.signal?.throwIfAborted() + const grantedSources = new Set( + access.kind === 'user' + ? (access.githubInstallationGrants?.map((grant) => grant.connectorId) ?? []) + : [] + ) + const excludedBefore = excludedSources.size + for (const candidate of candidates) { + if ( + candidate.installationSource && + candidate.connectorId && + !grantedSources.has(candidate.connectorId) + ) + excludedSources.add(candidate.connectorId) + } + const hydrated = await input.hydrate( + candidates.map((candidate) => candidate.id), + access + ) + const byId = new Map(hydrated.map((row) => [row.id, row])) + for (const candidate of candidates) { + const row = byId.get(candidate.id) + if (row) results.set(row.id, row) + if (results.size === input.topK) break + } + if (excludedSources.size > excludedBefore) offset = 0 + else { + offset += candidates.length + if (candidates.length < pageSize) break + } + } + input.signal?.throwIfAborted() + return [...results.values()] +} + +function excludeSearchSources(sourceIds: readonly string[]): SQL | undefined { + return sourceIds.length + ? sql`(${document.connectorId} IS NULL OR NOT (${inArray(document.connectorId, [...sourceIds])}))` + : undefined +} + +function hydrateSearchCandidates( + ids: string[], + access: KnowledgeAccessScope, + distance: SQL | SQL.Aliased, + filters: WorkspaceSearchFilters | undefined, + conditions: (SQL | undefined)[] +) { + return db + .select(getSearchResultFields(distance)) + .from(embedding) + .innerJoin(document, eq(embedding.documentId, document.id)) + .where( + and(inArray(embedding.id, ids), ...getVisibilityConditions(access, filters), ...conditions) + ) +} + /** Candidates each hybrid leg retrieves before the fused list is trimmed to `topK`. */ const HYBRID_CANDIDATE_MIN = 50 const HYBRID_CANDIDATE_MAX = 200 @@ -440,6 +571,45 @@ export async function handleTagOnlySearch(params: SearchParams): Promise + db + .select(SEARCH_READ_CANDIDATE_FIELDS) + .from(embedding) + .innerJoin(document, eq(embedding.documentId, document.id)) + .where( + and( + ...conditions, + ...getVisibilityConditions( + access, + params.filters, + knowledgeMetadataCandidateAccessCondition(access) + ), + excludeSearchSources(excludedSources) + ) + ) + .orderBy(embedding.id) + .limit(limit) + .offset(offset), + hydrate: (ids, authorized) => + hydrateSearchCandidates( + ids, + authorized, + sql`0`.as('distance'), + params.filters, + conditions + ), + }) + } + if (strategy.useParallel) { const parallelLimit = Math.ceil(topK / knowledgeBaseIds.length) + 5 @@ -486,6 +656,11 @@ export async function handleVectorOnlySearch(params: SearchParams): Promise selectRankedVectorResults( executor, @@ -520,6 +695,44 @@ export async function handleVectorOnlySearch(params: SearchParams): Promise a.distance - b.distance) } +/** The vector transaction ends after ranking, before any provider authorization request starts. */ +function selectLiveVectorResults( + params: SearchParams, + accessProvider: KnowledgeAccessProvider, + distance: SQL, + filters: (SQL | undefined)[] +): Promise { + const conditions = [inArray(embedding.knowledgeBaseId, params.knowledgeBaseIds), ...filters] + return selectAuthorizedSearchResults({ + accessProvider, + signal: params.signal, + topK: params.topK, + selectPage: (limit, offset, excludedSources) => + withVectorScanSettings((executor) => + executor + .select({ ...SEARCH_READ_CANDIDATE_FIELDS, distance: distance.as('distance') }) + .from(embedding) + .innerJoin(document, eq(embedding.documentId, document.id)) + .where( + and( + ...conditions, + ...getVisibilityConditions( + params.access, + params.filters, + knowledgeMetadataCandidateAccessCondition(params.access) + ), + excludeSearchSources(excludedSources) + ) + ) + .orderBy(distance, embedding.id) + .limit(limit) + .offset(offset) + ), + hydrate: (ids, authorized) => + hydrateSearchCandidates(ids, authorized, distance.as('distance'), params.filters, conditions), + }) +} + /** * Sort only chunk identities and distances before loading result content. Carrying * full chunk rows through the vector sort can spill to disk. The bounded subquery @@ -553,6 +766,8 @@ export interface KeywordSearchParams { knowledgeBaseIds: string[] topK: number access: KnowledgeAccessScope + accessProvider?: KnowledgeAccessProvider + signal?: AbortSignal query: string /** Query embedding, so keyword-only hits still carry a real cosine distance. */ queryVector: KnowledgeQueryVector @@ -570,11 +785,9 @@ export interface KeywordSearchParams { * leg there is no distance threshold — surfacing exact-token matches that are * semantically distant is the entire point of this leg. * - * Candidate gathering mirrors the vector leg's `getQueryStrategy`: across many - * knowledge bases a single global `LIMIT` lets whichever base ranks strongest - * lexically consume every slot, so an exact-token hit in a smaller base would - * never reach fusion. Both legs must draw candidates the same way, or rank - * fusion is combining rankings taken over differently-shaped pools. + * Candidate gathering mirrors the vector leg: resolved scopes use the same + * per-base strategy, and live user scopes verify bounded pages from the same + * global ranking pool before hydrating content. * * Ranking and hydration are two steps on purpose. Projecting the cosine * distance in the ranking query makes Postgres detoast the chunk's vector and @@ -597,6 +810,46 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise ? getStructuredTagFilters(structuredFilters, embedding) : [] + if (params.accessProvider && access.kind === 'user') { + const conditions = [ + inArray(embedding.knowledgeBaseId, knowledgeBaseIds), + sql`${embedding.contentTsv} @@ ${tsQuery}`, + ...tagFilterConditions, + ] + return selectAuthorizedSearchResults({ + accessProvider: params.accessProvider, + signal: params.signal, + topK, + selectPage: (limit, offset, excludedSources) => + db + .select({ ...SEARCH_READ_CANDIDATE_FIELDS, keywordRank: rankExpr.as('keyword_rank') }) + .from(embedding) + .innerJoin(document, eq(embedding.documentId, document.id)) + .where( + and( + ...conditions, + ...getVisibilityConditions( + access, + params.filters, + knowledgeMetadataCandidateAccessCondition(access) + ), + excludeSearchSources(excludedSources) + ) + ) + .orderBy(sql`${rankExpr} DESC`, embedding.id) + .limit(limit) + .offset(offset), + hydrate: (ids, authorized) => + hydrateSearchCandidates( + ids, + authorized, + embeddingDistance(queryVector.dimensions, queryVector.vector).as('distance'), + params.filters, + conditions + ), + }) + } + const rankConditions = (kbScope: SQL | undefined) => and( kbScope, @@ -743,6 +996,12 @@ export async function handleTagAndVectorSearch(params: SearchParams): Promise selectRankedVectorResults( executor, @@ -771,6 +1030,8 @@ export interface ExecuteKnowledgeSearchParams { topK: number /** What the caller may read; resolved from the principal by the use case, never from input. */ access: KnowledgeAccessScope + accessProvider?: KnowledgeAccessProvider + signal?: AbortSignal searchMode: KnowledgeSearchMode /** Lets a recently modified document edge past a stale one of similar relevance; off by default. */ boostRecency?: boolean @@ -812,6 +1073,8 @@ export async function executeKnowledgeSearch( topK, structuredFilters, access, + accessProvider: params.accessProvider, + signal: params.signal, filters: params.filters, }) } @@ -836,6 +1099,8 @@ export async function executeKnowledgeSearch( queryVector, distanceThreshold, access, + accessProvider: params.accessProvider, + signal: params.signal, filters: params.filters, }) : handleVectorOnlySearch({ @@ -844,6 +1109,8 @@ export async function executeKnowledgeSearch( queryVector, distanceThreshold, access, + accessProvider: params.accessProvider, + signal: params.signal, filters: params.filters, }) @@ -863,6 +1130,8 @@ export async function executeKnowledgeSearch( queryVector, structuredFilters, access, + accessProvider: params.accessProvider, + signal: params.signal, filters: params.filters, }).catch((error) => { logger.warn('Keyword search leg failed; falling back to vector-only results', { diff --git a/apps/sim/lib/knowledge/tags/service.ts b/apps/sim/lib/knowledge/tags/service.ts index 6d1d112cf60..8f837de9ccb 100644 --- a/apps/sim/lib/knowledge/tags/service.ts +++ b/apps/sim/lib/knowledge/tags/service.ts @@ -12,13 +12,12 @@ import { generateId } from '@sim/utils/id' import { and, eq, isNotNull, isNull, or, sql } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' import type { DbOrTx, DbTransaction } from '@/lib/db/types' -import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' -import type { KnowledgeAccessScope } from '@/lib/knowledge/access/types' import { getSlotsForFieldType, isValidSlotForFieldType, SUPPORTED_FIELD_TYPES, } from '@/lib/knowledge/constants' +import { type KnowledgeReadAccess, knowledgeReadAccessBatches } from '@/lib/knowledge/read-access' import type { BulkTagDefinitionsData, DocumentTagDefinition } from '@/lib/knowledge/tags/types' import type { CreateTagDefinitionData, @@ -806,7 +805,7 @@ export async function updateTagDefinition( export async function getTagUsage( knowledgeBaseId: string, requestId: string, - access: KnowledgeAccessScope + access: KnowledgeReadAccess ): Promise< Array<{ tagName: string @@ -832,7 +831,6 @@ export async function getTagUsage( eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), - knowledgeAccessCondition(access), isNotNull(sql`${sql.raw(tagSlot)}`), ] @@ -840,14 +838,19 @@ export async function getTagUsage( whereConditions.push(sql`${sql.raw(tagSlot)} != ''`) } - const documentsWithTag = await db - .select({ - id: document.id, - filename: document.filename, - tagValue: sql`${sql.raw(tagSlot)}::text`, - }) - .from(document) - .where(and(...whereConditions)) + const documentsWithTag: { id: string; filename: string; tagValue: string }[] = [] + for await (const accessCondition of knowledgeReadAccessBatches(access, whereConditions)) { + documentsWithTag.push( + ...(await db + .select({ + id: document.id, + filename: document.filename, + tagValue: sql`${sql.raw(tagSlot)}::text`, + }) + .from(document) + .where(and(...whereConditions, accessCondition))) + ) + } usage.push({ tagName: def.displayName, @@ -871,7 +874,7 @@ export async function getTagUsage( */ export async function getTagUsageStats( knowledgeBaseId: string, - access: KnowledgeAccessScope, + access: KnowledgeReadAccess, requestId: string ): Promise< Array<{ @@ -890,42 +893,54 @@ export async function getTagUsageStats( const tagSlot = def.tagSlot validateTagSlot(tagSlot) - const docCountResult = await db - .select({ count: sql`count(*)` }) - .from(document) - .where( - and( - eq(document.knowledgeBaseId, knowledgeBaseId), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - knowledgeAccessCondition(access), - sql`${sql.raw(tagSlot)} IS NOT NULL` + const conditions = [ + eq(document.knowledgeBaseId, knowledgeBaseId), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt), + ] + let documentCount = 0 + let chunkCount = 0 + for await (const accessCondition of knowledgeReadAccessBatches(access, conditions)) { + const docCountResult = await db + .select({ count: sql`count(*)` }) + .from(document) + .where( + and( + eq(document.knowledgeBaseId, knowledgeBaseId), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt), + accessCondition, + sql`${sql.raw(tagSlot)} IS NOT NULL` + ) ) - ) - const chunkCountResult = await db - .select({ count: sql`count(*)` }) - .from(embedding) - .innerJoin(document, eq(embedding.documentId, document.id)) - .where( - and( - eq(embedding.knowledgeBaseId, knowledgeBaseId), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - knowledgeAccessCondition(access), - sql`${sql.raw(`embedding.${tagSlot}`)} IS NOT NULL` + const chunkCountResult = await db + .select({ count: sql`count(*)` }) + .from(embedding) + .innerJoin(document, eq(embedding.documentId, document.id)) + .where( + and( + eq(embedding.knowledgeBaseId, knowledgeBaseId), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt), + accessCondition, + sql`${sql.raw(`embedding.${tagSlot}`)} IS NOT NULL` + ) ) - ) + documentCount += Number(docCountResult[0]?.count || 0) + chunkCount += Number(chunkCountResult[0]?.count || 0) + } stats.push({ id: def.id, tagSlot: def.tagSlot, displayName: def.displayName, fieldType: def.fieldType, - documentCount: Number(docCountResult[0]?.count || 0), - chunkCount: Number(chunkCountResult[0]?.count || 0), + documentCount, + chunkCount, }) } From 035f4329151156bec7cc32ac7f6b174612812f95 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Wed, 9 Sep 2026 16:40:46 -0700 Subject: [PATCH 3/3] fix(search): clarify public setup documentation --- .../self-hosting/integrations-oauth.mdx | 96 ++++++++++++++- apps/docs/content/docs/search/github.mdx | 113 ++---------------- .../[connectorType]/provider-detail.test.tsx | 1 + 3 files changed, 99 insertions(+), 111 deletions(-) diff --git a/apps/docs/content/docs/platform/self-hosting/integrations-oauth.mdx b/apps/docs/content/docs/platform/self-hosting/integrations-oauth.mdx index 6842f52e618..acc1fa6c884 100644 --- a/apps/docs/content/docs/platform/self-hosting/integrations-oauth.mdx +++ b/apps/docs/content/docs/platform/self-hosting/integrations-oauth.mdx @@ -6,6 +6,7 @@ description: Register OAuth apps so your users can connect Slack, Google, Jira, import { Callout } from 'fumadocs-ui/components/callout' import { Step, Steps } from 'fumadocs-ui/components/steps' import { FAQ } from '@/components/ui/faq' +import { Image } from '@/components/ui/image' **OAuth integrations need your own provider application on a self-hosted deployment.** Configure the OAuth services your team uses; API-key integrations can instead use keys supplied in their blocks. Users will see the connector in the UI, click "Connect", and get an error from the provider until the corresponding `*_CLIENT_ID` and `*_CLIENT_SECRET` are set. @@ -116,15 +117,100 @@ The same variables also power "Sign in with Microsoft". ### GitHub Search -Register a [GitHub App](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/registering-a-github-app) with repository **Contents: read-only**, **Metadata: read-only**, and account **Email addresses: read-only** permissions. Keep user access token expiration enabled so Sim can rotate access and refresh tokens. +Self-hosted GitHub Search uses a GitHub App for account connections and organization installation indexing. Register your own App and configure the server variables below. Sim Cloud users use the [GitHub Search setup flow](/search/github#add-a-repository) directly. -| Environment variables | Provider ID | + + + +#### Register the App + +For a team, open **Your organizations → Settings** for the organization that will own the App. For a personal App, open your account's **Settings**. Then choose **Developer settings → GitHub Apps → New GitHub App**. + +Give the App a unique, recognizable name, such as **Your Company Sim Search**, and set **Homepage URL** to your Sim URL. + +Under **Identifying and authorizing users → Redirect URI (callback URL)**, enter: + +```text +/api/auth/oauth2/callback/github-repositories +``` + +Replace `` with your configured public origin, such as `https://sim.example.com`, without a trailing slash. The scheme, hostname, port, and path must match exactly; `www` and non-`www` hosts are different. See GitHub's [callback matching rules](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/about-the-user-authorization-callback-url). + +| GitHub setting | Value for Sim Search | |---|---| -| `GITHUB_APP_CLIENT_ID`
`GITHUB_APP_CLIENT_SECRET` | `github-repositories` | +| Allow wildcard matching | Disabled | +| Expire user authorization tokens | Enabled | +| Request user authorization (OAuth) during installation | Disabled | +| Enable Device Flow | Disabled | +| Post installation → Setup URL | Empty | +| Webhook → Active | Disabled | + +Authorization starts from Sim so the callback can finish the pending connection. The connector polls GitHub's API and does not need a webhook. + +GitHub App registration with the Redirect URI, expiring tokens enabled, and installation authorization, Device Flow, and webhooks disabled + +*Example registration. Replace `sim.example.com` with your Sim domain.* + + + + +#### Set read permissions + +Expand **Permissions → Repository permissions**. Set **Contents → Access: Read-only**; leave the mandatory **Metadata** permission at **Read-only**. + +GitHub repository permissions with Contents set to Read-only and Metadata shown as mandatory Read-only + +Expand **Account permissions** and set **Email addresses → Access: Read-only**. + +GitHub account permissions with only Email addresses selected for Read-only access + +| Permission area | Permission | Access | +|---|---|---| +| Repository | Contents | Read-only | +| Repository | Metadata | Read-only | +| Account | Email addresses | Read-only | + +Leave every other permission at **No access**. Sim does not need issue, pull-request, administration, or write permissions. GitHub's [registration guide](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/registering-a-github-app) explains these settings. + +GitHub App user tokens use these permissions rather than OAuth scopes. An empty `scope` value in the token response is expected; see GitHub's [user token reference](https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/generating-a-user-access-token-for-a-github-app). + +Set **Where can this GitHub App be installed? → Any account** to support connections from accounts outside the App owner. This lets any GitHub account install and authorize the App, subject to that account's organization policies. Making the App public does not make repositories public or grant anyone Search access. See GitHub's [App visibility rules](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/making-a-github-app-public-or-private). + +Select **Create GitHub App**. + + + + +#### Configure Sim + +On the App's **General** settings page, copy its numeric **App ID** and **Client ID**, select **Generate a new client secret**, and generate a **Private key**. The App slug is the final part of its public URL: `https://github.com/apps/`. + +Set all five variables using values from your GitHub App, then restart Sim: + +```text +GITHUB_APP_ID= +GITHUB_APP_SLUG= +GITHUB_APP_CLIENT_ID= +GITHUB_APP_CLIENT_SECRET= +GITHUB_APP_PRIVATE_KEY= +``` + +The private key must include its PEM header, footer, and contents. Sim accepts actual newlines or escaped `\n` sequences. Keep the private key and client secret in the deployment's server configuration; organization admins select installations in Sim without entering these secrets. + +The **Client ID** is different from the numeric **App ID**. Use credentials from **Developer settings → GitHub Apps**. `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` belong to the separate GitHub sign-in integration and remain unchanged. Search does not read `GITHUB_REPO_CLIENT_ID` or `GITHUB_REPO_CLIENT_SECRET`. + +Keep **Expire user authorization tokens** enabled so Sim receives the refresh token it needs to [renew personal connections](https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/refreshing-user-access-tokens). + +Complete the installation through [the GitHub Search source setup](/search/github#add-a-repository). + +If you replace a deployment's GitHub App, an organization admin must first open **Settings → Connected accounts → Providers → Update configurations**. This applies the deployment's current App configuration to the existing providers while preserving their saved identities. Accounts whose App configuration changed must reconnect. Then reconnect personal GitHub accounts and connect an installation of the new App. Reconnecting alone cannot update the organization's saved App configuration. + + + -Register `https:///api/auth/oauth2/callback/github-repositories` as the callback. These App OAuth client credentials are separate from `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` used for Sim sign-in. Sim does not require an App private key. +If GitHub rejects `redirect_uri`, compare the App's registered callback with `NEXT_PUBLIC_APP_URL` followed by `/api/auth/oauth2/callback/github-repositories`. Keep wildcard matching disabled. If installation indexing is unavailable, confirm all five `GITHUB_APP_*` variables belong to the same App and include a complete RSA private key. -A repository or organization administrator installs the App on the repositories to search. Each member connects their own GitHub account, with a verified email matching their Sim account. Search indexes repository files that both the member and the installed App can access. GitHub workflow blocks and existing knowledge-base token connections continue to use personal access tokens. +GitHub workflow blocks and knowledge-base token connections continue to use personal access tokens. ### Everything else diff --git a/apps/docs/content/docs/search/github.mdx b/apps/docs/content/docs/search/github.mdx index 7a390dcb017..023068d0962 100644 --- a/apps/docs/content/docs/search/github.mdx +++ b/apps/docs/content/docs/search/github.mdx @@ -5,7 +5,6 @@ description: Index repository files with a GitHub App while preserving each pers import { Callout } from 'fumadocs-ui/components/callout' import { Step, Steps } from 'fumadocs-ui/components/steps' -import { Image } from '@/components/ui/image' GitHub Search indexes text files from repositories on `github.com`. An organization admin can install the GitHub App once and use it to index selected repositories. Each person connects their own GitHub account once to search the repositories they can access. Installing the App does not connect teammates or give them the installer's permissions. @@ -13,106 +12,11 @@ Admin setup uses your organization's **Settings → Sources** page. Teammates co ## Before you start -Your Sim deployment needs a GitHub App configured as described below. The repository must contain at least one commit. Each person needs a verified GitHub email address matching their Sim account; the address can be private or secondary. +The repository must contain at least one commit. Each person needs a verified GitHub email address matching their Sim account; the address can be private or secondary. -To connect an installation for central indexing, you must be a Sim organization admin and either own the GitHub personal account or be an owner of the GitHub organization where the App is installed. You must also be able to read the repository you add. - -## Configure the GitHub App - -This step belongs to the Sim deployment administrator. If the App is already configured, continue to [Add a repository](#add-a-repository). - -Create separate GitHub Apps for production and staging. Each deployment uses its own App, credentials, private key, and callback URL. This keeps test installations and authorizations separate from production. - - - - -### Register the App - -For a team, open **Your organizations → Settings** for the organization that will own the App. For a personal App, open your account's **Settings**. Then choose **Developer settings → GitHub Apps → New GitHub App**. - -Give the App a unique, recognizable name, such as **Your Company Sim Search**, and set **Homepage URL** to your Sim URL. - -Under **Identifying and authorizing users → Redirect URI (callback URL)**, enter: - -```text -/api/auth/oauth2/callback/github-repositories -``` - -Replace `` with that deployment's configured public origin, without a trailing slash. For example, `https://sim.example.com` and `https://staging.sim.example.com` need different callbacks on their respective Apps. The scheme, hostname, port, and path must match exactly; `www` and non-`www` hosts are different. See GitHub's [callback matching rules](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/about-the-user-authorization-callback-url). - -| GitHub setting | Value for Sim Search | -|---|---| -| Allow wildcard matching | Disabled | -| Expire user authorization tokens | Enabled | -| Request user authorization (OAuth) during installation | Disabled | -| Enable Device Flow | Disabled | -| Post installation → Setup URL | Empty | -| Webhook → Active | Disabled | - -Authorization starts from Sim so the callback can finish the pending connection. The connector polls GitHub's API and does not need a webhook. - -GitHub App registration with the Redirect URI, expiring tokens enabled, and installation authorization, Device Flow, and webhooks disabled - -*Example registration. Replace `sim.example.com` with your Sim domain.* - - - - -### Set read permissions - -Expand **Permissions → Repository permissions**. Set **Contents → Access: Read-only**; leave the mandatory **Metadata** permission at **Read-only**. - -GitHub repository permissions with Contents set to Read-only and Metadata shown as mandatory Read-only +On Sim Cloud, connect through the Sim Search GitHub App in the setup flow below. If you self-host Sim, a deployment administrator must first [configure GitHub Search](/platform/self-hosting/integrations-oauth#github-search). -Expand **Account permissions** and set **Email addresses → Access: Read-only**. - -GitHub account permissions with only Email addresses selected for Read-only access - -| Permission area | Permission | Access | -|---|---|---| -| Repository | Contents | Read-only | -| Repository | Metadata | Read-only | -| Account | Email addresses | Read-only | - -Leave every other permission at **No access**. Sim does not need issue, pull-request, administration, or write permissions. GitHub's [registration guide](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/registering-a-github-app) explains these settings. - -GitHub App user tokens use these permissions rather than OAuth scopes. An empty `scope` value in the token response is expected; see GitHub's [user token reference](https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/generating-a-user-access-token-for-a-github-app). - -For production, set **Where can this GitHub App be installed? → Any account**. This lets any GitHub account install and authorize the App, subject to that account's organization policies. Making the App public does not make repositories public or grant anyone Search access. See GitHub's [App visibility rules](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/making-a-github-app-public-or-private). - -Use **Any account** for staging too if testing with accounts outside the App owner's organization. **Only on this account** restricts installations to the owner and authorization to that organization's members; a private personal App only works for its owner. - -Select **Create GitHub App**. - - - - -### Configure Sim - -On the App's **General** settings page, copy its numeric **App ID** and **Client ID**, select **Generate a new client secret**, and generate a **Private key**. The App slug is the final part of its public URL: `https://github.com/apps/`. - -Set all five variables for each deployment, using values from that deployment's App, then restart Sim: - -```text -GITHUB_APP_ID= -GITHUB_APP_SLUG= -GITHUB_APP_CLIENT_ID= -GITHUB_APP_CLIENT_SECRET= -GITHUB_APP_PRIVATE_KEY= -``` - -The private key must include its PEM header, footer, and contents. Sim accepts actual newlines or escaped `\n` sequences. Keep the private key and client secret in the deployment's server configuration; organization admins select installations in Sim without entering these secrets. - -The **Client ID** is different from the numeric **App ID**. Use credentials from **Developer settings → GitHub Apps**. `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` belong to the separate GitHub sign-in integration and remain unchanged. Search does not read `GITHUB_REPO_CLIENT_ID` or `GITHUB_REPO_CLIENT_SECRET`. - -Keep **Expire user authorization tokens** enabled so Sim receives the refresh token it needs to [renew personal connections](https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/refreshing-user-access-tokens). - -Complete the installation from the Sim source setup below. - -If you replace a deployment's GitHub App, an organization admin must first open **Settings → Connected accounts → Providers → Update configurations**. This applies the deployment's current App configuration to the existing providers while preserving their saved identities. Accounts whose App configuration changed must reconnect. Then reconnect personal GitHub accounts and connect an installation of the new App. Reconnecting alone cannot update the organization's saved App configuration. - - - +To connect an installation for central indexing, you must be a Sim organization admin and either own the GitHub personal account or be an owner of the GitHub organization where the App is installed. You must also be able to read the repository you add. ## Add a repository @@ -184,18 +88,15 @@ This is an installation plus personal authorization flow. GitHub Search does not | Problem | Next step | |---|---| -| GitHub is unavailable in Search | Ask the deployment admin to configure the App client credentials and enable member connections. | -| GitHub App indexing is unavailable | Configure all five `GITHUB_APP_*` variables above. Confirm the App ID, slug, client credentials, and valid RSA PEM key all belong to the same App. | -| GitHub rejects `redirect_uri` | Register the exact callback on the GitHub App whose Client ID Sim uses: `http://localhost:3000/api/auth/oauth2/callback/github-repositories` for the default local server, or your production Sim origin followed by `/api/auth/oauth2/callback/github-repositories`. The scheme, host, port, and path must match; keep wildcard matching disabled. See [callback matching](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/about-the-user-authorization-callback-url). | -| Wrong app credentials | Copy the Client ID and client secret from **GitHub Apps**, not **OAuth Apps**. Set `GITHUB_APP_CLIENT_ID` and `GITHUB_APP_CLIENT_SECRET`, then restart Sim. A numeric App ID or private-key file cannot replace them. | +| GitHub is unavailable in Search | Ask your Sim organization admin to enable GitHub under **Settings → Sources**. For self-hosted Sim, also check the [GitHub App configuration](/platform/self-hosting/integrations-oauth#github-search). | | Repository cannot be read | Confirm the App is installed on that repository and your GitHub account has access. For SAML organizations, establish your GitHub SSO session before reconnecting. | -| A teammate cannot authorize the App | Set the production App to **Any account**. For a restricted staging App, confirm the tester meets its visibility rules. Check the GitHub organization's App policies and required approvals. | +| A teammate cannot authorize the App | Check the GitHub organization's App policies and required approvals. Self-hosted deployments must allow the teammate's account in their App visibility settings. | | No eligible installations found | Finish connecting your own GitHub account, install the configured App on your account or an organization you own, then select **Refresh**. An installation of a different App or one you only have repository access to cannot be selected. | | Repository is not accepted for an installation | Check `owner/repo`, the installation's account and repository selection, and your own access. Update the source's Repository field after a rename. After a transfer, add a source using an installation for the new owner. | | Identity verification fails | Verify the email used by your Sim account in GitHub's email settings, then reconnect. A public profile email alone is insufficient. | | Authorization fails after installation | Return to Sim and start **Connect account** there. Do not enable authorization during installation. | -| Account authorization did not complete | Start the connection again from Sim. If it repeats, verify the exact callback, matching GitHub App client credentials, **Email addresses: Read-only**, and enabled expiring user tokens. The deployment admin can inspect server logs for the underlying failure. | -| Update GitHub in Connected accounts before connecting this source | An organization admin must select **Settings → Connected accounts → Providers → Update configurations**, then reconnect GitHub. This is required when the deployment's GitHub App has changed. | +| Account authorization did not complete | Start the connection again from Sim. If it repeats, contact your organization admin or Sim support. For self-hosted Sim, check the [App callback and credentials](/platform/self-hosting/integrations-oauth#github-search). | +| Update GitHub in Connected accounts before connecting this source | An organization admin must select **Settings → Connected accounts → Providers → Update configurations**, then reconnect GitHub. | | Indexed files no longer appear | Confirm your own repository access, App repository selection, and connection status. Installation-indexed content is also withheld when GitHub cannot verify current access; retry once GitHub is available. | | Sync is incomplete | Review the source status. Very large Git trees, file size limits, and unreadable files can limit indexing. | | Empty repository returns an error | Add an initial commit, then sync again. GitHub does not return a file tree for an uninitialized repository. | diff --git a/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.test.tsx b/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.test.tsx index dd9ad344035..3873cc1d046 100644 --- a/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.test.tsx +++ b/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.test.tsx @@ -506,6 +506,7 @@ describe('organization provider management', () => { mocks.accounts.mockReturnValue({ data: { credentialGroup: null }, isPending: false }) await render('slack') await click('Set up Slack app') + await vi.waitFor(() => expect(mocks.updateUrl).toHaveBeenCalled()) const query = new URLSearchParams(mocks.updateUrl.mock.calls.at(-1)![0].queryString) expect(query.get('connectedAccounts')).toBe('slack') expect(query.has('addConnector')).toBe(false)