diff --git a/apps/docs/content/docs/search/connect-your-account.mdx b/apps/docs/content/docs/search/connect-your-account.mdx index 25e02ec92fe..7b031e1cf82 100644 --- a/apps/docs/content/docs/search/connect-your-account.mdx +++ b/apps/docs/content/docs/search/connect-your-account.mdx @@ -7,7 +7,7 @@ import { Callout } from 'fumadocs-ui/components/callout' import { Step, Steps } from 'fumadocs-ui/components/steps' import { Image } from '@/components/ui/image' -Your admin allows the integration and can configure shared source filters. You connect your own account so Sim can establish what you are allowed to search. Admins connect their own accounts through the same flow. +Your admin allows the integration and configures its connection method. For member sources, connect your own account so Sim can establish what you are allowed to search. Admins use the same flow. For central Google sources, join Sim with your matching verified Google Workspace email; no personal Google connection is needed. @@ -16,6 +16,8 @@ Your admin allows the integration and can configure shared source filters. You c Accept your Sim organization invitation or sign in through your organization's SSO. Use a verified Sim email that matches your account at the source. Organization Search does not require workspace access. +For a central Google source, use your primary Workspace email and open **Search** or **Home** after joining. The authorization steps below apply to sources that require a personal connection. + @@ -55,10 +57,10 @@ For a source configured inside a workspace, join that workspace and connect thro | Member accounts | Connect your own account, including when you are the admin. | | GitHub App installation | Connect GitHub once for this Sim organization. The App handles indexing; your account establishes which repositories you may search. | | Confluence service account | Connect Confluence to verify your identity; the service account handles the crawl. | -| Google Drive delegated service account | No personal connection is needed for that source. Your verified Sim email is matched to Drive permissions. | +| Google Workspace service account (Gmail, Calendar, Drive) | No personal connection is needed for that source. Your verified Sim email identifies your mailbox and calendar view, or is matched to Drive permissions. | | GitLab instance administrator | No personal connection is needed. Your verified Sim email must match the confirmed primary GitLab email. | -Connecting one Google service does not connect all of them. Gmail, Calendar, and Drive each have their own Search connection. +Gmail, Calendar, and Drive are separate Search sources. Connecting one Google service does not connect all of them. A central source can be searchable without a personal connection row in **Integrations**. ## If you received a connection request @@ -72,7 +74,7 @@ An account connection request does not invite you into the Sim organization. You On the main **Integrations** page, use **Reconnect** beside an expired connection to renew it. To withdraw an account, open its row's actions menu and select **Disconnect**, then confirm. If several accounts are connected, choose the account to disconnect. Disconnecting stops that account from being used for organization indexing and workflows, and removes Search access that depends on it. -Admins manage setup from **Settings → Sources**. Select **Manage** beside the integration; Google providers have **Accounts** and **Advanced** tabs, while other providers with personal connections have **Sources** and **Accounts**. This does not grant the admin access to every document. +Admins manage setup from **Settings → Sources**. Select **Manage** beside the integration. Providers with personal connections have **Accounts** and a source list under **Advanced** (Google) or **Sources**. Without personal connections, the source list opens directly. This does not grant the admin access to every document. ## If you get stuck @@ -85,7 +87,7 @@ Admins manage setup from **Settings → Sources**. Select **Manage** beside the | **Verify email** | Verify your Sim email to return to the connection page. Reopen the original link if you are not redirected. | | Expired or cancelled authorization | Return to the original connection page and start again. If the invitation itself expired, ask the admin for a new request. | | Access revoked | Ask the organization admin to restore your account contribution access before reconnecting. | -| Needs admin attention | Ask your admin to open **Settings → Sources**, select **Manage** beside the integration, and open the source from **Advanced** (Google providers) or **Sources** to inspect its error. | +| Needs admin attention | Ask your admin to open **Settings → Sources**, select **Manage** beside the integration, and open its source or sync configuration to inspect the error. | Your Sim role does not override document access at the source. Connecting a different account or receiving a Search link does not share someone else's mailbox, private calendar, or restricted documents with you. diff --git a/apps/docs/content/docs/search/gmail.mdx b/apps/docs/content/docs/search/gmail.mdx index 6eeb579613e..8945c941261 100644 --- a/apps/docs/content/docs/search/gmail.mdx +++ b/apps/docs/content/docs/search/gmail.mdx @@ -1,24 +1,35 @@ --- title: Gmail -description: Connect each teammate's Gmail account to search their email in Sim +description: Connect personal Gmail accounts or index Workspace mailboxes with a delegated service account --- import { Callout } from 'fumadocs-ui/components/callout' import { Step, Steps } from 'fumadocs-ui/components/steps' import { Image } from '@/components/ui/image' -Search email threads from your own Gmail account. An organization admin enables the source; each teammate connects their own account. An admin's connection does not make their mailbox available to the team. +Search email threads from your Gmail account. Members can connect their own accounts, or an administrator can index Google Workspace mailboxes with a service account. In either case, each mailbox stays private to its owner. Admin setup uses your organization's **Settings → Sources** page. Teammates connect from **Integrations** in the main sidebar. For workspace Search, use **Search → Add source** instead; **Create & Invite** is the workspace equivalent of **Add source**. -## Set up Gmail +## Choose your setup + +| Method | Use it when | What teammates do | +| --- | --- | --- | +| **Member accounts** | Each person should authorize their own Gmail account. | Connect the Google account matching their verified Sim email. | +| **Service account** | A Google Workspace administrator can authorize a central mailbox crawl. | Join the Sim organization with their matching verified primary Workspace email; no personal Gmail connection is needed. | + +These are alternative setup paths. When only a central Gmail source is configured, Integrations does not offer a personal Gmail **Connect** action. Existing member-account sources keep their connection actions. + +Central indexing does not make email searchable by the administrator, other recipients, or the rest of the organization. Mailbox delegation and shared-mailbox access are not mirrored; only the mailbox owner's verified email grants Search access. + +## Connect member accounts ### Allow Gmail -Open **Settings → Sources** and turn on **Gmail**. Gmail uses member accounts; there is no domain-wide or service-account crawl in Search. +Open **Settings → Sources** and turn on **Gmail**. This allows personal connections; it does not connect anyone's account. @@ -34,35 +45,85 @@ Open **Integrations** and select **Connect** beside Gmail. Authorize the Google An admin selects **Manage** beside Gmail in **Settings → Sources**, opens **Advanced**, then selects the configuration's **Settings** tab. Change **Labels**, **Date Range**, or other filters and save. -To create a separate configuration, use **Add sync configuration** on **Advanced**. Its form shows **Labels** and **Date Range** first; **More options** contains category exclusions, **Search Filter**, and **Metadata tags**. Select **Add source** to save. This does not connect accounts or invite people. - -One configuration is usually enough. Adding another creates a separate source; editing **Settings** updates the selected one. Every configuration applies to all active Gmail connections, including accounts connected later. It does not assign different filters to selected people or let teammates search each other's mail. +One member-account configuration is usually enough. Its filters apply to all active Gmail connections, including accounts connected later. It does not assign different filters to selected people or let teammates search each other's mail. **Add sync configuration** opens the [central service-account setup](#set-up-a-central-service-account); it does not edit this member configuration. Configurations are additive: a narrower one does not restrict an existing broader one, and overlapping configurations can index the same thread more than once. For one organization-wide policy, edit the existing configuration. -Gmail Search source configuration +## Set up a central service account + +Open **Settings → Sources**, enable **Gmail**, and select **Manage → Advanced → Add sync configuration**. If personal connections are disabled for your organization, select **Add source** from the provider page instead. + +This requires Google Workspace and a Workspace super administrator to authorize delegation. Consumer Gmail accounts cannot use this path. Each selected user must have Gmail enabled. + + + + +### Create the Google service account + +In [Google Cloud Console](https://console.cloud.google.com/), select your project and enable **Gmail API** and **Admin SDK API** under **APIs & Services → Library**. Open **IAM & Admin → Service Accounts → Create service account**, name it, and finish creation. Google Cloud project roles do not grant access to Workspace mailboxes and are not required for this crawl. + +Open its **Keys** tab and select **Add key → Create new key → JSON → Create**. Store the downloaded key securely. See [Google's key creation guide](https://docs.cloud.google.com/iam/docs/keys-create-delete#creating). + + + + +### Authorize domain-wide delegation + +Open the service account's **Details → Advanced settings** and copy its numeric **Client ID**. In the [Workspace Admin Console](https://admin.google.com/ac/owl/domainwidedelegation), sign in as a super administrator and open **Security → Access and data control → API controls → Manage Domain Wide Delegation → Add new**. + +Enter the Client ID and these exact scopes, separated by a comma: + +```text +https://www.googleapis.com/auth/gmail.readonly,https://www.googleapis.com/auth/admin.directory.user.readonly +``` + +Select **Authorize**, then **View details** to confirm both scopes were saved. If the same client also indexes Drive or Calendar, retain those services' required scopes. These Gmail crawl scopes do not allow sending or modifying mail. + +If your organization requires multi-party approval, another super administrator must approve the request. Delegation can take up to 24 hours to propagate. See Google's [delegation guide](https://knowledge.workspace.google.com/admin/apps/control-api-access-with-domain-wide-delegation). + + + + +### Configure Gmail in Sim + +Under **Indexing account**, select **Add service account** and paste the JSON key into **Add Google Service Account**, or select an existing service account. Set **Directory administrator email** to an active Workspace administrator who can read users in the Directory API. This account supplies directory access; each mailbox is read using that mailbox owner's delegated identity. + +Keep the default **Date Range** of **Last 6 months**, or adjust it and **Labels**. Use label names or system IDs such as `INBOX`; custom `Label_…` IDs are mailbox-specific. A label that does not exist in one mailbox simply matches no threads there. + +Under **More options → Users**, enter up to 100 primary Workspace email addresses, or leave blank for all active users in the same Workspace customer, including secondary domains. Suspended, archived, and guest users are excluded. Select **Connect & Sync**. Sim validates the directory administrator and selected users, then checks Gmail access for a sample user before saving. + +Teammates join the Sim organization with their matching verified primary email. They do not need to connect personal Google accounts for this source. + + + ## Connect your account +These steps apply to **Member accounts**. A central service-account source does not require a personal Gmail connection. + 1. Join the Sim organization and verify your Sim email address. Open **Integrations** and click **Connect** beside Gmail. 2. Complete the connection in the tab that opens. Choose the Google account whose verified email matches your Sim email, and grant the requested permissions. 3. Return to Integrations. The source shows its indexing status and the number of documents you can search. -Teammates follow these same steps after joining the organization. Once an admin approves Gmail, the first connection can create its source with default filters. Admins can configure shared filters beforehand or request connections from **Manage → Accounts → Request connections**. A connection request does not invite the recipient to the Sim organization. +Teammates follow these same steps after joining the organization. Once an admin allows Gmail, the first connection can create its source with default filters. Admins can edit those filters afterward or request connections from **Manage → Accounts → Request connections**. A connection request does not invite the recipient to the Sim organization. ## Source options -An admin opens **Settings → Sources**, selects **Manage** beside **Gmail**, opens **Advanced**, and selects the configuration's **Settings** tab to change these options. Filters apply separately to each connected mailbox. **Documents** shows indexed threads and **Sync history** shows recent runs. +An admin opens **Settings → Sources** and selects **Manage** beside **Gmail** to open its configuration list. Each row shows **Member accounts** or **Service account** beside its sync status. Open a configuration's **Settings** tab to edit its filters, then select **Save**. Filters apply separately to each mailbox in the source. **Documents** shows indexed threads and **Sync history** shows recent runs. + +**Sync using** identifies the configuration's fixed connection method. To replace a central credential, choose another **Indexing account** and select **Change indexing account**. | Option | Behavior | | --- | --- | -| Labels | Optional comma-separated names or system IDs, such as `Engineering, INBOX`. A thread matching any listed label is included. Leave empty for all labels. Custom IDs such as `Label_7` belong to one mailbox and cannot be used for member setup. | +| Labels | Optional comma-separated names or system IDs, such as `Engineering, INBOX`. A thread matching any listed label is included. Leave empty for all labels. Custom IDs such as `Label_7` belong to one mailbox and cannot be used for member or central setup. | +| Directory administrator email | Required for central indexing. An active Workspace administrator who can read Directory users; this does not limit the crawl to the administrator's mailbox. | +| Users | Central indexing only. Optional primary Workspace email addresses (up to 100); blank includes all active users in the customer. This selects which mailboxes to crawl. Each mailbox remains searchable only by its owner. | | Date Range | Last 6 months by default for Search sources. Choose the last 7, 30, or 90 days, a year, or all time. A knowledge-base connector outside Search defaults to all time. | | Exclude Promotions / Exclude Social | Both enabled by default. Choose **No** to include either category. | -| Search Filter | Optional [Gmail query](https://developers.google.com/workspace/gmail/api/guides/filtering), such as `from:team@example.com subject:release`. This filters what is indexed; it is not a Sim Search query. A source with a search filter cannot use Gmail's change history and relists the mailbox on every sync. | +| Search Filter | Optional [Gmail query](https://developers.google.com/workspace/gmail/api/guides/filtering), such as `from:team@example.com subject:release`. This filters what is indexed; it is not a Sim Search query. Member-account sources with a search filter relist the mailbox on every sync instead of using Gmail's change history. | In the add-source form, **More options** contains optional **Metadata tags**. Sync frequency and the general knowledge-base **Max Threads** setting are hidden in Search. @@ -72,7 +133,13 @@ Sim indexes the message text Gmail returns for each matching thread, plus subjec File attachments and image contents are not indexed. Thread discovery uses Gmail's default exclusion of Spam and Trash. A filter such as `has:attachment` selects the email thread; it does not index the attachment. Gmail API filtering also differs from Gmail's interface for aliases and thread-wide searches. See Google's [thread listing reference](https://developers.google.com/workspace/gmail/api/reference/rest/v1/users.threads/list) and [filtering guide](https://developers.google.com/workspace/gmail/api/guides/filtering). -Search schedules syncs hourly. The first sync lists every thread in scope and can take several runs for a large mailbox; results appear as documents are indexed. Later syncs read Gmail's change history instead of relisting the mailbox, so only threads that gained a message, were relabelled, or were deleted since the previous run are fetched. A full relisting runs about weekly, or sooner if Gmail no longer retains the history the source last read. Updates and removals are reconciled during background sync, rather than fetched live for each search. +Search schedules syncs hourly. The first sync lists every thread in scope and can take several runs for a large mailbox; results appear as documents are indexed. + +**Member accounts:** later syncs use each mailbox's Gmail change history, unless the configuration has a search filter. A full relisting runs about weekly, or sooner if Gmail no longer retains the saved history. + +**Service account:** each sync revisits the selected active mailboxes and resumes unfinished listings. It does not reuse one mailbox's history cursor across the company. Only new or changed threads need their bodies fetched. Failed mailbox reads leave the crawl incomplete; they are not treated as an empty mailbox for deletion reconciliation. + +Updates, removals, and access refresh in the background, rather than being checked live for each search. An empty mailbox or filters with no matching threads complete normally with zero documents. @@ -87,12 +154,15 @@ Threads that exceed indexing size limits are skipped and reconsidered when the t | Finish connecting in the other tab | Complete the Google flow, or use **Open again** while authorization is pending. If the popup was blocked or closed, allow popups and select **Connect** again. | | Reconnect | Click **Reconnect** and authorize the same account again. | | Unavailable or needs admin attention | Ask your Sim admin to check source status and the deployment's Google OAuth configuration. | +| Directory or delegation error | Check both central crawl scopes, the service-account key, and the Directory administrator's user-read privileges. A normal OAuth account cannot replace the central service account. | +| Gmail access fails for a selected user | Verify Gmail is enabled for that primary Workspace account and delegation is authorized. Narrow **Users** to accounts with Gmail enabled. Aliases and external accounts cannot be selected. | +| A central source indexes mail but a teammate sees no results | Confirm their verified Sim email is the mailbox's primary email and they belong to the Sim organization. Administrators do not receive other people's mailbox access. | ## Self-hosted operator setup For an External app in **Testing**, Google refresh tokens for these scopes expire after seven days. Before production use, configure the appropriate publishing status and complete any required verification; adding test users alone does not make a durable production connection. See [Google’s token expiration rules](https://developers.google.com/identity/protocols/oauth2#expiration). -Users do not need to create Google Cloud credentials. The deployment operator configures one Google OAuth client for the instance: +Member-account connections use the deployment's Google OAuth client below. Central service-account indexing uses the separate setup above and does not require each user to complete OAuth. 1. In [Google Cloud Console](https://console.cloud.google.com/), select your project. Open **APIs & Services → Library**, find **Gmail API**, and enable it. 2. Open **Google Auth platform → Branding**. Select **Get started** if needed, then enter the app name, support email, and contact email. Under **Audience**, use **Internal** only for an app limited to your Google Workspace organization; otherwise use **External** and add test users while testing. Review the app's permissions under **Data Access → Add or remove scopes**, using the current Sim scopes below. Follow Google's [consent and verification guidance](https://developers.google.com/workspace/guides/configure-oauth-consent) for your audience. @@ -107,7 +177,7 @@ https:///api/auth/oauth2/callback/google-email This Google Cloud example uses one client for all three services. Replace `https://sim.example.com` with your Sim origin and add only the callbacks for services you enable. -The current Sim Gmail connection uses these scopes: +The member-account OAuth connection uses these scopes: ```text openid @@ -119,5 +189,5 @@ https://www.googleapis.com/auth/gmail.labels ``` - Google's `gmail.readonly` scope is sufficient for Search's email reads. Sim currently shares its Gmail OAuth connection with workflow actions and requires the broader scope set above; do not substitute `gmail.readonly` in this setup. Search does not send or modify email. See [Google's scope descriptions](https://developers.google.com/workspace/gmail/api/auth/scopes). + Google's `gmail.readonly` scope is sufficient for Search's email reads and is used by the central service-account path. Member-account connections share their OAuth credentials with workflow actions and require the broader set above. Search does not send or modify email. See [Google's scope descriptions](https://developers.google.com/workspace/gmail/api/auth/scopes). diff --git a/apps/docs/content/docs/search/google-calendar.mdx b/apps/docs/content/docs/search/google-calendar.mdx index 1ce80d7ad57..5435e41ebd8 100644 --- a/apps/docs/content/docs/search/google-calendar.mdx +++ b/apps/docs/content/docs/search/google-calendar.mdx @@ -1,24 +1,35 @@ --- title: Google Calendar -description: Search calendar events using each teammate's own Google access +description: Connect personal Calendar accounts or index your company with a delegated service account --- import { Callout } from 'fumadocs-ui/components/callout' import { Step, Steps } from 'fumadocs-ui/components/steps' import { Image } from '@/components/ui/image' -Search meetings and event details available to your Google account. An organization admin enables the source; every teammate connects their own account. Google controls which calendar and event details each person can read. +Search meetings and event details available to your Google account. Members can connect personal accounts, or a Google Workspace administrator can configure a central service-account crawl. Both paths keep each person's own view of events separate. Admin setup uses your organization's **Settings → Sources** page. Teammates connect from **Integrations** in the main sidebar. For workspace Search, use **Search → Add source** instead; **Create & Invite** is the workspace equivalent of **Add source**. -## Set up Google Calendar +## Choose your setup + +| Method | Use it when | What teammates do | +| --- | --- | --- | +| **Member accounts** | Each person should connect their own Calendar access. No Google Workspace administrator setup is needed. | Connect their own Google Calendar account. | +| **Service account** | A Workspace administrator can authorize a central crawl with domain-wide delegation. | Join the Sim organization with matching verified email addresses; no personal Calendar connection is needed for this source. | + +Central indexing reads calendars as each selected user. An organizer's private event details are never reused as an attendee's copy. Each indexed copy is searchable only by the corresponding user, including when several users can read the same shared calendar. + +These are alternative setup paths. When only a central Calendar source is configured, Integrations does not offer a personal Calendar **Connect** action. Existing member-account sources keep their connection actions. + +## Connect member accounts ### Allow and connect Google Calendar -An admin opens **Settings → Sources** and turns on **Google Calendar**. Then each person opens **Integrations**, selects **Connect** beside Google Calendar, and authorizes their matching Google account. The first connection creates the default sync configuration. Search uses member accounts; an admin or service account cannot connect on behalf of everyone. +An admin opens **Settings → Sources** and turns on **Google Calendar**. Then each person opens **Integrations**, selects **Connect** beside Google Calendar, and authorizes their matching Google account. The first connection creates the default member-account sync configuration. @@ -32,21 +43,69 @@ An admin selects **Manage** beside Google Calendar in **Settings → Sources**, -### Save or add a configuration +### Save the configuration -The default date range covers the previous and next 30 days. Save any changes to the existing configuration. To create a separate one, select **Add sync configuration** on **Advanced**. In that form, **More options** contains **Search Query**, **Include Attendees**, and **Metadata tags**. Select **Add source** to save it; teammates connect their own accounts from Integrations. +The default date range covers the previous and next 30 days. Save any changes to the existing configuration. Its calendar and date filters apply separately to each connected member's access. **Add sync configuration** opens the [central service-account setup](#set-up-a-central-service-account); it does not edit this member configuration. -Google Calendar Search source configuration - - `primary` means the connected person's main calendar. A calendar selected from the list is a specific calendar ID, even when it is your main calendar. That same ID applies to every member, and only members with access to it can search its events. + `primary` means the connected or impersonated person's main calendar. A calendar selected from the list is a specific calendar ID, even when it is your main calendar. That same ID applies to every selected user, and only users with access to it can search its events. +## Set up a central service account + +Open **Settings → Sources** and turn on **Google Calendar**. Select **Manage → Advanced → Add sync configuration** to open the central service-account form. If personal connections are disabled for your organization, select **Add source** from the provider page instead. + +This requires a Google Workspace customer and a super administrator to authorize domain-wide delegation. Consumer Gmail accounts cannot use this path. + + + + +### Prepare the service account + +In [Google Cloud Console](https://console.cloud.google.com/), select your project and enable **Google Calendar API** and **Admin SDK API** under **APIs & Services → Library**. Open **IAM & Admin → Service Accounts → Create service account** and create the account. Google Cloud project roles do not grant Calendar access and are not required for this crawl. + +Open the service account's **Keys** tab, then select **Add key → Create new key → JSON → Create**. Keep the downloaded key secure; you will add it to Sim. See [Google's key creation guide](https://docs.cloud.google.com/iam/docs/keys-create-delete#creating). + + + + +### Authorize domain-wide delegation + +Copy the service account's numeric **Client ID** from **Details → Advanced settings**. As a Workspace super administrator, open **Security → Access and data control → API controls → Manage Domain Wide Delegation → Add new** in the [Admin Console](https://admin.google.com/ac/owl/domainwidedelegation). + +Enter the Client ID and these exact comma-separated **OAuth scopes**: + +```text +https://www.googleapis.com/auth/calendar.events.readonly,https://www.googleapis.com/auth/admin.directory.user.readonly +``` + +Select **Authorize** and verify both scopes under **View details**. If you reuse a Drive or Gmail service account, retain its existing delegated scopes and add any missing Calendar scopes. An existing Drive authorization alone does not grant Calendar access. Delegation can take up to 24 hours to propagate; organizations requiring multi-party approval need another super administrator to approve the change. See [Google's delegation guide](https://knowledge.workspace.google.com/admin/apps/control-api-access-with-domain-wide-delegation). + +The **Directory administrator email** must be an active Workspace administrator with permission to read users. A super administrator has this permission; a custom administrator role can supply it. This identity lists the directory. Sim obtains a separate read-only Calendar token for each selected user; it does not read everyone's events as the administrator. + + + + +### Add the credential and choose users + +Under **Indexing account**, select **Add service account** or an existing service account. In **Add Google Service Account**, give the credential a name and paste its JSON key. Back in the source form, enter the **Directory administrator email**. + +Leave **Calendar IDs** empty for each user's `primary` calendar. To include shared calendars, enter their IDs, optionally alongside `primary`. IDs apply to each selected user who can read that calendar; this does not share calendars or expand anyone's Google access. Central setup uses manual IDs because an administrator's calendar picker would not represent every user's calendars. + +Choose the **Date Range**. Under **More options**, leave **Users** blank to include all active users in the Workspace customer, including secondary domains, or enter up to 100 primary email addresses separated by commas. Suspended, archived, and guest accounts are excluded. Choose **Connect & Sync**. + +Sim verifies Directory access and selected users, then probes one selected user's primary calendar to check the delegated Calendar permission. Shared-calendar access is checked separately for each user during sync. Teammates join the Sim organization with their matching verified primary Workspace email; they do not connect personal accounts for this source. + + + + ## Connect your account +These steps apply to **Member accounts**. When only a central Calendar source is configured, teammates use Search or Home directly and are not offered a personal Calendar connection for that source. + 1. Join the Sim organization and verify your Sim email. Open **Integrations** and click **Connect** beside Google Calendar. 2. In the connection tab, choose the Google account whose verified email matches your Sim email. Grant the requested permissions. 3. Return to Integrations to see indexing status and your searchable document count. @@ -55,11 +114,15 @@ Teammates repeat only these connection steps after joining the organization. The ## Source options -An admin opens **Settings → Sources**, selects **Manage** beside **Google Calendar**, opens **Advanced**, and selects the configuration's **Settings** tab to change these options. **Documents** shows indexed events and **Sync history** shows recent runs. +An admin opens **Settings → Sources** and selects **Manage** beside **Google Calendar** to open its configuration list. Each row shows **Member accounts** or **Service account** beside its sync status. Open a configuration's **Settings** tab to edit its filters, then select **Save**. **Documents** shows indexed events and **Sync history** shows recent runs. + +**Sync using** identifies the configuration's fixed connection method. To replace a central credential, choose another **Indexing account** and select **Change indexing account**. | Option | Behavior | | --- | --- | | Calendars / Calendar IDs | Empty defaults to each member's `primary` calendar. Explicit IDs restrict the source to those calendars. Multiple IDs are comma-separated; combine `primary` with shared calendar IDs if needed. | +| Directory administrator email | Central indexing only. Required to enumerate Workspace users; it does not limit the source to the administrator's events. | +| Users | Central indexing only. Optional primary email addresses (up to 100); blank includes all active users in the Workspace customer. Each user's event copies remain private to that user. | | Date Range | Previous and next 30 days by default. Alternatives are the previous 30 days, next 30 days, or 90 days in each direction. The window moves forward on later syncs. | | Search Query | Optional text filter applied by Google to event titles, descriptions, locations, and organizer or attendee names and emails. Leave empty to include all matching events in the date range. | | Include Attendees | **Yes** by default. **No** omits organizer and attendee identity fields and keeps the attendee count. It does not redact names written into titles or descriptions. | @@ -70,9 +133,9 @@ In the add-source form, **More options** contains optional **Metadata tags**. Se Sim indexes event titles, descriptions, times, locations, and the selected attendee information. All-day events and individual occurrences of recurring meetings are supported. An invitation you declined stays searchable and is marked `Response: declined`. Results link back to Google Calendar. -Cancelled events, attachment contents, meeting recordings, and transcripts are not indexed. Status entries such as working location, out of office, focus time, and birthdays are not indexed. A shared calendar where you can see only free or busy times contributes nothing, since those blocks have no title or description. Events outside the selected date window are excluded. Private event details that Google withholds are not available in Search; see [Google's calendar sharing rules](https://developers.google.com/workspace/calendar/api/concepts/sharing). +Cancelled events, attachment contents, meeting recordings, and transcripts are not indexed. Status entries such as working location, out of office, focus time, and birthdays, and automatically generated reservation events from Gmail are not indexed. A shared calendar where you can see only free or busy times contributes nothing, since those blocks have no title or description. Events outside the selected date window are excluded. Private event details that Google withholds are not available in Search; see [Google's calendar sharing rules](https://developers.google.com/workspace/calendar/api/concepts/sharing). -Search schedules syncs hourly. Event edits, cancellations, access changes, and events moving outside the date window are reconciled during background sync. The first sync may take longer, and results appear as indexing progresses. +Search schedules syncs hourly. Event edits, cancellations, access changes, inactive or removed users, and events moving outside the date window are reconciled during completed background syncs. Central crawls page through each selected user and resume unfinished work before removing documents no longer listed. Authorization, quota, and provider failures stop the sync rather than treating unread calendars as empty. The first sync may take longer, and results appear as indexing progresses; Search is not a live Calendar read. ## Troubleshooting @@ -84,12 +147,15 @@ Search schedules syncs hourly. Event edits, cancellations, access changes, and e | A different email is requested | Choose the Google account matching your verified Sim email. | | Reconnect | Click **Reconnect** and complete Google authorization again. Allow pop-ups if the connection tab does not open. | | Unavailable or needs admin attention | Ask your Sim admin to check source status and the deployment's Google OAuth configuration. | +| Service-account authorization or Directory error | Confirm both delegated scopes, enabled APIs, and the Directory administrator's user-read privilege. Check whether delegation still awaits approval or propagation. | +| User not found or inactive | Use an active primary email in the same Workspace customer. Aliases, external or guest accounts, suspended users, and archived users cannot be selected. | +| A central source has no results for a teammate | Confirm their primary Workspace email matches their verified Sim email, they belong to the Sim organization, and they are included in **Users**. Check calendar IDs and **Sync history**. | ## Self-hosted operator setup For an External app in **Testing**, Google refresh tokens for these scopes expire after seven days. Before production use, configure the appropriate publishing status and complete any required verification; adding test users alone does not make a durable production connection. See [Google’s token expiration rules](https://developers.google.com/identity/protocols/oauth2#expiration). -The deployment operator configures Google OAuth once; teammates then use the normal connection flow. +The deployment operator configures Google OAuth for member accounts and the member-mode calendar picker. Central service accounts use the separate delegation setup above. 1. In [Google Cloud Console](https://console.cloud.google.com/), select your project. Open **APIs & Services → Library**, find **Google Calendar API**, and enable it. 2. Open **Google Auth platform → Branding** and configure the app name and contact details. Under **Audience**, choose **Internal** for your Google Workspace organization only, or **External** for other users. Add test users while an external app is testing. Review **Data Access → Add or remove scopes** using the current Sim scopes below. See Google's [consent and verification guidance](https://developers.google.com/workspace/guides/configure-oauth-consent). @@ -104,7 +170,7 @@ https:///api/auth/oauth2/callback/google-calendar This Google Cloud example uses one client for all three services. Replace `https://sim.example.com` with your Sim origin and add only the callbacks for services you enable. -The current Sim Calendar connection uses these scopes: +The member-account OAuth connection uses these scopes: ```text openid @@ -114,5 +180,5 @@ https://www.googleapis.com/auth/calendar ``` - Search's reads can use `calendar.events.readonly`, `calendar.calendarlist.readonly`, and `calendar.calendars.readonly` for events, the calendar list, and calendar details. Sim currently shares its Calendar OAuth connection with workflow actions and requires the broader `calendar` scope above. Do not replace it with read-only scopes in this setup. Search does not change calendars or events. See [Google's scope descriptions](https://developers.google.com/workspace/calendar/api/auth). + The central service account uses `calendar.events.readonly` for event reads and the separate Directory scope listed earlier. Member connections share their OAuth credentials with workflow actions and require the broader `calendar` scope above. Search does not change calendars or events. See [Google's scope descriptions](https://developers.google.com/workspace/calendar/api/auth). diff --git a/apps/docs/content/docs/search/google-drive.mdx b/apps/docs/content/docs/search/google-drive.mdx index 230449ffbb2..8cde905b3e9 100644 --- a/apps/docs/content/docs/search/google-drive.mdx +++ b/apps/docs/content/docs/search/google-drive.mdx @@ -1,6 +1,6 @@ --- title: Google Drive -description: Connect Drive files through member accounts or a delegated service account +description: Connect personal Drive accounts or index your company with a delegated service account --- import { Callout } from 'fumadocs-ui/components/callout' @@ -16,10 +16,12 @@ Admin setup uses your organization's **Settings → Sources** page. Teammates co | Method | Use it when | What teammates do | | --- | --- | --- | | **Member accounts** | Each person should connect their own Drive access. No Google Workspace administrator setup is needed. | Connect their own Google Drive accounts. | -| **Service account** | A Google Workspace administrator can configure delegation and directory access for a central crawl. | Sign in to Sim with matching verified email addresses; no personal Drive connection is needed for this source. | +| **Service account** | A Google Workspace administrator can configure delegation and directory access for a central crawl. | Join the Sim organization with matching verified email addresses; no personal Drive connection is needed for this source. | + +These are alternative setup paths. When only a central Drive source is configured, Integrations does not offer a personal Drive **Connect** action. Teammates use Search or Home directly. Existing member-account sources keep their connection actions. - A central crawl indexes only files the configured **Crawl as** account can access. Domain-wide delegation does not make this connector crawl every employee's Drive. Share the intended content with the indexing account, or use member accounts for each person's accessible files. + A central crawl reads each selected employee's Drive through domain-wide delegation, including private My Drive files and shared-drive files they can access. Leave **Users** blank to include all active users in your Google Workspace customer, including secondary domains. Files keep their original user and group permissions; indexing a private file does not make it visible to other employees. ## Connect member accounts @@ -52,11 +54,11 @@ Keep **Sync documents with → Connected members** unless a dedicated account sh ## Set up a central service account -Open **Settings → Sources** and turn on **Google Drive**. Select **Manage → Advanced → Add sync configuration**. This opens central service-account setup. If your organization has only central indexing enabled, select **Add source** from the provider's **Sources** tab instead. Teammates do not need a personal Drive connection for this source. +Open **Settings → Sources** and turn on **Google Drive**. Select **Manage → Advanced → Add sync configuration** to open the central service-account form directly. If personal connections are disabled for your organization, select **Add source** from the provider page instead. Teammates do not need a personal Drive connection for this source. This requires a Google Workspace domain and a Workspace super administrator to authorize domain-wide delegation. Consumer Gmail accounts cannot use this path. -Google Drive central source setup with an indexing account, sharing policy, and folder scope +Google Drive central source setup with a service account, Directory administrator email, sharing policy, and optional folders @@ -83,55 +85,70 @@ In the service account's **Details**, expand **Advanced settings** and copy its Paste that Client ID into **Client ID**, then enter these exact scopes as a comma-separated list under **OAuth scopes**: ```text -https://www.googleapis.com/auth/drive.readonly,https://www.googleapis.com/auth/admin.directory.group.readonly,https://www.googleapis.com/auth/admin.directory.domain.readonly +https://www.googleapis.com/auth/drive.readonly,https://www.googleapis.com/auth/admin.directory.user.readonly,https://www.googleapis.com/auth/admin.directory.group.readonly,https://www.googleapis.com/auth/admin.directory.domain.readonly ``` -Select **Authorize**, then **View details** to confirm all three scopes were saved. If your organization requires multi-party approval, another super administrator must approve the request. Delegation changes can take up to 24 hours to propagate. See Google's [Admin Console delegation guide](https://knowledge.workspace.google.com/admin/apps/control-api-access-with-domain-wide-delegation). +Select **Authorize**, then **View details** to confirm all four scopes were saved. If you reuse a Gmail or Calendar service account, retain those services' required scopes and add any missing Drive scopes. If your organization requires multi-party approval, another super administrator must approve the request. Delegation changes can take up to 24 hours to propagate. See Google's [Admin Console delegation guide](https://knowledge.workspace.google.com/admin/apps/control-api-access-with-domain-wide-delegation). These are Search's central crawl scopes. The general [Google service account guide](/integrations/google-service-account) includes broader scopes for workflow actions; do not copy those into this Search setup. -Group permissions require groups and memberships that the indexing administrator can read in this Google Workspace customer. External groups and unresolvable nested groups are not supported. Google Drive target-audience shares are not mapped; use explicit user, supported group, or domain permissions instead. +The **Directory administrator email** must belong to an active Workspace administrator with permission to read users, groups, group memberships, and domains. A super administrator has these privileges; a custom administrator role can supply them instead. This identity enumerates the directory. Sim obtains separate read-only Drive tokens for the selected users. + +Group permissions require groups and memberships that this administrator can read in this Google Workspace customer. External groups and unresolvable nested groups are not supported. Google Drive target-audience shares are not mapped; use explicit user, supported group, or domain permissions instead. ### Add the credential in Sim -Under **Indexing account**, choose the service-account connection action, or select an existing service account. Paste the JSON key into **Add Google Service Account**, give it a name, and add it. Sim returns you to the source form with that credential selected. +Under **Indexing account**, choose **Add service account**, or select an existing service account. Paste the JSON key into **Add Google Service Account**, give it a name, and add it. Sim returns you to the source form with that credential selected. Add Google Service Account credential modal in Sim -### Choose the indexing identity +### Choose the directory administrator and users + +Set **Directory administrator email** to the Workspace administrator described above. Under **More options**, leave **Users** blank for everyone, or enter up to 100 primary Workspace email addresses separated by commas. Suspended, archived, and guest accounts are excluded. + +Leave **Folders** empty to include supported files each selected user can access. To narrow the source, select folders visible to the Directory administrator or enter **Folder IDs** manually. The same folder filter applies to each selected user and does not grant access. Choose **Connect & Sync**. Sim validates the administrator and any selected users, and probes Drive access for an active user, before accepting the source. -Set **Crawl as** to a Google Workspace administrator who can read groups, memberships, and domains, and can access the content you want indexed. Select folders if needed, then choose **Connect & Sync**. Sim validates Drive and Directory access before accepting the source. +The crawl pages through each user’s files and shared drives, including shared-drive files the user has never opened. It resumes unfinished work and indexes a shared file once even when several users can access it. A file must be downloadable by at least one selected user to be indexed; the same requirement applies to a shortcut’s target. Sim must also verify its permissions before showing it in Search. Google can let a reader download a file while refusing to list its permissions; include an owner or another user who can read those permissions. Externally owned files can remain hidden when no selected user can verify their permissions. + +Teammates join the Sim organization with their matching verified email; they do not connect personal Google accounts for this central source. ## Source options +An admin opens **Settings → Sources** and selects **Manage** beside **Google Drive** to open its configuration list. Each row shows **Member accounts** or **Service account** beside its sync status. Open a configuration's **Settings** tab to edit its filters, then select **Save**. **Documents** shows indexed files and **Sync history** shows recent runs. + +**Sync using** identifies the configuration's fixed connection method. To replace a central credential, choose another **Indexing account** and select **Change indexing account**. + | Option | Behavior | | --- | --- | | Folders / Folder IDs | Optional. Includes files in each selected folder and its accessible subfolders. A folder selection does not grant access. | | File Type | All supported files by default, or only Google Docs, Sheets, Slides, or text formats. **Plain text files only** also includes CSV, HTML, Markdown, JSON, and XML. | -| Crawl as | Required for the central service account. In Member accounts, it optionally supplies the impersonated user when a dedicated service account fetches content. It has no effect on ordinary OAuth accounts. | +| Directory administrator email | Required for central indexing. Supplies Directory access for user enumeration and permission groups; it does not limit the crawl to this administrator's files. | +| Users | Central indexing only. Optional primary email addresses (up to 100); blank includes all active users in this Workspace customer. This selects which users' Drives to crawl, not who may search the resulting files. | +| Crawl as | In Member accounts, optionally supplies the impersonated user when a dedicated service account fetches content. It has no effect on ordinary OAuth accounts. | | Openly shared files | Applies only to central crawls; it has no effect in Member accounts. **Keep out of search** by default. You can include discoverable domain shares or discoverable public shares. Link-only sharing does not grant Search access; named user and group permissions still apply. | -| Metadata tags | Optional owner, file type, modification date, and starred metadata. In the add-source form, these and **File Type** are under **More options**. | +| Metadata tags | Optional owner, file type, modification date, and starred metadata. In the add-source form, these, **Users**, and **File Type** are under **More options**. | Sim exports Docs and Slides as text and Sheets as XLSX spreadsheets. Supported uploaded files use the knowledge-base document pipeline, including PDF and Office formats. Unsupported files and oversized exports cannot be indexed; Google limits Workspace exports to 10 MB. See [Drive export formats](https://developers.google.com/workspace/drive/api/guides/ref-export-formats) and [download limits](https://developers.google.com/workspace/drive/api/guides/manage-downloads). -Search schedules syncs hourly. Content, deletions, and permissions refresh in the background; results are not a live read from Drive. Open **Settings → Sources**, select **Manage** beside **Google Drive**, then open its configuration under **Advanced** to inspect **Documents**, edit **Settings**, or review **Sync history**. **Accounts** on the provider page shows personal account connections where configured; it does not list the central service-account credential. +Search schedules syncs hourly. Central crawls revisit the selected users' files and permissions, including unchanged files, so permission changes and a new employee's older files are included. Unfinished crawls resume before deletion reconciliation. Content, deletions, and permissions refresh in the background; results are not a live read from Drive. **Accounts** on the provider page shows personal account connections where configured; it does not list the central service-account credential. ## Troubleshooting | Problem | Next step | | --- | --- | -| Directory access failed | Check the delegated scopes and the **Crawl as** user's administrator privileges. A normal Google OAuth credential cannot supply this central Search path. | -| Missing files in a central crawl | Open them as the **Crawl as** user. Delegation does not grant that user access to all domain files. Check folder and file-type filters. | -| A teammate sees no results | Confirm their verified Sim email matches the Drive permission or group membership. For member accounts, finish their personal Drive connection too. | +| Directory access failed | Check all four delegated scopes and the **Directory administrator email** user's administrator privileges. A normal Google OAuth credential cannot supply this central Search path. | +| Missing files in a central crawl | Check **Users**, folder and file-type filters, and whether selected active Workspace users can download the file and read its permissions. Opening a file alone does not prove either. Check Sync history for errors. Files reachable only by excluded or inactive accounts are not crawled; files with unverified permissions stay hidden. | +| User not found or inactive | Use a primary email in the same Google Workspace customer. Aliases, external or guest accounts, suspended users, and archived users cannot be selected for crawling. | +| A teammate sees no results | Confirm they have joined the Sim organization and their verified Sim email matches the Drive permission or group membership. For member accounts, finish their personal Drive connection too. | | A public or shared-link file is missing | Check **Openly shared files**. Link-only sharing does not grant Search access. A named user or group permission can still make the file searchable. | | Reconnect or credential error | Reauthorize the member account, or replace the service-account credential and verify delegation, as applicable. | @@ -164,5 +181,5 @@ https://www.googleapis.com/auth/drive.file ``` - Google's `drive.readonly` scope covers Search's file reads. Sim's existing OAuth connection also supports workflow actions and requires the broader scopes above; do not substitute read-only scopes for member OAuth. The central service account uses the separate read-only Drive and Directory scopes listed earlier. See [Google's Drive scope descriptions](https://developers.google.com/workspace/drive/api/guides/api-specific-auth). + Google's `drive.readonly` scope covers Search's file reads. Sim's existing OAuth connection also supports workflow actions and requires the broader scopes above; do not substitute read-only scopes for member OAuth. The central service account uses the four separate read-only Drive and Directory scopes listed earlier. Adding company-wide indexing requires the Directory user-read scope in the service account's domain-wide delegation; it does not change the member OAuth app scopes. See [Google's Drive scope descriptions](https://developers.google.com/workspace/drive/api/guides/api-specific-auth). diff --git a/apps/docs/content/docs/search/index.mdx b/apps/docs/content/docs/search/index.mdx index 104f2f83852..b3b9aa797be 100644 --- a/apps/docs/content/docs/search/index.mdx +++ b/apps/docs/content/docs/search/index.mdx @@ -27,7 +27,7 @@ The switch saves immediately and permits the integration in your organization. I For personal Gmail, Calendar, or Drive, teammates can connect immediately from **Integrations**. The first account creates the default sync configuration. Admins can adjust it later under **Manage → Advanced** on the provider page. -For other sources, select **Set up** or **Manage**, then **Add source**. For a central Drive service account, use **Manage → Advanced → Add sync configuration**. For Slack, complete **Set up Slack app** first. Follow the provider's **Setup guide** to choose the indexing account and content. **More options** contains secondary filters and **Metadata tags**. +For other sources, select **Set up** or **Manage**, then **Add source**. For a central Gmail, Calendar, or Drive service account, use **Manage → Advanced → Add sync configuration**. For Slack, complete **Set up Slack app** first. Follow the provider's **Setup guide** to configure its connection and content. **More options** contains secondary filters and **Metadata tags**. In the source form, select **Connect & Sync** for a central account or **Add source** for member accounts. Creating a source does not invite people or authorize their accounts. @@ -36,7 +36,7 @@ In the source form, select **Connect & Sync** for a central account or **Add sou ### Connect and search -Open **Integrations** in the main sidebar and select **Connect** if prompted—even if you created the source. Complete authorization in the new tab. Open **Search** to find documents, or **Home** to ask the assistant about them. Documents become available as background indexing progresses. +For member-account sources, open **Integrations** in the main sidebar and select **Connect** if prompted—even if you created the source. Complete authorization in the new tab. Central Google sources need no personal connection: join with a matching verified Workspace email and open **Search** to find documents, or **Home** to ask the assistant about them. Documents become available as background indexing progresses. @@ -47,21 +47,22 @@ Source availability depends on the deployment and organization policy. An unavai ## Choose the right connection method -Most sources use member accounts. GitHub supports a central App installation with a personal connection for each reader. Google Drive and Confluence also support a central service-account connection; GitLab requires an administrator token for a self-managed instance. +Sources can use member accounts or a central connection. Gmail, Google Calendar, and Google Drive support Google Workspace service accounts with domain-wide delegation. Confluence supports a central service account with a personal identity connection for each reader. GitHub supports an App installation, and GitLab requires an administrator token for a self-managed instance. | Method | What the admin does | What teammates do | | --- | --- | --- | | **Member accounts** | Allows the provider and adjusts shared filters when needed. | Connect their own accounts. Sim lists documents using each member's access. | | **GitHub App installation** | Installs the App, selects it under **Sync documents with**, and adds repository sources. | Connect GitHub once. Sim checks each reader's current repository access before returning installation-indexed content. | -| **Service account** (Drive and Confluence) | Connects an account that can read the content and the source's permissions or directory. | Join the organization with a matching verified identity. Confluence also requires each person to connect their account. | +| **Google Workspace service account** | Configures domain-wide delegation, supplies a Directory administrator email, and selects users and content filters. | Join with a matching verified email. No personal Google connection is needed for that source. Gmail remains private to the mailbox owner; Calendar preserves each person's event view; Drive uses file permissions. | +| **Confluence service account** | Connects an account that can read the selected content and its permissions. | Join the organization and connect Confluence to establish identity. | | **Administrator token** (GitLab) | Connects a self-managed instance administrator token and selects projects to index. | Join the organization with a verified Sim email matching GitLab. No personal connection is needed. | -Adding a Google Drive or Confluence source from the admin page starts central setup. Drive uses **Manage → Advanced → Add sync configuration**; Confluence uses **Set up/Manage → Add source**. For personal connections, use **Integrations → Connect** in the main sidebar. An approved provider can create its first member source there; required repository, site, or project fields are collected before authorization. Admins can edit that source's filters afterward in its **Settings** tab. When personal connections are disabled for the organization, central Drive uses **Sources → Add source** instead of **Advanced**. +Adding a Google or Confluence source from the admin page opens its central setup form directly. Google providers use **Manage → Advanced → Add sync configuration**; Confluence uses **Set up/Manage → Add source**. For personal connections, use **Integrations → Connect** in the main sidebar. An approved provider can create its first member source there; required repository, site, or project fields are collected before authorization. Admins can edit that source's filters afterward in its **Settings** tab. When personal connections are disabled for the organization, Google providers use **Add source** on the provider page instead of **Advanced**. Some member sources offer **Sync documents with**, either directly in setup or under **More options**. **Connected members** uses members' accounts for both content and access checks. Selecting a dedicated account uses it to fetch content; members still connect to establish which documents they may search. For GitHub organization sources, choose **Connect GitHub App** in this field to [connect an installation](/search/github#add-a-repository). **Account for browsing** only helps an admin pick source options—it does not enroll that account for Search. - An administrator connection does not grant everyone access to everything. Search applies the source's supported permission rules. It also does not automatically discover every employee's data: the indexing account must be able to read the configured content. + An administrator connection does not grant everyone access to everything. Search applies the source's supported permission rules. ## Connector guides @@ -71,8 +72,8 @@ Some member sources offer **Sync documents with**, either directly in setup or u | [Confluence](/search/confluence) | Pages and blog posts | Service account or member accounts; each teammate connects | | [GitHub](/search/github) | Repository text files | App installation or member indexing; each teammate connects | | [GitLab](/search/gitlab) | Repository files, wikis, issues, merge requests | Self-managed instance administrator token; no member connection | -| [Gmail](/search/gmail) | Email thread text | Each member's Gmail account | -| [Google Calendar](/search/google-calendar) | Calendar events | Each member's Google Calendar account | +| [Gmail](/search/gmail) | Email thread text | Delegated service account or member accounts | +| [Google Calendar](/search/google-calendar) | Meetings | Delegated service account or member accounts | | [Google Drive](/search/google-drive) | Supported Drive files | Delegated service account or member accounts | | [Jira](/search/jira) | Issues | Each member's Jira account | | [Slack](/search/slack) | Channel messages and threads | Slack app installation plus each member's authorization | @@ -91,9 +92,9 @@ These requests are separate from organization invitations. They let recipients c ## Manage sources and documents -Open **Settings → Sources** and select **Manage** beside the integration. Gmail, Calendar, and Drive list their sync configurations under **Advanced** and personal connections under **Accounts**. Other providers with personal connections use **Sources** and **Accounts**; GitLab opens directly to the source list. Select a source or sync configuration to manage it. Multiple sources can have different folder, repository, space, or project scopes. +Open **Settings → Sources** and select **Manage** beside the integration. When personal connections are enabled, Gmail, Calendar, and Drive list their sync configurations under **Advanced** and connections under **Accounts**. Other providers with personal connections use **Sources** and **Accounts**. Providers without personal connections open directly to the source list. Select a source or sync configuration to manage it. Multiple sources can have different folder, repository, space, or project scopes. -For Gmail, one configuration is usually enough. **Add sync configuration** creates another source; editing **Settings** updates the selected source. Each configuration applies to all connected Gmail accounts, including accounts connected later, using each person's own mailbox permissions. Configurations are not assigned to individual people. +For Gmail, one configuration is usually enough. **Add sync configuration** creates another source; editing **Settings** updates the selected source. A member configuration applies to all connected Gmail accounts, including accounts connected later. A central configuration applies to the selected Workspace **Users**, or all active users when that field is blank. Its labels and filters are evaluated separately in each mailbox. Gmail Advanced tab with a sync configuration, search, and Add sync configuration action @@ -132,9 +133,9 @@ Search runs background syncs on an hourly schedule. Large sources, provider limi ## If indexing needs attention -A completed sync means the source was checked; some documents may still be indexing. Integrations lists personal connections; a central source that needs no personal account, such as delegated Drive, can be searchable without appearing there. In the main **Integrations** page, each connected source row shows how many documents you can search and whether indexing failed for any documents you can access. +A completed sync means the source was checked; some documents may still be indexing. Integrations lists personal connections; a central Google source can be searchable without appearing there. In the main **Integrations** page, each connected source row shows how many documents you can search and whether indexing failed for any documents you can access. -As an admin, open **Settings → Sources**, select **Manage** beside the integration, then open the source from **Advanced** (Google providers) or **Sources**. In **Documents**, select **Failed** from the status dropdown to inspect those files. Use the search field to find a document by name. Select **Retry indexing** beside a file to try again. **Exclude** removes a file from search; select **Excluded** and then **Restore** to include it again. Fix a disconnected account or source configuration before retrying a sync that needs attention. +As an admin, open **Settings → Sources**, select **Manage** beside the integration, then open its source or sync configuration. In **Documents**, select **Failed** from the status dropdown to inspect those files. Use the search field to find a document by name. Select **Retry indexing** beside a file to try again. **Exclude** removes a file from search; select **Excluded** and then **Restore** to include it again. Fix a disconnected account or source configuration before retrying a sync that needs attention. Empty source Documents tab with search and an Included status filter diff --git a/apps/docs/content/docs/search/jira.mdx b/apps/docs/content/docs/search/jira.mdx index 7306de72675..aea69cfbdf1 100644 --- a/apps/docs/content/docs/search/jira.mdx +++ b/apps/docs/content/docs/search/jira.mdx @@ -43,7 +43,7 @@ Under **Account for browsing**, select an account or choose **Connect Jira accou If you already know the project keys, use the switch beside **Projects** to select manual input and enter keys such as `ENG, SUPPORT`. Manual input lets you configure the source without connecting a browsing account first. -**Account for browsing** only populates the project picker. It does not enroll you or share that account's issue access with teammates. +**Account for browsing** only populates the project picker. You can use OAuth or **Add service account** for this step; neither enrolls that account for Search or enables central indexing. Each teammate still connects their own Jira account. { + it('uses Sources terminology in search and its empty state', async () => { + await render('?search=not-a-real-source') + expect(container.querySelector('input[placeholder="Search sources..."]')).toHaveValue( + 'not-a-real-source' + ) + expect(container.textContent).toContain('No matching sources') + expect(container.textContent).not.toContain('No matching integrations') + }) it('offers Drive account management before anyone has connected', async () => { mocks.overview.mockReturnValue({ data: { diff --git a/apps/sim/app/o/[organizationId]/settings/components/integrations/organization-integrations-setup.tsx b/apps/sim/app/o/[organizationId]/settings/components/integrations/organization-integrations-setup.tsx index 406f1bfd564..a3de964ef15 100644 --- a/apps/sim/app/o/[organizationId]/settings/components/integrations/organization-integrations-setup.tsx +++ b/apps/sim/app/o/[organizationId]/settings/components/integrations/organization-integrations-setup.tsx @@ -61,7 +61,7 @@ export function OrganizationIntegrationsSetup() { return ( {availability.integrationAvailabilityError && ( Loading sources… ) : visible.length === 0 ? ( - No matching integrations + No matching sources ) : ( visible.map(([type, meta]) => { const provider = providers.get(type) diff --git a/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.test.tsx b/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.test.tsx index 7a7cd0d3c8d..7dda89df3ff 100644 --- a/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.test.tsx +++ b/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.test.tsx @@ -53,11 +53,17 @@ vi.mock('@/lib/credential-groups/providers', () => ({ })) vi.mock('@/connectors/registry', () => ({ CONNECTOR_META_REGISTRY: { - google_drive: { name: 'Google Drive', auth: { mode: 'oauth', provider: 'google-drive' } }, - gmail: { name: 'Gmail', auth: { mode: 'oauth', provider: 'google-email' } }, + google_drive: { + name: 'Google Drive', + auth: { mode: 'oauth', provider: 'google-drive', adminCredentialType: 'service_account' }, + }, + gmail: { + name: 'Gmail', + auth: { mode: 'oauth', provider: 'google-email', adminCredentialType: 'service_account' }, + }, google_calendar: { name: 'Google Calendar', - auth: { mode: 'oauth', provider: 'google-calendar' }, + auth: { mode: 'oauth', provider: 'google-calendar', adminCredentialType: 'service_account' }, }, slack: { name: 'Slack', auth: { mode: 'oauth', provider: 'slack' } }, gitlab: { name: 'GitLab', auth: { mode: 'apiKey' } }, @@ -111,11 +117,17 @@ vi.mock('@/app/o/[organizationId]/settings/components/integrations/slack-account import { SettingsHeaderProvider, SettingsHeaderShell } from '@/components/settings/settings-header' import { OrganizationProviderDetail } from '@/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail' -const provider = { connectorType: 'google_drive', approved: true, status: 'active' } +const provider = { + connectorType: 'google_drive', + approved: true, + status: 'active', + sourceCount: 0, +} const source = { connectorId: 'source-one', connectorType: 'google_drive', sourceDescription: 'Engineering handbook', + accessMode: 'admin', enabled: true, hasSyncError: false, isSyncing: false, @@ -227,7 +239,7 @@ describe('organization provider management', () => { }) mocks.accounts.mockReturnValue({ data: { credentialGroup: null }, isPending: false }) mocks.sources.mockReturnValue({ data: [], isPending: false }) - await render(connectorType) + await render(connectorType, '?view=accounts') expect(container.textContent).toContain('Waiting for connections') expect(container.textContent).toContain( 'Members connect their accounts from Integrations. Indexing starts automatically.' @@ -241,9 +253,7 @@ describe('organization provider management', () => { expect.objectContaining({ enabled: false }) ) await click('Advanced') - expect(container.textContent).toContain( - 'No accounts connected yet. A sync configuration will be created when someone connects.' - ) + expect(container.textContent).toContain('No sync configurations yet.') expect(container.textContent).toContain('Add sync configuration') await click('Add sync configuration') await vi.waitFor(() => { @@ -257,6 +267,146 @@ describe('organization provider management', () => { } ) + it.each(['gmail', 'google_calendar', 'google_drive'])( + 'does not ask for personal connections when %s already has a central source', + async (connectorType) => { + mocks.overview.mockReturnValue({ + data: { + providers: [{ ...provider, connectorType, sourceCount: 1 }], + }, + }) + mocks.accounts.mockReturnValue({ data: { credentialGroup: null }, isPending: false }) + mocks.access = { admin: true, members: true } + await render(connectorType) + + expect(container.textContent).toContain('Engineering handbook') + expect(container.textContent).not.toContain('No connected member accounts.') + expect(container.textContent).not.toContain( + 'Members connect their accounts from Integrations.' + ) + await click('Add sync configuration') + await vi.waitFor(() => { + const params = mocks.updateUrl.mock.calls.at(-1)![0].searchParams + expect(params.get('addConnector')).toBe(connectorType) + expect(params.get('source-access')).toBeNull() + }) + } + ) + + describe.each(['gmail', 'google_calendar', 'google_drive'])( + '%s default management view', + (connectorType) => { + function withSources(accessModes: string[]) { + mocks.overview.mockReturnValue({ + data: { providers: [{ ...provider, connectorType, sourceCount: accessModes.length }] }, + isPending: false, + }) + mocks.sources.mockReturnValue({ + data: accessModes.map((accessMode, index) => ({ + ...source, + connectorType, + accessMode, + connectorId: `source-${index}`, + })), + isPending: false, + }) + } + + it.each([ + { name: 'member-only', modes: ['members'] }, + { name: 'central-only', modes: ['admin'] }, + { name: 'mixed', modes: ['admin', 'members'] }, + { name: 'not configured', modes: [] }, + ])('opens configurations for the $name setup', async ({ modes }) => { + withSources(modes) + await render(connectorType) + expect(container.querySelector('[role="radio"][aria-checked="true"]')).toHaveTextContent( + 'Advanced' + ) + expect(mocks.sources).toHaveBeenLastCalledWith( + expect.any(Object), + expect.objectContaining({ enabled: true }) + ) + expect(mocks.accounts).toHaveBeenLastCalledWith(undefined) + expect(container.textContent).toContain('Add sync configuration') + if (modes.length === 0) + expect(container.textContent).toContain('No sync configurations yet.') + }) + + it.each(['accounts', 'sources'])('honors explicit %s links', async (view) => { + withSources(['admin']) + await render(connectorType, `?view=${view}`) + expect(mocks.sources).toHaveBeenLastCalledWith( + expect.any(Object), + expect.objectContaining({ enabled: view === 'sources' }) + ) + expect(container.querySelector('[role="radio"][aria-checked="true"]')).toHaveTextContent( + view === 'accounts' ? 'Accounts' : 'Advanced' + ) + }) + + it('loads the configuration list in parallel with its overview and retains the default', async () => { + mocks.overview.mockReturnValue({ isPending: true }) + await render(connectorType) + expect(container.textContent).toContain('Loading integration…') + expect(container.textContent).not.toContain('No connected member accounts.') + expect(mocks.accounts).toHaveBeenLastCalledWith(undefined) + expect(mocks.sources).toHaveBeenLastCalledWith( + expect.any(Object), + expect.objectContaining({ enabled: true }) + ) + withSources(['members']) + await render(connectorType) + expect(container.querySelector('[role="radio"][aria-checked="true"]')).toHaveTextContent( + 'Advanced' + ) + expect(container.textContent).toContain('Engineering handbook') + expect(mocks.sources).toHaveBeenLastCalledWith( + expect.any(Object), + expect.objectContaining({ enabled: true }) + ) + expect(mocks.accounts).toHaveBeenLastCalledWith(undefined) + }) + + it('preserves an explicit Accounts choice when the overview changes', async () => { + withSources(['members']) + await render(connectorType) + await click('Accounts') + await vi.waitFor(() => + expect(mocks.updateUrl.mock.calls.at(-1)?.[0].searchParams.get('view')).toBe('accounts') + ) + withSources(['admin']) + await render(connectorType) + expect(container.querySelector('[role="radio"][aria-checked="true"]')).toHaveTextContent( + 'Accounts' + ) + expect(mocks.sources).toHaveBeenLastCalledWith( + expect.any(Object), + expect.objectContaining({ enabled: false }) + ) + }) + + it.each([ + { accessMode: 'admin', method: 'Service account' }, + { accessMode: 'members', method: 'Member accounts' }, + ])( + 'identifies $method configurations without changing their title or destination', + async ({ accessMode, method }) => { + withSources([accessMode]) + mocks.sources.mockReturnValue({ + data: [{ ...source, connectorType, accessMode }], + isPending: false, + }) + await render(connectorType) + expect(container.textContent).toContain(`${method} · Last synced`) + expect( + container.querySelector('a[aria-label="Open Engineering handbook"]') + ).toHaveAttribute('href', '/o/org-one/settings/integrations/sources/source-one') + } + ) + } + ) + it.each(['active', 'disabled'])( 'removes only Slack account setup after confirmation, including a %s option', async (status) => { @@ -389,7 +539,7 @@ describe('organization provider management', () => { expect(mocks.people).not.toHaveBeenCalled() expect( container.querySelector( - `input[placeholder="${params === '?view=sources' ? 'Search sync configurations...' : 'Search people...'}"]` + `input[placeholder="${params === '?view=accounts' ? 'Search people...' : 'Search sync configurations...'}"]` ) ).toBeEnabled() } diff --git a/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.tsx b/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.tsx index 8e8003d9bfa..cbf18f2f4e0 100644 --- a/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.tsx +++ b/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.tsx @@ -60,7 +60,7 @@ export function OrganizationProviderDetail({ connectorType }: OrganizationProvid const automaticSetup = Boolean(meta && canConnectWithDefaults(meta) && searchAccess.memberScoped) const [view, setView] = useQueryState( organizationProviderTabParam.key, - organizationProviderTabParam.parser.withDefault(automaticSetup ? 'accounts' : 'sources') + organizationProviderTabParam.parser ) const [search, setSearch] = useSettingsSearch() const [peopleSearch, setPeopleSearch] = useOrganizationAccountPeopleSearch() @@ -260,7 +260,13 @@ export function OrganizationProviderDetail({ connectorType }: OrganizationProvid )} @@ -363,7 +369,9 @@ export function OrganizationProviderDetail({ connectorType }: OrganizationProvid ? needsSlackSetup ? 'Set up the Slack app to connect accounts.' : automaticSetup - ? 'Members connect their accounts from Integrations. Indexing starts automatically.' + ? provider && provider.sourceCount > 0 + ? 'No connected member accounts.' + : 'Members connect their accounts from Integrations. Indexing starts automatically.' : 'Add a source to set up account connections.' : 'Activate this integration to set up account connections.'} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/source-chip.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/source-chip.test.tsx index fbba6a94034..dcc94a58b90 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/source-chip.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/source-chip.test.tsx @@ -33,6 +33,28 @@ afterEach(() => { }) describe('citation labels', () => { + it('labels a Slack channel citation without losing its message title or permalink', () => { + const source = { + url: 'https://example.slack.com/archives/C123/p1789000000000000', + title: `#engineering: Release notes https://example.com/${'a'.repeat(300)}`, + connectorType: 'slack', + } + const view = mount() + const link = view.querySelector('a')! + expect(link.textContent).toBe('#engineering') + expect(link.getAttribute('href')).toBe(source.url) + + act(() => { + link.dispatchEvent( + new MouseEvent('pointerover', { bubbles: true, clientX: 200, clientY: 200 }) + ) + }) + const tooltip = document.querySelector('[role="tooltip"]')! + expect(tooltip.textContent).toContain(source.title) + expect(tooltip.textContent).toContain(source.url) + expect(link.getAttribute('aria-describedby')).toBe(tooltip.id) + }) + it.each([ { url: 'https://mail.google.com/mail/u/0/#all/thread', @@ -46,6 +68,21 @@ describe('citation labels', () => { siteName: 'Slack', connectorType: 'slack', }, + { + url: 'https://example.slack.com/archives/D123/message', + title: 'Direct message: Release handoff', + connectorType: 'slack', + }, + { + url: 'https://example.slack.com/archives/G123/message', + title: 'Alice, Bob: Release handoff', + connectorType: 'slack', + }, + { + url: 'https://example.com/page', + title: '#engineering: Release handoff', + connectorType: 'confluence', + }, { url: 'https://docs.github.com/page', title: 'Managing repositories', @@ -78,4 +115,16 @@ describe('citation labels', () => { expect(view.textContent?.match(/Launch checklist/g)).toHaveLength(1) expect(view.textContent).toContain('Gmail') }) + + it('keeps the full Slack message title in source cards', () => { + const source = { + url: 'https://example.slack.com/archives/C123/p1789000000000000', + title: '#engineering: Release handoff', + connectorType: 'slack', + } + const view = mount() + const link = view.querySelector('[data-source-link]')! + expect(link.textContent).toBe(source.title) + expect(link.getAttribute('href')).toBe(source.url) + }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/source-chip.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/source-chip.tsx index 8e3ae251d43..5c3c924fc6b 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/source-chip.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/source-chip.tsx @@ -48,6 +48,9 @@ interface SourceChipProps { */ export function SourceChip({ source }: SourceChipProps) { const hostname = externalLinkHostname(source.url) + /** Slack's connector prefixes channel titles with `#channel: `; direct messages omit `#`. */ + const slackChannel = + source.connectorType === 'slack' ? source.title?.match(/^(#[^:\s]+): /)?.[1] : undefined const ConnectorIcon = source.connectorType ? BRAND_ICON_BY_BASE_TYPE.get(source.connectorType) : undefined @@ -76,17 +79,17 @@ export function SourceChip({ source }: SourceChipProps) { onError={hideBrokenFavicon} /> ) : null} - + - + {source.title ? ( - + {source.title} - {source.url} + {source.url} ) : ( - {source.url} + {source.url} )} diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.test.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.test.tsx index 6eb9503c972..8a7edaaa78d 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.test.tsx @@ -72,6 +72,8 @@ vi.mock('@/hooks/use-permission-config', () => ({ ['slack', { oauthAvailable: true, state: 'ready' }], ['slack_v2', { oauthAvailable: true, state: 'ready' }], ['google_drive', { oauthAvailable: true, state: 'ready' }], + ['gmail_v2', { oauthAvailable: true, state: 'ready' }], + ['google_calendar_v2', { oauthAvailable: true, state: 'ready' }], ['confluence_v2', { oauthAvailable: true, state: 'ready' }], ]), oauthServiceAvailability: new Map( @@ -807,6 +809,54 @@ describe('Search setup options', () => { }) describe('Account connection dropdown', () => { + it.each(['google_drive', 'gmail', 'google_calendar'])( + 'opens only service-account creation for a central %s source and submits that credential', + async (connectorType) => { + mocks.credentials = [ + { id: 'personal-account', name: 'Personal Google account', type: 'oauth' }, + ] + mocks.serviceAccountTarget = { + serviceAccountProviderId: 'google-service-account', + serviceName: 'Google', + serviceIcon: googleDriveConnectorMeta.icon, + label: 'Add service account', + hidden: false, + } + mocks.resolveSourceConfig.mockReturnValue({ adminEmail: 'admin@example.com' }) + await render({ + initialConnectorType: connectorType, + lockedAccessMode: 'admin', + scope: { kind: 'organization', organizationId: 'org-1' }, + }) + expect(button('Connect & Sync')).toBeDisabled() + await act(async () => combobox('Select a service account').click()) + const options = Array.from(document.querySelectorAll('[role="option"]')) + expect(options.map((option) => option.textContent?.trim())).toEqual(['Add service account']) + await act(async () => + options[0].dispatchEvent(new MouseEvent('mousedown', { bubbles: true })) + ) + expect(mocks.serviceAccountModal).toHaveBeenLastCalledWith( + expect.objectContaining({ + organizationId: 'org-1', + serviceAccountProviderId: 'google-service-account', + }) + ) + await act(async () => button('Finish service account setup').click()) + expect(combobox('New service account')).toHaveAttribute('aria-disabled', 'false') + await act(async () => button('Connect & Sync').click()) + expect(mocks.create).toHaveBeenCalledWith( + expect.objectContaining({ + connectorType, + credentialId: 'new-service-account', + accessMode: 'admin', + sourceConfig: { adminEmail: 'admin@example.com' }, + }), + expect.any(Object) + ) + expect(mocks.oauthModal).not.toHaveBeenCalled() + } + ) + it('only offers service accounts when creating a central Confluence source', async () => { mocks.credentials = [{ id: 'personal-account', name: 'Personal Confluence', type: 'oauth' }] mocks.serviceAccountTarget = { diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx index 915509b8e27..5b5a2eda369 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx @@ -286,7 +286,9 @@ export function AddConnectorModal({ } = useConnectorConfigFields({ connectorConfig, accessMode: access.accessMode, - initialSourceConfig: draft?.sourceConfig, + initialSourceConfig: isSearchIndex + ? { ...connectorConfig?.searchDefaultSourceConfig, ...draft?.sourceConfig } + : draft?.sourceConfig, initialCanonicalModes: draft?.canonicalModes, initialSelectionLabels: draft?.selectionLabels, }) @@ -423,7 +425,9 @@ export function AddConnectorModal({ const handleSelectType = (type: string) => { if (setupDraftKey) useConnectorSetupStore.getState().clearDraft(setupDraftKey) setSelectedType(type) - setSourceConfig({}) + setSourceConfig( + isSearchIndex ? { ...CONNECTOR_META_REGISTRY[type]?.searchDefaultSourceConfig } : {} + ) setSelectedCredentialId(null) setContentCredentialId(null) setAccess( diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-config-fields/connector-config-fields.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-config-fields/connector-config-fields.tsx index 4bef71c5123..3e088bbdd0f 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-config-fields/connector-config-fields.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-config-fields/connector-config-fields.tsx @@ -70,6 +70,11 @@ export function ConnectorConfigFields({ {connectorConfig.configFields.map((field) => { if (!isFieldVisible(field)) return null + const title = accessMode === 'admin' ? (field.titleInAdminMode ?? field.title) : field.title + const description = + accessMode === 'admin' + ? (field.descriptionInAdminMode ?? field.description) + : field.description const canonicalId = field.canonicalParamId const hasCanonicalPair = canonicalId && (canonicalGroups.get(canonicalId)?.length ?? 0) === 2 @@ -91,24 +96,24 @@ export function ConnectorConfigFields({ > - {field.title} + {title} {isConnectorFieldRequired(field, connectorConfig, accessMode) && ( * )} - {field.description && ( + {description && ( - {field.description} + {description} )} @@ -120,7 +125,7 @@ export function ConnectorConfigFields({ variant='quiet' size='icon' disabled={disabled} - aria-label={`Switch ${field.title} to ${field.mode === 'basic' ? 'manual input' : 'selector'}`} + aria-label={`Switch ${title} to ${field.mode === 'basic' ? 'manual input' : 'selector'}`} onClick={() => onToggleCanonicalMode(canonicalId)} > @@ -167,7 +172,7 @@ export function ConnectorConfigFields({ : undefined } onChange={(value) => onFieldChange(field.id, value)} - placeholder={field.placeholder || `Select ${field.title.toLowerCase()}`} + placeholder={field.placeholder || `Select ${title.toLowerCase()}`} /> ) : ( void + onMultiSelectChange?: (value: string[]) => void +} + +const mocks = vi.hoisted(() => ({ + execute: vi.fn(), + combobox: vi.fn((_props: ComboboxProps) => null), +})) + +vi.mock('@sim/emcn', () => ({ ChipCombobox: mocks.combobox })) +vi.mock('next/navigation', () => ({ useParams: () => ({ workspaceId: 'workspace-1' }) })) +vi.mock('@/hooks/use-debounce', () => ({ useDebounce: (value: string) => value })) +vi.mock('@/lib/selectors/client/execute-selector', () => ({ + executeSelectorRequest: mocks.execute, +})) + +import { ConnectorSelectorField } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field' + +const field: ConnectorConfigField & { selectorKey: 'confluence.spaces' } = { + id: 'spaces', + title: 'Spaces', + type: 'selector', + selectorKey: 'confluence.spaces', + dependsOn: ['domain'], +} +const domainField: ConnectorConfigField = { id: 'domain', title: 'Domain', type: 'short-input' } +const options = [ + { id: 'ENG', label: 'Engineering (ENG)' }, + { id: 'OPS', label: 'Operations (OPS)' }, +] + +beforeEach(() => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + vi.clearAllMocks() + mocks.execute.mockImplementation(async ({ request }: { request: SelectorRequest }) => + request.kind === 'list' + ? { kind: 'list', items: options } + : { kind: 'detail', item: { id: request.id, label: `Saved ${request.id}` } } + ) +}) + +it.each([true, false])( + 'selects loaded options without requests or list loading (multi: %s)', + async (multi) => { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + const root = createRoot(document.createElement('div')) + function Harness() { + const [value, setValue] = useState(multi ? [] : '') + return ( + setValue(nextValue)} + credentialId='credential-1' + sourceConfig={{ domain: 'acme.atlassian.net', spaces: value }} + configFields={[domainField, field]} + canonicalModes={{}} + /> + ) + } + try { + await act(async () => + root.render( + + + + ) + ) + await act(async () => + vi.waitFor(() => expect(mocks.combobox.mock.lastCall?.[0].options).toHaveLength(2), { + interval: 1, + }) + ) + let current = mocks.combobox.mock.lastCall![0] + mocks.combobox.mockClear() + for (const selection of [['ENG'], ['ENG', 'OPS'], ['OPS'], []]) { + await act(async () => { + if (multi) current.onMultiSelectChange?.(selection) + else current.onChange(selection.at(-1) ?? '') + }) + current = mocks.combobox.mock.lastCall![0] + } + expect(mocks.execute).toHaveBeenCalledTimes(1) + expect(mocks.combobox.mock.calls.length).toBeGreaterThanOrEqual(multi ? 4 : 3) + for (const [props] of mocks.combobox.mock.calls) { + expect(props.isLoading).toBe(false) + expect(props.options).toEqual(options.map(({ id, label }) => ({ value: id, label }))) + } + } finally { + await act(async () => root.unmount()) + client.clear() + } + } +) + +it('keeps the loaded list visible while resolving a saved selection on another page', async () => { + let resolveDetail!: (result: SelectorResult) => void + const pendingDetail = new Promise((resolve) => { + resolveDetail = resolve + }) + mocks.execute.mockImplementation(({ request }: { request: SelectorRequest }) => + request.kind === 'list' ? Promise.resolve({ kind: 'list', items: options }) : pendingDetail + ) + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + const root = createRoot(document.createElement('div')) + try { + await act(async () => + root.render( + + + + ) + ) + await act(async () => + vi.waitFor( + () => { + const props = mocks.combobox.mock.lastCall![0] + expect(props.options).toHaveLength(2) + expect(props.isLoading).toBe(false) + }, + { interval: 1 } + ) + ) + await act(async () => + resolveDetail({ kind: 'detail', item: { id: 'OLD', label: 'Saved OLD' } }) + ) + await act(async () => + vi.waitFor( + () => + expect(mocks.combobox.mock.lastCall?.[0].options).toContainEqual({ + value: 'OLD', + label: 'Saved OLD', + }), + { interval: 1 } + ) + ) + expect(mocks.execute).toHaveBeenCalledWith( + expect.objectContaining({ request: { kind: 'detail', id: 'OLD' } }) + ) + } finally { + await act(async () => root.unmount()) + client.clear() + } +}) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.tsx index d4ab15a4138..8a02e6a09d1 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.tsx @@ -114,22 +114,21 @@ export function ConnectorSelectorField({ surfaceId: `connector:${field.id}`, }) - /** - * Label every selected value, including values restored from saved config that no - * in-session search would have resolved. Opaque revisions bind each label request to - * the active context without placing credential or dependency values in its query key. - */ const singleValue = Array.isArray(value) ? value[0] : value const selectedIds = useMemo( () => (Array.isArray(value) ? value : value ? [value] : []).filter(Boolean), [value] ) + const missingSelectedIds = useMemo(() => { + const loadedIds = new Set(options.map((option) => option.id)) + return selectedIds.filter((id) => !loadedIds.has(id)) + }, [options, selectedIds]) const { data: selectedOptions, isLoading: isLoadingSelectedOptions } = useSelectorOptionDetails( field.selectorKey, { context, scope, - detailIds: isEnabled ? selectedIds : [], + detailIds: isEnabled ? missingSelectedIds : [], surfaceId: `connector:${field.id}`, } ) @@ -198,7 +197,7 @@ export function ConnectorSelectorField({ : field.placeholder || `Select ${field.title.toLowerCase()}` } disabled={disabled || !credentialId || !depsResolved} - isLoading={isEnabled && (isLoading || isLoadingSelectedOptions)} + isLoading={isEnabled && (isLoading || (options.length === 0 && isLoadingSelectedOptions))} hasMore={hasMore} isLoadingMore={isFetchingMore} isLoadingAll={isLoadingAll} @@ -226,7 +225,7 @@ export function ConnectorSelectorField({ : field.placeholder || `Select ${field.title.toLowerCase()}` } disabled={disabled || !credentialId || !depsResolved} - isLoading={isEnabled && (isLoading || isLoadingSelectedOptions)} + isLoading={isEnabled && (isLoading || (options.length === 0 && isLoadingSelectedOptions))} hasMore={hasMore} isLoadingMore={isFetchingMore} isLoadingAll={isLoadingAll} diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.test.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.test.tsx index c36bca66312..eb0a2a7d9ac 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.test.tsx @@ -366,7 +366,7 @@ describe('connector settings service-account choices', () => { }, ] await render(confluenceConnectorMeta) - expect(container.textContent).toContain('Service account') + expect(container.textContent).toContain('Indexing account') await openAccountChoices() expect( Array.from(document.querySelectorAll('[role="option"]')).map((node) => diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.tsx index 65f3abd95eb..2f4ed40efcd 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.tsx @@ -280,11 +280,7 @@ export function ConnectorSettingsFields({ {connectorConfig && needsWorkspaceCredential && canAdmin && ( ({ GmailIcon: () => null, GoogleDriveIcon: () => null })) +vi.mock('@/components/icons', () => ({ + GmailIcon: () => null, + GoogleCalendarIcon: () => null, + GoogleDriveIcon: () => null, +})) import { describeSearchSource, SOURCE_LABELS_KEY } from '@/lib/sim-search/source-identity' import { @@ -14,6 +18,7 @@ import { useConnectorConfigFields, } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields' import { gmailConnectorMeta } from '@/connectors/gmail/meta' +import { googleCalendarConnectorMeta } from '@/connectors/google-calendar/meta' import { googleDriveConnectorMeta } from '@/connectors/google-drive/meta' import type { ConnectorMeta } from '@/connectors/types' @@ -49,13 +54,16 @@ describe('useConnectorConfigFields member configuration', () => { container.remove() }) - it('offers only manual label names for member Gmail setup', () => { - render({ accessMode: 'members' }) + it.each(['members', 'admin'] as const)( + 'offers manual label names for %s Gmail setup', + (accessMode) => { + render({ accessMode }) - expect(visibleLabelFields()).toEqual(['label']) - expect(current!.canonicalModes.label).toBe('advanced') - expect(current!.canonicalGroups.get('label')?.map((field) => field.id)).toEqual(['label']) - }) + expect(visibleLabelFields()).toEqual(['label']) + expect(current!.canonicalModes.label).toBe('advanced') + expect(current!.canonicalGroups.get('label')?.map((field) => field.id)).toEqual(['label']) + } + ) it('resolves manual names and system IDs through the existing canonical label field', () => { render({ accessMode: 'members' }) @@ -85,17 +93,20 @@ describe('useConnectorConfigFields member configuration', () => { expect(current!.resolveSourceConfig()).toMatchObject({ label: ['INBOX', 'Label_7'] }) }) - it('keeps a visible manual field when a saved member draft selected basic mode', () => { - render({ - accessMode: 'members', - initialCanonicalModes: { label: 'basic' }, - initialSourceConfig: { labelSelector: ['Label_7'], label: ['Engineering'] }, - }) - - expect(visibleLabelFields()).toEqual(['label']) - expect(current!.canonicalModes.label).toBe('advanced') - expect(current!.resolveSourceConfig()).toMatchObject({ label: ['Engineering'] }) - }) + it.each(['members', 'admin'] as const)( + 'keeps a visible manual field when a saved %s draft selected basic mode', + (accessMode) => { + render({ + accessMode, + initialCanonicalModes: { label: 'basic' }, + initialSourceConfig: { labelSelector: ['Label_7'], label: ['Engineering'] }, + }) + + expect(visibleLabelFields()).toEqual(['label']) + expect(current!.canonicalModes.label).toBe('advanced') + expect(current!.resolveSourceConfig()).toMatchObject({ label: ['Engineering'] }) + } + ) it('keeps fields visible and preserves edits when switching access modes without remounting', () => { render({ @@ -170,6 +181,56 @@ describe('useConnectorConfigFields member configuration', () => { expect(current!.resolveSourceConfig()).toMatchObject({ openSharing: 'domain' }) }) + it.each([googleDriveConnectorMeta, googleCalendarConnectorMeta, gmailConnectorMeta])( + 'offers directory user selection only for central $name crawls', + (connectorConfig) => { + const field = connectorConfig.configFields.find((field) => field.id === 'userEmails')! + render({ connectorConfig, accessMode: 'admin' }) + expect(current.isFieldVisible(field)).toBe(true) + act(() => current.handleFieldChange('userEmails', 'first@example.com, second@example.com')) + expect(current.resolveSourceConfig().userEmails).toEqual([ + 'first@example.com', + 'second@example.com', + ]) + + render({ connectorConfig, accessMode: 'members' }) + expect(current.isFieldVisible(field)).toBe(false) + render({ connectorConfig, accessMode: 'workspace' }) + expect(current.isFieldVisible(field)).toBe(false) + render({ connectorConfig, accessMode: 'admin' }) + expect(current.isFieldVisible(field)).toBe(true) + expect(current.resolveSourceConfig().userEmails).toEqual([ + 'first@example.com', + 'second@example.com', + ]) + } + ) + + it('uses manual calendar IDs centrally without reusing a saved administrator calendar selection', () => { + render({ + connectorConfig: googleCalendarConnectorMeta, + accessMode: 'admin', + initialCanonicalModes: { calendarId: 'basic' }, + initialSourceConfig: { + calendarSelector: ['administrator@example.com'], + calendarId: ['primary', 'shared@group.calendar.google.com'], + }, + }) + const visibleCalendars = () => + googleCalendarConnectorMeta.configFields + .filter((field) => field.canonicalParamId === 'calendarId' && current.isFieldVisible(field)) + .map((field) => field.id) + expect(visibleCalendars()).toEqual(['calendarId']) + expect(current.resolveSourceConfig().calendarId).toEqual([ + 'primary', + 'shared@group.calendar.google.com', + ]) + + render({ connectorConfig: googleCalendarConnectorMeta, accessMode: 'members' }) + expect(visibleCalendars()).toEqual(['calendarSelector']) + expect(current.resolveSourceConfig().calendarId).toEqual(['administrator@example.com']) + }) + it('persists selector labels with the canonical IDs without changing provider values', () => { render({ connectorConfig: googleDriveConnectorMeta }) act(() => diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields.ts index 66b298d6688..6d8619ec5fa 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields.ts @@ -109,6 +109,8 @@ export function useConnectorConfigFields({ if (!connectorConfig) return groups for (const field of connectorConfig.configFields) { if (accessMode === 'members' && field.hideInMemberMode) continue + if (accessMode === 'admin' && field.hideInAdminMode) continue + if (accessMode !== 'admin' && field.showInAdminModeOnly) continue if (!field.canonicalParamId) continue const existing = groups.get(field.canonicalParamId) if (existing) existing.push(field) @@ -177,6 +179,8 @@ export function useConnectorConfigFields({ const isFieldVisible = useCallback( (field: ConnectorConfigField): boolean => { if (accessMode === 'members' && field.hideInMemberMode) return false + if (accessMode === 'admin' && field.hideInAdminMode) return false + if (accessMode !== 'admin' && field.showInAdminModeOnly) return false if (!field.canonicalParamId || !field.mode) return true const activeMode = canonicalModes[field.canonicalParamId] ?? 'basic' return field.mode === activeMode diff --git a/apps/sim/app/workspace/[workspaceId]/search/components/search-source-setup.test.tsx b/apps/sim/app/workspace/[workspaceId]/search/components/search-source-setup.test.tsx index 39d1feed7da..41900dd4c2a 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/components/search-source-setup.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/components/search-source-setup.test.tsx @@ -374,6 +374,99 @@ function organizationSetup() { } describe('organization setup entry points', () => { + it.each([ + { type: 'google_drive', mode: 'admin', name: 'Google Drive' }, + { type: 'gmail', mode: 'admin', name: 'Gmail' }, + { type: 'google_calendar', mode: 'admin', name: 'Google Calendar' }, + { type: 'confluence', mode: 'admin', name: 'Confluence' }, + { type: 'gitlab', mode: 'admin', name: 'GitLab' }, + { type: 'gmail', mode: 'members', name: 'Gmail' }, + { type: 'google_calendar', mode: 'members', name: 'Google Calendar' }, + { type: 'jira', mode: 'members', name: 'Jira' }, + { type: 'github', mode: 'members', name: 'GitHub' }, + { type: 'slack', mode: 'members', name: 'Slack' }, + ])( + 'opens the known $name configuration after preparing its missing index', + async ({ type, mode, name }) => { + mocks.bases = [] + const query = `?addConnector=${type}&source-access=${mode}` + await render(organizationSetup(), query) + expect(mocks.prepare).toHaveBeenCalledExactlyOnceWith({ + organizationId: 'org-1', + connectorType: type, + accessMode: mode, + }) + expect(document.body.textContent).toContain(`Configure ${name}`) + expect(document.body.textContent).not.toContain('Continue setup') + expect(document.body.textContent).not.toContain('Find a source') + await render(organizationSetup(), query) + expect(mocks.prepare).toHaveBeenCalledOnce() + mocks.bases = [{ id: 'kb-search', name: 'Sim Search', isSearchIndex: true }] + await render(organizationSetup(), query) + expect(document.body.textContent).toContain(`Configure ${name}`) + expect(document.body.textContent).not.toContain('Loading source setup') + expect(document.querySelector('button[aria-label="Choose another source"]')).toBeNull() + expect(mocks.prepare).toHaveBeenCalledOnce() + } + ) + + it('waits for index discovery without showing a redundant provider row', async () => { + mocks.bases = [] + mocks.basesPending = true + await render(organizationSetup(), '?addConnector=google_drive') + expect(mocks.prepare).not.toHaveBeenCalled() + expect(document.body.textContent).toContain('Loading source setup') + expect(document.body.textContent).not.toContain('Continue setup') + mocks.basesPending = false + mocks.bases = [{ id: 'kb-search', name: 'Sim Search', isSearchIndex: true }] + await render(organizationSetup(), '?addConnector=google_drive') + expect(mocks.prepare).not.toHaveBeenCalled() + expect(document.body.textContent).not.toContain('Loading source setup') + }) + + it.each(['loading', 'error'] as const)( + 'does not prepare while availability is %s', + async (state) => { + mocks.bases = [] + mocks.availabilityReady = false + mocks.availabilityLoading = state === 'loading' + mocks.availabilityError = state === 'error' ? new Error('Availability failed') : null + await render(organizationSetup(), '?addConnector=google_drive') + expect(mocks.prepare).not.toHaveBeenCalled() + expect(document.body.textContent).not.toContain('Continue setup') + } + ) + + it('keeps the broad provider picker without automatically preparing an index', async () => { + mocks.bases = [] + await render(setup(), '?addConnector=') + expect(document.body.textContent).toContain('Find a source') + expect(document.body.textContent).toContain('Google Drive') + expect(document.body.textContent).toContain('Confluence') + expect(mocks.prepare).not.toHaveBeenCalled() + }) + + it('retries failed preparation only when requested', async () => { + mocks.bases = [] + await render(organizationSetup(), '?addConnector=google_drive') + mocks.prepareError = new Error('Could not prepare the Search index') + await render(organizationSetup(), '?addConnector=google_drive') + expect(mocks.prepare).toHaveBeenCalledOnce() + expect(document.body.textContent).toContain('Could not prepare the Search index') + await click(button('Try again')) + expect(mocks.prepare).toHaveBeenCalledTimes(2) + }) + + it('does not prepare after the selected setup is closed while index discovery finishes', async () => { + mocks.bases = [] + mocks.basesPending = true + await render(organizationSetup(), '?addConnector=google_drive') + await render(organizationSetup()) + mocks.basesPending = false + await render(organizationSetup()) + expect(mocks.prepare).not.toHaveBeenCalled() + }) + it('uses central mode and its own draft even when the saved draft contains member mode', async () => { mocks.credentials = [ { @@ -417,11 +510,18 @@ describe('organization setup entry points', () => { accessMode: 'admin', }) await act(async () => mocks.create.mock.calls[0][1].onSuccess(created)) - expect(mocks.urlUpdate).toHaveBeenLastCalledWith(expect.objectContaining({ queryString: '' })) - expect(mocks.push).toHaveBeenCalledWith('/o/org-1/settings/integrations/sources/new-source') + await vi.waitFor( + () => { + expect(mocks.urlUpdate).toHaveBeenLastCalledWith( + expect.objectContaining({ queryString: '' }) + ) + expect(mocks.push).toHaveBeenCalledWith('/o/org-1/settings/integrations/sources/new-source') + }, + { interval: 1 } + ) }) - it.each(['github', 'gmail', 'google_calendar', 'jira'])( + it.each(['github', 'jira'])( 'returns old %s organization setup links to personal integrations without loading the index', async (type) => { await render(organizationSetup(), `?addConnector=${type}`) @@ -433,14 +533,70 @@ describe('organization setup entry points', () => { } ) + it.each(['google_drive', 'gmail', 'google_calendar'])( + 'opens %s central setup directly and keeps explicit member links in member mode', + async (type) => { + mocks.bases = [] + await render(organizationSetup(), `?addConnector=${type}`) + expect(mocks.prepare).toHaveBeenLastCalledWith({ + organizationId: 'org-1', + connectorType: type, + accessMode: 'admin', + }) + expect(mocks.replace).not.toHaveBeenCalled() + expect(document.body.textContent).not.toContain('Continue setup') + mocks.bases = [{ id: 'kb-search', name: 'Sim Search', isSearchIndex: true }] + await render(organizationSetup(), `?addConnector=${type}`) + expect(button('Connect & Sync')).toBeDisabled() + expect(document.body.textContent).not.toContain('Sync using') + await render(organizationSetup(), `?addConnector=${type}&source-access=members`) + expect(button('Add source')).toBeEnabled() + expect(document.body.textContent).not.toContain('Connect & Sync') + expect(document.body.textContent).not.toContain('Directory administrator email') + await click(button('Add source')) + expect(mocks.create).toHaveBeenCalledWith( + expect.objectContaining({ + connectorType: type, + accessMode: 'members', + credentialId: undefined, + }), + expect.any(Object) + ) + } + ) + + it.each([ + { type: 'google_drive', provider: 'google-drive', block: 'google_drive' }, + { type: 'gmail', provider: 'google-email', block: 'gmail_v2' }, + { type: 'google_calendar', provider: 'google-calendar', block: 'google_calendar_v2' }, + ])( + 'prepares central $type without a personal OAuth rollout and refuses a disabled member entry', + async ({ type, provider, block }) => { + mocks.bases = [] + mocks.features = { knowledgeMemberAccess: false, knowledgeSourceMirroredAccess: true } + mocks.unavailableProviders = [provider] + mocks.integrationAvailability.set(block, { oauthAvailable: false, state: 'limited' }) + await render(organizationSetup(), `?addConnector=${type}`) + expect(mocks.prepare).toHaveBeenCalledExactlyOnceWith({ + organizationId: 'org-1', + connectorType: type, + accessMode: 'admin', + }) + expect(document.body.textContent).not.toContain('Not available in this organization') + await render(organizationSetup(), `?addConnector=${type}&source-access=members`) + expect(document.body.textContent).toContain('Not available in this organization') + expect(mocks.prepare).toHaveBeenCalledOnce() + } + ) + it('prepares explicit member sources under the organization without switching them to central mode', async () => { mocks.bases = [] await render(organizationSetup(), '?addConnector=confluence&source-access=members') - await click(button('Continue setup')) - expect(mocks.prepare).toHaveBeenCalledWith( - { organizationId: 'org-1', connectorType: 'confluence', accessMode: 'members' }, - expect.any(Object) - ) + expect(mocks.prepare).toHaveBeenCalledWith({ + organizationId: 'org-1', + connectorType: 'confluence', + accessMode: 'members', + }) }) it.each([ @@ -603,13 +759,16 @@ describe('Search source setup with real connector dialogs', () => { }) it.each(['gmail', 'jira', 'github', 'google_calendar'])( - 'sets up %s with member access and no shared workspace or admin mode', + 'retains %s member setup without workspace-wide access', async (type) => { await render(setup(), `?addConnector=${type}`) + if (type === 'gmail' || type === 'google_calendar') { + await click(button('Member accounts')) + } expect(document.body.textContent).toContain('Member accounts') expect( - Array.from(document.querySelectorAll('button')).some((node) => - ['Workspace', 'Admin or service account'].includes(node.textContent ?? '') + Array.from(document.querySelectorAll('button')).some( + (node) => node.textContent === 'Workspace' ) ).toBe(false) expect(document.body.textContent).not.toContain('Sync Frequency') @@ -651,11 +810,11 @@ describe('Search source setup with real connector dialogs', () => { it('prepares a canonical index instead of an ordinary base with the Search name', async () => { mocks.bases = [{ id: 'ordinary-base', name: 'Sim Search', isSearchIndex: false }] await render(setup(), '?addConnector=gitlab') - await click(button('Continue setup')) - expect(mocks.prepare).toHaveBeenCalledWith( - { workspaceId: 'workspace-1', connectorType: 'gitlab', accessMode: 'admin' }, - expect.any(Object) - ) + expect(mocks.prepare).toHaveBeenCalledWith({ + workspaceId: 'workspace-1', + connectorType: 'gitlab', + accessMode: 'admin', + }) expect(mocks.connectorsQuery).toHaveBeenLastCalledWith(undefined) }) @@ -670,18 +829,17 @@ describe('Search source setup with real connector dialogs', () => { />, '?addConnector=slack' ) - await click(button('Continue setup')) - expect(mocks.prepare).toHaveBeenCalledWith( - { organizationId: 'org-1', connectorType: 'slack', accessMode: 'members' }, - expect.any(Object) - ) + expect(mocks.prepare).toHaveBeenCalledWith({ + organizationId: 'org-1', + connectorType: 'slack', + accessMode: 'members', + }) }) it('does not reuse mutation data after the current index has been removed', async () => { mocks.prepareData = { knowledgeBaseId: 'kb-search' } mocks.bases = [] await render(setup(), '?addConnector=gitlab') - await click(button('Continue setup')) expect(mocks.prepare).toHaveBeenCalled() expect(document.querySelector('input[placeholder="Enter your GitLab PAT"]')).toBeNull() }) @@ -814,11 +972,11 @@ describe('Search source setup with real connector dialogs', () => { mocks.features.knowledgeSourceMirroredAccess = false mocks.bases = [] await render(setup(), '?addConnector=slack') - await click(button('Continue setup')) - expect(mocks.prepare).toHaveBeenCalledWith( - { workspaceId: 'workspace-1', connectorType: 'slack', accessMode: 'members' }, - expect.any(Object) - ) + expect(mocks.prepare).toHaveBeenCalledWith({ + workspaceId: 'workspace-1', + connectorType: 'slack', + accessMode: 'members', + }) }) it('blocks unavailable catalog providers and duplicate preparation while preserving retry feedback', async () => { @@ -1119,6 +1277,131 @@ describe('administrator source prerequisites in real connector dialogs', () => { mocks.credentials = [driveCredential] }) + it.each(['admin', 'members'] as const)( + 'shows and saves Gmail’s Search default date window in %s mode', + async (accessMode) => { + mocks.credentials = [ + { + id: 'gmail-service', + name: 'Gmail indexing', + provider: 'google-email', + type: 'service_account', + }, + ] + await render( + + ) + expect(document.body.textContent).toContain('Last 6 months') + expect(document.body.textContent).not.toContain('All time (default)') + if (accessMode === 'admin') await fill(adminEmailPlaceholder, 'admin@example.com') + await click(button(accessMode === 'admin' ? 'Connect & Sync' : 'Create & Invite')) + expect(mocks.create).toHaveBeenCalledWith( + expect.objectContaining({ + accessMode, + connectorType: 'gmail', + sourceConfig: expect.objectContaining({ dateRange: '6m' }), + }), + expect.any(Object) + ) + } + ) + + it('preserves a deliberate Gmail date-range draft and keeps general KB defaults separate', async () => { + const key = 'gmail-all-time' + useConnectorSetupStore.getState().saveDraft(key, { + sourceConfig: { dateRange: 'all' }, + canonicalModes: {}, + accessMode: 'members', + credentialId: null, + contentCredentialId: null, + disabledTagIds: [], + savedAt: Date.now(), + }) + await render( + + ) + expect(document.body.textContent).toContain('All time') + expect(document.body.textContent).not.toContain('Last 6 months') + await click(button('Create & Invite')) + expect(mocks.create).toHaveBeenCalledWith( + expect.objectContaining({ sourceConfig: expect.objectContaining({ dateRange: 'all' }) }), + expect.any(Object) + ) + await render( + + ) + expect(document.body.textContent).toContain('All time (default)') + expect(document.body.textContent).not.toContain('Last 6 months') + }) + + it('initializes Search defaults when Gmail is selected from the broad source picker', async () => { + await render( + + ) + const source = document.querySelector('button[aria-label="Gmail"]') + expect(source).not.toBeNull() + await click(source!) + expect(document.body.textContent).toContain('Last 6 months') + await click(button('Create & Invite')) + expect(mocks.create).toHaveBeenCalledWith( + expect.objectContaining({ + connectorType: 'gmail', + sourceConfig: expect.objectContaining({ dateRange: '6m' }), + }), + expect.any(Object) + ) + }) + + it('does not reapply the default date window over a user edit on later renders', async () => { + const modal = ( + + ) + await render(modal) + await chooseCombo('Last 6 months', 'All time') + await render(cloneElement(modal)) + expect(document.body.textContent).toContain('All time') + expect(document.body.textContent).not.toContain('Last 6 months') + await click(button('More options')) + await click(button('More options')) + await click(button('Create & Invite')) + expect(mocks.create).toHaveBeenCalledWith( + expect.objectContaining({ + sourceConfig: expect.objectContaining({ dateRange: 'all' }), + }), + expect.any(Object) + ) + }) + it.each(['ready', 'limited', 'unavailable', 'misconfigured'] as const)( 'uses canonical Confluence availability for inline service-account setup when %s', async (state) => { @@ -1173,88 +1456,103 @@ describe('administrator source prerequisites in real connector dialogs', () => { } ) - it('marks Crawl as required in Drive administrator mode and refuses empty or blank subjects', async () => { - await render( - - ) - expect(document.body.textContent).toContain('Crawl as*') - expect(button('Connect & Sync')).toBeDisabled() - await click(button('Connect & Sync')) - expect(mocks.create).not.toHaveBeenCalled() - await fill(adminEmailPlaceholder, ' ') - expect(button('Connect & Sync')).toBeDisabled() + it.each([ + { type: 'google_drive', provider: 'google-drive' }, + { type: 'gmail', provider: 'google-email' }, + { type: 'google_calendar', provider: 'google-calendar' }, + ])( + 'requires the Directory administrator email in $type administrator mode and refuses empty or blank subjects', + async ({ type, provider }) => { + mocks.credentials = [{ ...driveCredential, provider }] + await render( + + ) + expect(document.body.textContent).toContain('Directory administrator email*') + expect(button('Connect & Sync')).toBeDisabled() + await click(button('Connect & Sync')) + expect(mocks.create).not.toHaveBeenCalled() + await fill(adminEmailPlaceholder, ' ') + expect(button('Connect & Sync')).toBeDisabled() + + await fill(adminEmailPlaceholder, 'admin@example.com') + expect(button('Connect & Sync')).toBeEnabled() + await click(button('Connect & Sync')) + expect(mocks.create).toHaveBeenCalledWith( + expect.objectContaining({ + connectorType: type, + accessMode: 'admin', + credentialId: driveCredential.id, + sourceConfig: expect.objectContaining({ adminEmail: 'admin@example.com' }), + }), + expect.any(Object) + ) + } + ) - await fill(adminEmailPlaceholder, 'admin@example.com') - expect(button('Connect & Sync')).toBeEnabled() - await click(button('Connect & Sync')) - expect(mocks.create).toHaveBeenCalledWith( - expect.objectContaining({ - connectorType: 'google_drive', + it.each([ + { type: 'google_drive', provider: 'google-drive', name: 'Google Drive' }, + { type: 'gmail', provider: 'google-email', name: 'Gmail' }, + { type: 'google_calendar', provider: 'google-calendar', name: 'Google Calendar' }, + ])( + 'excludes personal OAuth accounts and stale OAuth drafts from $type administrator setup', + async ({ type, provider, name }) => { + const oauthCredential = { + id: 'drive-personal', + name: 'Personal Drive account', + provider, + type: 'oauth' as const, + } + mocks.credentials = [oauthCredential] + const setupDraftKey = `user-1:workspace-1:kb-search:${type}` + useConnectorSetupStore.getState().saveDraft(setupDraftKey, { + sourceConfig: { adminEmail: 'admin@example.com' }, + canonicalModes: {}, accessMode: 'admin', - credentialId: driveCredential.id, - sourceConfig: expect.objectContaining({ adminEmail: 'admin@example.com' }), - }), - expect.any(Object) - ) - }) - - it('excludes personal OAuth accounts and stale OAuth drafts from Drive administrator setup', async () => { - const oauthCredential = { - id: 'drive-personal', - name: 'Personal Drive account', - provider: 'google-drive', - type: 'oauth' as const, + credentialId: oauthCredential.id, + contentCredentialId: null, + disabledTagIds: [], + savedAt: Date.now(), + }) + const modal = ( + + ) + await render(modal) + expect(document.body.textContent).toContain('Service account') + expect(document.body.textContent).not.toContain(oauthCredential.name) + expect(button('Connect & Sync')).toBeDisabled() + const picker = Array.from(document.querySelectorAll('[role="combobox"]')).find( + (node) => node.textContent?.includes('Select a service account') + )! + await click(picker) + expect(document.body.textContent).not.toContain(`Connect ${name} account`) + expect(document.body.textContent).not.toContain(oauthCredential.name) + await click(picker) + mocks.credentials = [oauthCredential, { ...driveCredential, provider }] + await render(cloneElement(modal)) + + expect(button('Connect & Sync')).toBeEnabled() + await click(button('Connect & Sync')) + + expect(mocks.create).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ credentialId: driveCredential.id, accessMode: 'admin' }), + expect.any(Object) + ) } - mocks.credentials = [oauthCredential] - const setupDraftKey = 'user-1:workspace-1:kb-search:google_drive' - useConnectorSetupStore.getState().saveDraft(setupDraftKey, { - sourceConfig: { adminEmail: 'admin@example.com' }, - canonicalModes: {}, - accessMode: 'admin', - credentialId: oauthCredential.id, - contentCredentialId: null, - disabledTagIds: [], - savedAt: Date.now(), - }) - const modal = ( - - ) - await render(modal) - expect(document.body.textContent).toContain('Service account') - expect(document.body.textContent).not.toContain(oauthCredential.name) - expect(button('Connect & Sync')).toBeDisabled() - const picker = Array.from(document.querySelectorAll('[role="combobox"]')).find( - (node) => node.textContent?.includes('Select a service account') - )! - await click(picker) - expect(document.body.textContent).not.toContain('Connect Google Drive account') - expect(document.body.textContent).not.toContain(oauthCredential.name) - await click(picker) - mocks.credentials = [oauthCredential, driveCredential] - await render(cloneElement(modal)) - - expect(button('Connect & Sync')).toBeEnabled() - await click(button('Connect & Sync')) - - expect(mocks.create).toHaveBeenCalledExactlyOnceWith( - expect.objectContaining({ credentialId: driveCredential.id, accessMode: 'admin' }), - expect.any(Object) - ) - }) + ) it('replaces an existing Drive administrator account through the access operation', async () => { const oauthCredential = { @@ -1346,7 +1644,7 @@ describe('administrator source prerequisites in real connector dialogs', () => { })} /> ) - expect(document.body.textContent).toContain('Crawl as*') + expect(document.body.textContent).toContain('Directory administrator email*') await fill(adminEmailPlaceholder, '') expect(button('Save')).toBeDisabled() await click(button('Save')) diff --git a/apps/sim/app/workspace/[workspaceId]/search/components/search-source-setup.tsx b/apps/sim/app/workspace/[workspaceId]/search/components/search-source-setup.tsx index 1e6838ec7c0..9c46ae6d391 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/components/search-source-setup.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/components/search-source-setup.tsx @@ -1,6 +1,6 @@ 'use client' -import { useEffect, useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { Chip, ChipInput, @@ -102,8 +102,10 @@ export function SearchSourceSetup({ managedSourceParam.parser.withOptions({ history: 'replace' }) ) const [search, setSearch] = useState('') + const attemptedPreparation = useRef(null) const router = useRouter() const prepare = usePrepareSearchSource() + const { mutate: prepareSource, isPending: preparing } = prepare const selectedMeta = selectedType ? CONNECTOR_META_REGISTRY[selectedType] : undefined const redirectPersonalSetup = Boolean( scope.kind === 'organization' && @@ -139,8 +141,6 @@ export function SearchSourceSetup({ canAdmin && managedSource && !redirectManagement ? knowledgeBaseId : undefined ) - if (!canAdmin || !open || redirectManagement || redirectPersonalSetup) return null - const close = () => { if (prepare.isPending) return if (selectedType !== null) void setSelectedType(null) @@ -178,6 +178,61 @@ export function SearchSourceSetup({ return 'members' as const } + const selectedAccessMode = selectedType ? initialMode(selectedType) : undefined + const selectedAvailability = selectedMeta + ? getConnectorAccessAvailability(selectedMeta, integrationAvailability, { + memberAccessAvailable, + mirroredAccessAvailable, + oauthServiceAvailability, + isIntegrationAvailabilityReady, + }) + : undefined + const selectedAvailable = selectedAvailability + ? scope.kind === 'organization' + ? selectedAccessMode === 'admin' + ? selectedAvailability.admin + : selectedAvailability.members + : selectedAvailability.admin || selectedAvailability.members + : false + const canPrepareSelected = + canAdmin && + open && + !redirectManagement && + !redirectPersonalSetup && + !index.isPending && + !index.isError && + !integrationAvailabilityError && + selectedAvailable + const userId = session?.user?.id + + /** A known provider only needs the canonical index prepared, not another selection step. */ + useEffect(() => { + if (!selectedType || knowledgeBaseId) { + attemptedPreparation.current = null + return + } + if (!canPrepareSelected || !selectedAccessMode || !userId || preparing) return + const requestKey = `${userId}:${resourceScopeKey(scope)}:${selectedType}:${selectedAccessMode}` + if (attemptedPreparation.current === requestKey) return + attemptedPreparation.current = requestKey + prepareSource({ + ...resourceScopeFields(scope), + connectorType: selectedType, + accessMode: selectedAccessMode, + }) + }, [ + selectedType, + knowledgeBaseId, + canPrepareSelected, + selectedAccessMode, + userId, + preparing, + scope, + prepareSource, + ]) + + if (!canAdmin || !open || redirectManagement || redirectPersonalSetup) return null + if ( !failedQuery && knowledgeBaseId && @@ -232,9 +287,7 @@ export function SearchSourceSetup({ setup['source-access'] === 'members' || meta.mirrorsSourceAcls || type === 'slack') && - (selectedType - ? type === selectedType - : `${meta.name} ${meta.description}`.toLowerCase().includes(normalizedSearch)) + `${meta.name} ${meta.description}`.toLowerCase().includes(normalizedSearch) ) return ( @@ -280,18 +333,41 @@ export function SearchSourceSetup({ {index.isPending ? 'Loading source…' : 'This source is no longer available.'} + ) : selectedType ? ( + + {!selectedAvailable ? ( + + Not available in this {scope.kind} + + ) : prepare.error ? ( + { + if (!canPrepareSelected || !selectedAccessMode || !userId || preparing) return + prepareSource({ + ...resourceScopeFields(scope), + connectorType: selectedType, + accessMode: selectedAccessMode, + }) + }} + variant='inline' + /> + ) : ( + Loading source setup… + )} + ) : ( <> - {!selectedType && ( - - setSearch(event.target.value)} - /> - - )} + + setSearch(event.target.value)} + /> +
{visibleTypes.map(([type, meta]) => { @@ -347,7 +423,7 @@ export function SearchSourceSetup({ ) }} > - {selectedType ? 'Continue setup' : 'Set up'} + Set up ) : undefined } diff --git a/apps/sim/connectors/confluence/confluence.ts b/apps/sim/connectors/confluence/confluence.ts index a45dad1173b..b64e5e6a27b 100644 --- a/apps/sim/connectors/confluence/confluence.ts +++ b/apps/sim/connectors/confluence/confluence.ts @@ -469,9 +469,8 @@ interface ContentLocation { * configured space: a connector over two spaces must not let a reader of one * into the unrestricted pages of the other. * - * A page whose restrictions could not be read this run is omitted, which the - * engine stores as readable by nobody, and the rest of the batch still - * resolves — the same per-document containment Drive has. + * Unresolved pages are omitted while the rest of the batch completes. The engine + * hides them unless another observation verified their ACL during the same crawl. */ async function resolveConfluenceAcls( accessToken: string, @@ -531,7 +530,7 @@ async function resolveConfluenceAcls( resolved.set(externalId, { spaceId: location.spaceId, chain }) } catch (error) { unreadable += 1 - logger.warn("Could not read a page's permissions; it stays readable by nobody", { + logger.warn("Could not verify a page's permissions", { cloudId, externalId, error: getErrorMessage(error), @@ -554,7 +553,7 @@ async function resolveConfluenceAcls( } if (unreadable > 0) { - logger.warn('Some Confluence pages had unreadable permissions and stay readable by nobody', { + logger.warn('Some Confluence pages had unresolved permissions', { cloudId, unreadable, }) diff --git a/apps/sim/connectors/gmail/company-crawl.test.ts b/apps/sim/connectors/gmail/company-crawl.test.ts new file mode 100644 index 00000000000..6918529e1ce --- /dev/null +++ b/apps/sim/connectors/gmail/company-crawl.test.ts @@ -0,0 +1,515 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { fetchProvider, listUsers, getUser } = vi.hoisted(() => ({ + fetchProvider: vi.fn(), + listUsers: vi.fn(), + getUser: vi.fn(), +})) + +vi.mock('@/lib/knowledge/documents/utils', () => ({ + fetchWithRetry: fetchProvider, + VALIDATE_RETRY_OPTIONS: {}, +})) +vi.mock('@/components/icons', () => ({ GmailIcon: () => null })) +vi.mock('@/connectors/google-workspace/users', () => ({ + GOOGLE_WORKSPACE_USERS_PAGE_SIZE: 100, + listGoogleWorkspaceUsers: listUsers, + getGoogleWorkspaceUser: getUser, + selectedGoogleWorkspaceUsers: (value: unknown) => + typeof value === 'string' + ? value + .split(',') + .map((email) => email.trim()) + .filter(Boolean) + : [], +})) + +import { gmailConnector } from '@/connectors/gmail/gmail' +import { gmailConnectorMeta } from '@/connectors/gmail/meta' + +const ALICE = { + id: 'directory-alice', + email: 'alice@example.com', + customerId: 'customer-1', + active: true, +} +const BOB = { + id: 'directory-bob', + email: 'bob@example.com', + customerId: 'customer-1', + active: true, +} +const ADMIN = { + id: 'directory-admin', + email: 'admin@example.com', + customerId: 'customer-1', + active: true, +} +const CONFIG = { adminEmail: ADMIN.email, dateRange: '6m' } + +function centralContext() { + return { + mirrorsSourceAcls: true, + getDelegatedAccessToken: vi.fn(async (email: string) => `delegated:${email}`), + } +} + +function providerResponse(url: string, init?: RequestInit): Response { + const parsed = new URL(url) + const token = new Headers(init?.headers).get('Authorization') + const mailbox = token?.includes(BOB.email) ? 'Bob' : 'Alice' + if (parsed.pathname.endsWith('/profile')) return Response.json({ emailAddress: ALICE.email }) + if (parsed.pathname.endsWith('/labels')) { + return Response.json({ labels: [{ id: 'Label_7', name: `${mailbox} label` }] }) + } + if (parsed.pathname.endsWith('/threads')) { + return Response.json({ threads: [{ id: 'same-thread', historyId: '10' }] }) + } + return Response.json({ + id: 'same-thread', + historyId: '10', + messages: [ + { + id: 'message-1', + threadId: 'same-thread', + labelIds: ['Label_7'], + payload: { + mimeType: 'text/plain', + headers: [ + { name: 'Subject', value: `${mailbox} private thread` }, + { name: 'From', value: 'someone-else@example.com' }, + { name: 'To', value: 'entire-company@example.com' }, + ], + body: { data: Buffer.from(`${mailbox} private body`).toString('base64url') }, + }, + }, + ], + }) +} + +beforeEach(() => { + vi.clearAllMocks() + listUsers.mockResolvedValue({ users: [ALICE, BOB] }) + getUser.mockImplementation( + async (_token: string, key: string) => + [ALICE, BOB, ADMIN].find((user) => user.id === key || user.email === key) ?? null + ) + fetchProvider.mockImplementation(async (url: string, init?: RequestInit) => + providerResponse(url, init) + ) +}) + +afterEach(() => vi.useRealTimers()) + +describe('company-wide Gmail indexing', () => { + it('forwards cancellation to mailbox listing and refuses follow-up metadata requests', async () => { + const controller = new AbortController() + const context = { ...centralContext(), signal: controller.signal } + fetchProvider.mockImplementation(async (_url: string, init?: RequestInit) => { + expect(init?.signal).toBe(controller.signal) + controller.abort() + return Response.json({ threads: [{ id: 'same-thread' }] }) + }) + await expect( + gmailConnector.listDocuments('directory-token', CONFIG, undefined, context) + ).rejects.toThrow() + expect(fetchProvider).toHaveBeenCalledOnce() + }) + + it('cancels label resolution without caching the cancellation as an unavailable label index', async () => { + const controller = new AbortController() + const context = { ...centralContext(), signal: controller.signal } + fetchProvider.mockImplementation(async (_url: string, init?: RequestInit) => { + expect(init?.signal).toBe(controller.signal) + controller.abort() + throw new DOMException('Cancelled', 'AbortError') + }) + await expect( + gmailConnector.listDocuments( + 'directory-token', + { ...CONFIG, label: 'Engineering' }, + undefined, + context + ) + ).rejects.toThrow('abort') + expect(fetchProvider).toHaveBeenCalledOnce() + }) + + it('forwards the page signal to thread hydration, label reads, and separately stored MIME bodies', async () => { + const controller = new AbortController() + const context = { ...centralContext(), signal: controller.signal } + const page = await gmailConnector.listDocuments('directory-token', CONFIG, undefined, context) + fetchProvider.mockImplementation(async (url: string, init?: RequestInit) => { + expect(init?.signal).toBe(controller.signal) + if (url.includes('/attachments/')) + return Response.json({ data: Buffer.from('Private body').toString('base64url'), size: 12 }) + if (url.endsWith('/labels')) return providerResponse(url, init) + return Response.json({ + id: 'same-thread', + historyId: '10', + messages: [ + { + id: 'message-1', + threadId: 'same-thread', + labelIds: ['Label_7'], + payload: { + mimeType: 'text/plain', + headers: [{ name: 'Subject', value: 'Private subject' }], + body: { attachmentId: 'body-1', size: 12 }, + }, + }, + ], + }) + }) + const hydrated = await gmailConnector.getDocument( + 'directory-token', + CONFIG, + page.documents[0].externalId, + context + ) + expect(hydrated?.content).toContain('Private body') + expect(hydrated?.acl).toEqual([`u:${ALICE.email}`]) + }) + + it('passes the signal into the delegated validation probe', async () => { + const controller = new AbortController() + const context = { ...centralContext(), signal: controller.signal } + await expect( + gmailConnector.validateConfig('directory-token', CONFIG, context) + ).resolves.toEqual({ valid: true }) + expect(fetchProvider).toHaveBeenCalledWith( + expect.stringContaining('/profile'), + expect.objectContaining({ signal: controller.signal }), + expect.any(Object) + ) + }) + + it('bounds thread-list response bytes and page length instead of accepting a truncated corpus', async () => { + fetchProvider.mockResolvedValueOnce( + new Response('untrusted', { headers: { 'Content-Length': String(9 * 1024 * 1024) } }) + ) + await expect( + gmailConnector.listDocuments('directory-token', CONFIG, undefined, centralContext()) + ).rejects.toThrow('maximum size') + fetchProvider.mockResolvedValueOnce( + Response.json({ + threads: Array.from({ length: 101 }, (_, index) => ({ + id: `thread-${index}`, + historyId: '10', + })), + }) + ) + await expect( + gmailConnector.listDocuments('directory-token', CONFIG, undefined, centralContext()) + ).rejects.toThrow('malformed thread listing') + }) + + it('requires a service account with separate read-only directory and mailbox scopes', () => { + expect(gmailConnectorMeta.auth).toEqual({ + mode: 'oauth', + provider: 'google-email', + requiredScopes: ['https://www.googleapis.com/auth/gmail.modify'], + adminCredentialType: 'service_account', + serviceAccountScopes: ['https://www.googleapis.com/auth/gmail.readonly'], + adminServiceAccountScopes: ['https://www.googleapis.com/auth/admin.directory.user.readonly'], + serviceAccountDelegationScopes: ['https://www.googleapis.com/auth/gmail.readonly'], + serviceAccountSubjectFieldId: 'adminEmail', + }) + expect(gmailConnectorMeta.supportsSeparateContentCredential).toBeUndefined() + expect(gmailConnectorMeta.searchDefaultSourceConfig).toEqual({ dateRange: '6m' }) + expect( + gmailConnectorMeta.configFields.find((field) => field.id === 'labelSelector') + ).toMatchObject({ + hideInMemberMode: true, + hideInAdminMode: true, + }) + }) + + it('keeps identical thread IDs, bodies, label caches, and owner ACLs separate', async () => { + const context = centralContext() + const first = await gmailConnector.listDocuments('directory-token', CONFIG, undefined, context) + const alice = first.documents[0] + const aliceBody = await gmailConnector.getDocument( + 'directory-token', + CONFIG, + alice.externalId, + context + ) + const second = await gmailConnector.listDocuments( + 'directory-token', + CONFIG, + first.nextCursor, + context + ) + const bob = second.documents[0] + const bobBody = await gmailConnector.getDocument( + 'directory-token', + CONFIG, + bob.externalId, + context + ) + + expect(first.hasMore).toBe(true) + expect(second.hasMore).toBe(false) + expect(alice.externalId).not.toBe(bob.externalId) + expect(alice.externalId).toContain(encodeURIComponent(ALICE.id)) + expect(bob.externalId).toContain(encodeURIComponent(BOB.id)) + expect(alice.acl).toEqual([`u:${ALICE.email}`]) + expect(bob.acl).toEqual([`u:${BOB.email}`]) + expect(aliceBody).toMatchObject({ + acl: [`u:${ALICE.email}`], + contentHash: alice.contentHash, + metadata: { labels: ['Alice label'] }, + }) + expect(bobBody).toMatchObject({ + acl: [`u:${BOB.email}`], + contentHash: bob.contentHash, + metadata: { labels: ['Bob label'] }, + }) + expect(aliceBody?.content).toContain('Alice private body') + expect(bobBody?.content).toContain('Bob private body') + expect(context.getDelegatedAccessToken.mock.calls.map(([email]) => email)).toEqual([ + ALICE.email, + BOB.email, + ]) + for (const [, init] of fetchProvider.mock.calls) { + expect(new Headers(init?.headers).get('Authorization')).not.toBe('Bearer directory-token') + } + }) + + it('rejects hydration without its listed page and after advancing to another mailbox', async () => { + const context = centralContext() + const first = await gmailConnector.listDocuments('directory-token', CONFIG, undefined, context) + const externalId = first.documents[0].externalId + const callsBefore = fetchProvider.mock.calls.length + await expect( + gmailConnector.getDocument('directory-token', CONFIG, externalId, centralContext()) + ).rejects.toThrow() + expect(fetchProvider).toHaveBeenCalledTimes(callsBefore) + await gmailConnector.listDocuments('directory-token', CONFIG, first.nextCursor, context) + const callsAfter = fetchProvider.mock.calls.length + await expect( + gmailConnector.getDocument('directory-token', CONFIG, externalId, context) + ).rejects.toThrow() + await expect( + gmailConnector.getDocument('directory-token', CONFIG, 'same-thread', context) + ).rejects.toThrow() + expect(fetchProvider).toHaveBeenCalledTimes(callsAfter) + }) + + it('replays a page with the same query, mailbox namespace, and ACL in a fresh context', async () => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-09-01T12:00:00Z')) + const first = await gmailConnector.listDocuments( + 'directory-token', + CONFIG, + undefined, + centralContext() + ) + const firstQuery = new URL(fetchProvider.mock.calls[0][0]).searchParams.get('q') + expect(first.currentCursor).toBeDefined() + vi.setSystemTime(new Date('2026-09-03T12:00:00Z')) + const resumedContext = centralContext() + const resumed = await gmailConnector.listDocuments( + 'directory-token', + CONFIG, + first.currentCursor, + resumedContext + ) + expect(resumed.documents).toEqual(first.documents) + expect(new URL(fetchProvider.mock.calls[1][0]).searchParams.get('q')).toBe(firstQuery) + const body = await gmailConnector.getDocument( + 'directory-token', + CONFIG, + resumed.documents[0].externalId, + resumedContext + ) + expect(body?.content).toContain('Alice private body') + }) + + it('does not advance to the next mailbox until all thread pages are exhausted', async () => { + fetchProvider.mockImplementation(async (url: string, init?: RequestInit) => { + const parsed = new URL(url) + const token = new Headers(init?.headers).get('Authorization') + if ( + parsed.pathname.endsWith('/threads') && + token?.includes(ALICE.email) && + !parsed.searchParams.has('pageToken') + ) { + return Response.json({ threads: [], nextPageToken: 'alice-next' }) + } + return providerResponse(url, init) + }) + const context = centralContext() + const first = await gmailConnector.listDocuments('directory-token', CONFIG, undefined, context) + expect(first.documents).toEqual([]) + expect(first.hasMore).toBe(true) + const next = await gmailConnector.listDocuments( + 'directory-token', + CONFIG, + first.nextCursor, + context + ) + expect(next.documents[0].acl).toEqual([`u:${ALICE.email}`]) + expect(new URL(fetchProvider.mock.calls[1][0]).searchParams.get('pageToken')).toBe('alice-next') + const last = await gmailConnector.listDocuments( + 'directory-token', + CONFIG, + next.nextCursor, + context + ) + expect(last.documents[0].acl).toEqual([`u:${BOB.email}`]) + }) + + it('excludes a user suspended between directory discovery and crawl', async () => { + getUser.mockImplementation(async (_token: string, key: string) => + key === ALICE.id ? { ...ALICE, active: false } : BOB + ) + const context = centralContext() + const first = await gmailConnector.listDocuments('directory-token', CONFIG, undefined, context) + expect(first.documents).toEqual([]) + expect(first.hasMore).toBe(true) + expect(context.getDelegatedAccessToken).not.toHaveBeenCalled() + const next = await gmailConnector.listDocuments( + 'directory-token', + CONFIG, + first.nextCursor, + context + ) + expect(next.documents[0].acl).toEqual([`u:${BOB.email}`]) + }) + + it('does not fall back to the administrator when delegation is revoked', async () => { + const context = centralContext() + context.getDelegatedAccessToken.mockRejectedValue(new Error('Delegation revoked')) + await expect( + gmailConnector.listDocuments('directory-token', CONFIG, undefined, context) + ).rejects.toThrow('Delegation revoked') + expect(fetchProvider).not.toHaveBeenCalled() + }) + + it('fails an unreadable mailbox instead of reporting a complete empty corpus', async () => { + fetchProvider.mockResolvedValue(new Response(null, { status: 403 })) + await expect( + gmailConnector.listDocuments('directory-token', CONFIG, undefined, centralContext()) + ).rejects.toThrow('403') + }) + + it('invalidates previous hydration authority when the next mailbox fails', async () => { + const context = centralContext() + const first = await gmailConnector.listDocuments('directory-token', CONFIG, undefined, context) + fetchProvider.mockResolvedValueOnce(new Response(null, { status: 503 })) + await expect( + gmailConnector.listDocuments('directory-token', CONFIG, first.nextCursor, context) + ).rejects.toThrow('503') + const calls = fetchProvider.mock.calls.length + await expect( + gmailConnector.getDocument('directory-token', CONFIG, first.documents[0].externalId, context) + ).rejects.toThrow() + expect(fetchProvider).toHaveBeenCalledTimes(calls) + }) + + it('continues directory pagination when the first page has no selected mailboxes', async () => { + listUsers + .mockResolvedValueOnce({ users: [ALICE], nextPageToken: 'directory-next' }) + .mockResolvedValueOnce({ users: [BOB] }) + const context = centralContext() + const config = { ...CONFIG, userEmails: BOB.email } + const first = await gmailConnector.listDocuments('directory-token', config, undefined, context) + expect(first.documents).toEqual([]) + expect(first.hasMore).toBe(true) + expect(fetchProvider).not.toHaveBeenCalled() + const next = await gmailConnector.listDocuments( + 'directory-token', + config, + first.nextCursor, + context + ) + expect(next.documents[0].acl).toEqual([`u:${BOB.email}`]) + expect(listUsers.mock.calls[1][1]).toBe('directory-next') + expect(next.hasMore).toBe(false) + }) + + it('retains the hydrated revision when a thread changes after listing', async () => { + const context = centralContext() + const page = await gmailConnector.listDocuments('directory-token', CONFIG, undefined, context) + fetchProvider.mockImplementation(async (url: string, init?: RequestInit) => { + const response = providerResponse(url, init) + if (!new URL(url).pathname.endsWith('/threads/same-thread')) return response + const body = await response.json() + return Response.json({ ...body, historyId: '20' }) + }) + const body = await gmailConnector.getDocument( + 'directory-token', + CONFIG, + page.documents[0].externalId, + context + ) + expect(page.documents[0].contentHash).toBe('gmail:same-thread:10:body-v2') + expect(body?.contentHash).toBe('gmail:same-thread:20:body-v2') + expect(body?.acl).toEqual([`u:${ALICE.email}`]) + }) + + it('rejects an OAuth-only context and an invalid company cursor before Gmail reads', async () => { + await expect( + gmailConnector.listDocuments('oauth-token', CONFIG, undefined, { mirrorsSourceAcls: true }) + ).rejects.toThrow('service account') + let failure: unknown + try { + await gmailConnector.listDocuments( + 'directory-token', + CONFIG, + 'other-mailbox-cursor', + centralContext() + ) + } catch (error) { + failure = error + } + expect(gmailConnector.isListingCursorInvalidError?.(failure)).toBe(true) + expect(fetchProvider).not.toHaveBeenCalled() + }) + + it.each([{ maxThreads: 25 }, { label: ['Label_7'] }])( + 'rejects unsafe central settings before provider reads: %o', + async (extra) => { + await expect( + gmailConnector.listDocuments( + 'directory-token', + { ...CONFIG, ...extra }, + undefined, + centralContext() + ) + ).rejects.toThrow() + expect(listUsers).not.toHaveBeenCalled() + expect(fetchProvider).not.toHaveBeenCalled() + } + ) + + it('validates delegated Gmail access without requiring a label in every mailbox', async () => { + const context = centralContext() + const result = await gmailConnector.validateConfig( + 'directory-token', + { ...CONFIG, userEmails: ALICE.email, label: ['Engineering'] }, + context + ) + expect(result).toEqual({ valid: true }) + expect(context.getDelegatedAccessToken).toHaveBeenCalledWith(ALICE.email) + expect( + fetchProvider.mock.calls.some(([url]) => new URL(url).pathname.endsWith('/labels')) + ).toBe(false) + }) + + it('never reuses one mailbox history cursor for a company crawl', async () => { + const context = centralContext() + await expect( + gmailConnector.getChangeCursor!('directory-token', CONFIG, context) + ).rejects.toThrow('complete mailbox listings') + await expect( + gmailConnector.listChanges!('directory-token', CONFIG, '{"historyId":"10"}', context) + ).rejects.toThrow('complete mailbox listings') + expect(fetchProvider).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/connectors/gmail/gmail.test.ts b/apps/sim/connectors/gmail/gmail.test.ts index 43457cbc002..0623b164f61 100644 --- a/apps/sim/connectors/gmail/gmail.test.ts +++ b/apps/sim/connectors/gmail/gmail.test.ts @@ -45,12 +45,7 @@ function mockPages(pages: { threads: unknown[]; nextPageToken?: string }[]) { mockFetchWithRetry.mockImplementation(async (url: string) => { urls.push(url) const page = pages[call++] ?? { threads: [] } - return { - ok: true, - status: 200, - json: async () => page, - text: async () => JSON.stringify(page), - } as unknown as Response + return Response.json(page) }) return urls } @@ -726,15 +721,15 @@ describe('Gmail separately stored message bodies', () => { }) describe('Gmail Search member isolation', () => { - it('offers only the existing member account access path', () => { + it('preserves the member OAuth path separately from service-account indexing', () => { expect(gmailConnectorMeta.search).toBe(true) - expect(gmailConnectorMeta.auth).toEqual({ + expect(gmailConnectorMeta.auth).toMatchObject({ mode: 'oauth', provider: 'google-email', requiredScopes: ['https://www.googleapis.com/auth/gmail.modify'], }) expect(gmailConnectorMeta.permissionScopedListing).toEqual({ capFieldIds: ['maxThreads'] }) - expect(gmailConnectorMeta.mirrorsSourceAcls).toBeUndefined() + expect(gmailConnectorMeta.mirrorsSourceAcls).toBe(true) expect(gmailConnectorMeta.supportsSeparateContentCredential).toBeUndefined() }) @@ -900,6 +895,7 @@ describe('Gmail thread revisions and deferred content', () => { .mockResolvedValueOnce(new Response(null, { status: 404 })) const context: Record = {} expect(await gmailConnector.listDocuments('token', {}, undefined, context)).toEqual({ + currentCursor: expect.any(String), documents: [], hasMore: false, nextCursor: undefined, @@ -954,7 +950,12 @@ describe('Gmail listing completeness and filters', () => { undefined, context ) - ).toEqual({ documents: [], hasMore: false, nextCursor: undefined }) + ).toEqual({ + currentCursor: expect.any(String), + documents: [], + hasMore: false, + nextCursor: undefined, + }) expect(context.totalThreadsFetched).toBe(0) expect(context.listingCapped).toBeUndefined() expect(parseBody).not.toHaveBeenCalled() @@ -966,6 +967,7 @@ describe('Gmail listing completeness and filters', () => { mockFetchWithRetry.mockResolvedValueOnce(Response.json(body)) const context: Record = {} expect(await gmailConnector.listDocuments('token', {}, undefined, context)).toEqual({ + currentCursor: expect.any(String), documents: [], hasMore: false, nextCursor: undefined, diff --git a/apps/sim/connectors/gmail/gmail.ts b/apps/sim/connectors/gmail/gmail.ts index 9d9fa545d97..1f65f4354fe 100644 --- a/apps/sim/connectors/gmail/gmail.ts +++ b/apps/sim/connectors/gmail/gmail.ts @@ -5,6 +5,12 @@ import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_THREADS, gmailConnectorMeta } from '@/connectors/gmail/meta' +import { + getGoogleWorkspaceDocument, + InvalidGoogleWorkspaceCursor, + listGoogleWorkspaceDocuments, + validateGoogleWorkspaceConfig, +} from '@/connectors/google-workspace/company-crawl' import type { ConnectorConfig, ExternalChange, @@ -23,6 +29,7 @@ import { memberDocumentId, parseDefaultedUnlimitedSafeInteger, parseMultiValue, + parseOptionalUnlimitedSafeInteger, parseTagDate, sizeLimitSkipReason, sourceDocumentId, @@ -35,6 +42,7 @@ const THREADS_PER_PAGE = 100 const BODY_RESPONSE_ENVELOPE_BYTES = 1024 /** Bounds base64 bodies, alternative MIME parts, and headers before parsing the thread JSON. */ const MAX_THREAD_RESPONSE_BYTES = 32 * 1024 * 1024 +const MAX_METADATA_RESPONSE_BYTES = 8 * 1024 * 1024 /** History records that can move a thread into or out of the configured scope. */ const HISTORY_TYPES = ['messageAdded', 'messageDeleted', 'labelAdded', 'labelRemoved'] as const @@ -91,6 +99,7 @@ interface GmailThreadList { interface GmailBodyContext { accessToken: string remainingBytes: number + signal?: AbortSignal } function isConfirmedResponseOverflow(error: unknown, label: string): boolean { @@ -109,8 +118,8 @@ function parseListingCursor(cursor?: string): { pageToken?: string; searchQuery? const parsed: unknown = JSON.parse(cursor) if ( isPlainRecord(parsed) && - typeof parsed.pageToken === 'string' && - parsed.pageToken.length > 0 && + (parsed.pageToken === undefined || + (typeof parsed.pageToken === 'string' && parsed.pageToken.length > 0)) && typeof parsed.searchQuery === 'string' ) { return { pageToken: parsed.pageToken, searchQuery: parsed.searchQuery } @@ -136,7 +145,9 @@ function parseThreadList(value: unknown): GmailThreadList { if ( !isPlainRecord(value) || (value.threads !== undefined && - (!Array.isArray(value.threads) || !value.threads.every(isThreadMetadata))) || + (!Array.isArray(value.threads) || + value.threads.length > THREADS_PER_PAGE || + !value.threads.every(isThreadMetadata))) || (value.nextPageToken !== undefined && (typeof value.nextPageToken !== 'string' || value.nextPageToken.length === 0)) ) { @@ -154,6 +165,27 @@ interface GmailLabel { type?: string } +async function readLabels(response: Response): Promise { + const data = await readResponseJsonWithLimit(response, { + maxBytes: MAX_METADATA_RESPONSE_BYTES, + label: 'Gmail labels', + }) + if ( + !isPlainRecord(data) || + !Array.isArray(data.labels) || + !data.labels.every( + (label): label is GmailLabel => + isPlainRecord(label) && + typeof label.id === 'string' && + label.id.length > 0 && + typeof label.name === 'string' && + (label.type === undefined || typeof label.type === 'string') + ) + ) + throw new Error('Gmail returned malformed labels') + return data.labels +} + const LABEL_CACHE_KEY = '_gmailLabelCache' interface GmailLabelIndex { @@ -188,6 +220,8 @@ async function getLabelIndex( accessToken: string, syncContext?: Record ): Promise { + const signal = syncContext?.signal instanceof AbortSignal ? syncContext.signal : undefined + signal?.throwIfAborted() if (syncContext && LABEL_CACHE_KEY in syncContext) { return syncContext[LABEL_CACHE_KEY] as GmailLabelIndex | null } @@ -196,6 +230,7 @@ async function getLabelIndex( try { const response = await fetchWithRetry(`${GMAIL_API_BASE}/labels`, { method: 'GET', + signal, headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json', @@ -206,15 +241,12 @@ async function getLabelIndex( throw new GmailApiError('Failed to fetch Gmail labels', response.status) } if (response.ok) { - const data: unknown = await response.json() - if (!isPlainRecord(data) || !Array.isArray(data.labels)) { - throw new Error('Gmail returned malformed labels') - } - index = buildLabelIndex(data.labels as GmailLabel[]) + index = buildLabelIndex(await readLabels(response)) } else { logger.warn('Failed to fetch Gmail labels', { status: response.status }) } } catch (error) { + signal?.throwIfAborted() if (error instanceof GmailApiError && error.status === 401) throw error logger.warn('Failed to fetch Gmail labels', { error: toError(error).message }) } @@ -354,6 +386,7 @@ async function readMessageBody( messageId: string, context: GmailBodyContext ): Promise { + context.signal?.throwIfAborted() const body = part.body if (!body) return '' if (body.size !== undefined && body.size > context.remainingBytes) { @@ -366,6 +399,7 @@ async function readMessageBody( `${GMAIL_API_BASE}/messages/${encodeURIComponent(messageId)}/attachments/${encodeURIComponent(body.attachmentId)}?${params}`, { method: 'GET', + signal: context.signal, headers: { Authorization: `Bearer ${context.accessToken}`, Accept: 'application/json' }, } ) @@ -473,7 +507,8 @@ function getHeader(payload: GmailMessagePart | undefined, name: string): string */ async function formatThread( thread: GmailThread, - accessToken: string + accessToken: string, + signal?: AbortSignal ): Promise<{ content: string subject: string @@ -501,7 +536,7 @@ async function formatThread( const labelIds = [...labelIdSet] const lines = new BoundedLines() - const bodyContext = { accessToken, remainingBytes: CONNECTOR_TEXT_DOCUMENT_MAX_BYTES } + const bodyContext = { accessToken, remainingBytes: CONNECTOR_TEXT_DOCUMENT_MAX_BYTES, signal } if ( !lines.push( `Subject: ${subject}`, @@ -515,6 +550,7 @@ async function formatThread( } for (const msg of messages) { + signal?.throwIfAborted() const msgFrom = getHeader(msg.payload, 'From') || 'Unknown' const msgDate = getHeader(msg.payload, 'Date') || '' const body = msg.payload ? await extractBody(msg.payload, msg.id, bodyContext) : '' @@ -552,8 +588,10 @@ async function formatThread( async function fetchThread( accessToken: string, threadId: string, - format: 'full' | 'minimal' | 'metadata' = 'full' + format: 'full' | 'minimal' | 'metadata' = 'full', + signal?: AbortSignal ): Promise { + signal?.throwIfAborted() const params = new URLSearchParams({ format }) if (format === 'minimal') params.set('fields', 'id,historyId,snippet') /** Enough to place every message against the configured scope without any body. */ @@ -563,6 +601,7 @@ async function fetchThread( const response = await fetchWithRetry(url, { method: 'GET', + signal, headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json', @@ -757,13 +796,16 @@ function threadInScope(thread: GmailThread, scope: GmailChangeScope): boolean { }) } -export const gmailConnector: ConnectorConfig = { +const gmailMailboxConnector: ConnectorConfig = { ...gmailConnectorMeta, isCredentialInvalidError: (error) => error instanceof GmailApiError && error.status === 401, /** The mailbox's current history id; `users.history.list` replays everything after it. */ - getChangeCursor: async (accessToken: string): Promise => { + getChangeCursor: async (accessToken, _sourceConfig, syncContext): Promise => { + if (syncContext?.mirrorsSourceAcls === true) { + throw new Error('Company-wide Gmail indexing uses complete mailbox listings') + } const response = await fetchWithRetry(`${GMAIL_API_BASE}/profile?fields=historyId`, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, @@ -796,6 +838,9 @@ export const gmailConnector: ConnectorConfig = { cursor: string, syncContext?: Record ): Promise => { + if (syncContext?.mirrorsSourceAcls === true) { + throw new Error('Company-wide Gmail indexing uses complete mailbox listings') + } const { historyId, pageToken } = parseChangeCursor(cursor) let labelIndex = EMPTY_LABEL_INDEX if (parseMultiValue(sourceConfig.label).length > 0) { @@ -853,6 +898,8 @@ export const gmailConnector: ConnectorConfig = { cursor?: string, syncContext?: Record ): Promise => { + const signal = syncContext?.signal instanceof AbortSignal ? syncContext.signal : undefined + signal?.throwIfAborted() const { pageToken, searchQuery: savedSearchQuery } = parseListingCursor(cursor) let searchQuery = savedSearchQuery const configuredLabels = parseMultiValue(sourceConfig.label) @@ -917,6 +964,7 @@ export const gmailConnector: ConnectorConfig = { const response = await fetchWithRetry(url, { method: 'GET', + signal, headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json', @@ -930,12 +978,18 @@ export const gmailConnector: ConnectorConfig = { /** Gmail can return 204 when an empty listing has no requested metadata fields. */ const { threads, nextPageToken } = parseThreadList( - response.status === 204 ? {} : await response.json() + response.status === 204 + ? {} + : await readResponseJsonWithLimit(response, { + maxBytes: MAX_METADATA_RESPONSE_BYTES, + label: 'Gmail thread listing', + }) ) + signal?.throwIfAborted() const stubs = await mapWithConcurrency(threads, 5, async (thread) => { const metadata = thread.historyId ? thread - : await fetchThread(accessToken, thread.id, 'minimal') + : await fetchThread(accessToken, thread.id, 'minimal', signal) return metadata ? threadToStub(metadata, syncContext) : null }) const documents = stubs.filter((stub): stub is ExternalDocument => stub !== null) @@ -962,6 +1016,7 @@ export const gmailConnector: ConnectorConfig = { */ return { documents, + currentCursor: JSON.stringify({ ...(pageToken ? { pageToken } : {}), searchQuery }), /** Relative dates and resolved label names must stay fixed across checkpoint resumes. */ nextCursor: !hitLimit && nextPageToken @@ -977,22 +1032,24 @@ export const gmailConnector: ConnectorConfig = { externalId: string, syncContext?: Record ): Promise => { + const signal = syncContext?.signal instanceof AbortSignal ? syncContext.signal : undefined + signal?.throwIfAborted() const threadId = sourceDocumentId(externalId, syncContext) if (!threadId) return null let thread: GmailThread | null try { - thread = await fetchThread(accessToken, threadId) + thread = await fetchThread(accessToken, threadId, 'full', signal) } catch (error) { if (!isConfirmedResponseOverflow(error, 'Gmail thread response')) throw error - const before = await fetchThread(accessToken, threadId, 'minimal') + const before = await fetchThread(accessToken, threadId, 'minimal', signal) if (!before) return null /** The capped response has no verified revision; bracket one bounded retry before caching a skip. */ try { - thread = await fetchThread(accessToken, threadId) + thread = await fetchThread(accessToken, threadId, 'full', signal) } catch (retryError) { if (!isConfirmedResponseOverflow(retryError, 'Gmail thread response')) throw retryError - const after = await fetchThread(accessToken, threadId, 'minimal') + const after = await fetchThread(accessToken, threadId, 'minimal', signal) if (!after) return null if (before.historyId !== after.historyId) { throw new Error('Gmail thread changed while checking its size') @@ -1010,7 +1067,7 @@ export const gmailConnector: ConnectorConfig = { let formatted: Awaited> try { - formatted = await formatThread(thread, accessToken) + formatted = await formatThread(thread, accessToken, signal) } catch (error) { if (error instanceof ConnectorFileTooLargeError) { return { @@ -1037,8 +1094,10 @@ export const gmailConnector: ConnectorConfig = { validateConfig: async ( accessToken: string, - sourceConfig: Record + sourceConfig: Record, + syncContext?: Record ): Promise<{ valid: boolean; error?: string }> => { + const signal = syncContext?.signal instanceof AbortSignal ? syncContext.signal : undefined /** The same parser the sync uses, so a value that saves is a value that syncs. */ try { parseDefaultedUnlimitedSafeInteger( @@ -1056,6 +1115,7 @@ export const gmailConnector: ConnectorConfig = { profileUrl, { method: 'GET', + signal, headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json', @@ -1081,6 +1141,7 @@ export const gmailConnector: ConnectorConfig = { labelsUrl, { method: 'GET', + signal, headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json', @@ -1093,8 +1154,7 @@ export const gmailConnector: ConnectorConfig = { return { valid: false, error: 'Failed to fetch labels' } } - const labelsData = await labelsResponse.json() - const labels = (labelsData.labels || []) as GmailLabel[] + const labels = await readLabels(labelsResponse) labelIndex = buildLabelIndex(labels) const missing = configuredLabels.filter( (value) => !labelIndex.byId[value] && !labelIndex.idByLowerName[value.toLowerCase()] @@ -1124,6 +1184,7 @@ export const gmailConnector: ConnectorConfig = { testUrl, { method: 'GET', + signal, headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json', @@ -1168,3 +1229,71 @@ export const gmailConnector: ConnectorConfig = { return result }, } + +/** A complete mailbox corpus cannot stop at a per-user cap or reuse another mailbox's label IDs. */ +function centralGmailConfig(sourceConfig: Record): Record { + if ( + parseOptionalUnlimitedSafeInteger( + sourceConfig.maxThreads, + 'Max Threads must be a non-negative whole number' + ) > 0 + ) { + throw new Error( + 'Max Threads is not supported for company-wide indexing; narrow Users or filters instead' + ) + } + if (parseMultiValue(sourceConfig.label).some((label) => label.startsWith('Label_'))) { + throw new Error( + 'Use Gmail label names instead of account-specific label IDs for company-wide indexing' + ) + } + return { ...sourceConfig, maxThreads: 0 } +} + +export const gmailConnector: ConnectorConfig = { + ...gmailMailboxConnector, + isListingCursorInvalidError: (error) => error instanceof InvalidGoogleWorkspaceCursor, + listDocuments: async (accessToken, sourceConfig, cursor, syncContext) => + syncContext?.mirrorsSourceAcls === true + ? listGoogleWorkspaceDocuments({ + provider: 'gmail', + accessToken, + sourceConfig: centralGmailConfig(sourceConfig), + cursor, + syncContext, + listUserDocuments: gmailMailboxConnector.listDocuments, + }) + : gmailMailboxConnector.listDocuments(accessToken, sourceConfig, cursor, syncContext), + getDocument: (accessToken, sourceConfig, externalId, syncContext) => + syncContext?.mirrorsSourceAcls === true + ? getGoogleWorkspaceDocument({ + provider: 'gmail', + sourceConfig, + externalId, + syncContext, + getUserDocument: gmailMailboxConnector.getDocument, + }) + : gmailMailboxConnector.getDocument(accessToken, sourceConfig, externalId, syncContext), + validateConfig: async (accessToken, sourceConfig, syncContext) => { + if (syncContext?.mirrorsSourceAcls !== true) { + return gmailMailboxConnector.validateConfig(accessToken, sourceConfig, syncContext) + } + try { + const config = centralGmailConfig(sourceConfig) + const delegated = await validateGoogleWorkspaceConfig({ + provider: 'gmail', + accessToken, + sourceConfig: config, + syncContext, + }) + /** A label can exist in another selected mailbox even when the validation sample lacks it. */ + return gmailMailboxConnector.validateConfig( + delegated.accessToken, + { ...config, label: [] }, + delegated.syncContext + ) + } catch (error) { + return { valid: false, error: getErrorMessage(error, 'Failed to validate Gmail indexing') } + } + }, +} diff --git a/apps/sim/connectors/gmail/meta.ts b/apps/sim/connectors/gmail/meta.ts index 7afa39167e6..23f60b9ec50 100644 --- a/apps/sim/connectors/gmail/meta.ts +++ b/apps/sim/connectors/gmail/meta.ts @@ -16,18 +16,49 @@ export const gmailConnectorMeta: ConnectorMeta = { mode: 'oauth', provider: 'google-email', requiredScopes: ['https://www.googleapis.com/auth/gmail.modify'], + adminCredentialType: 'service_account', + serviceAccountScopes: ['https://www.googleapis.com/auth/gmail.readonly'], + adminServiceAccountScopes: ['https://www.googleapis.com/auth/admin.directory.user.readonly'], + serviceAccountDelegationScopes: ['https://www.googleapis.com/auth/gmail.readonly'], + serviceAccountSubjectFieldId: 'adminEmail', }, permissionScopedListing: { capFieldIds: ['maxThreads'] }, + mirrorsSourceAcls: true, + adminSetupHint: + 'Use a service account with domain-wide delegation to index selected Google Workspace mailboxes. Each mailbox remains private to its owner.', /** A personal mailbox is indexed from the last six months unless the source says otherwise. */ searchDefaultSourceConfig: { dateRange: '6m' }, configFields: [ + { + id: 'adminEmail', + title: 'Directory administrator email', + showInAdminModeOnly: true, + type: 'short-input', + required: false, + placeholder: 'admin@yourcompany.com', + description: + 'A Google Workspace administrator who can read the user directory. Mail is read as each selected user.', + }, + { + id: 'userEmails', + title: 'Users', + showInAdminModeOnly: true, + setupGroup: 'options', + type: 'short-input', + multi: true, + required: false, + placeholder: 'All active Google Workspace users', + description: + 'Optional primary email addresses, separated by commas (up to 100). Leave blank to index all active users across this Google Workspace customer.', + }, { id: 'labelSelector', title: 'Labels', type: 'selector', selectorKey: 'gmail.labels', hideInMemberMode: true, + hideInAdminMode: true, canonicalParamId: 'label', mode: 'basic', multi: true, diff --git a/apps/sim/connectors/google-calendar/company-crawl.test.ts b/apps/sim/connectors/google-calendar/company-crawl.test.ts new file mode 100644 index 00000000000..d1e49c9d82c --- /dev/null +++ b/apps/sim/connectors/google-calendar/company-crawl.test.ts @@ -0,0 +1,470 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockListUsers, mockGetUser } = vi.hoisted(() => ({ + mockListUsers: vi.fn(), + mockGetUser: vi.fn(), +})) + +vi.mock('@/connectors/google-workspace/users', () => ({ + GOOGLE_WORKSPACE_USERS_PAGE_SIZE: 100, + listGoogleWorkspaceUsers: mockListUsers, + getGoogleWorkspaceUser: mockGetUser, + selectedGoogleWorkspaceUsers: (value: unknown) => + typeof value === 'string' + ? value + .split(',') + .map((email) => email.trim()) + .filter(Boolean) + : [], +})) + +import { googleCalendarConnector } from '@/connectors/google-calendar/google-calendar' +import { googleCalendarConnectorMeta } from '@/connectors/google-calendar/meta' + +const ALICE = { id: 'id-alice', email: 'alice@example.com', customerId: 'customer-1', active: true } +const BOB = { id: 'id-bob', email: 'bob@example.com', customerId: 'customer-1', active: true } +const EVENT = { + id: 'meeting-1', + eventType: 'default', + summary: 'Private planning', + description: 'Owner-only details', + updated: '2026-09-09T00:00:00Z', + start: { dateTime: '2026-09-10T10:00:00Z', timeZone: 'UTC' }, + end: { dateTime: '2026-09-10T11:00:00Z', timeZone: 'UTC' }, +} + +function response(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }) +} + +const fetchMock = vi.fn() + +function context() { + return { + mirrorsSourceAcls: true, + getDelegatedAccessToken: vi.fn(async (email: string) => `delegated:${email}`), + } +} + +beforeEach(() => { + vi.clearAllMocks() + mockListUsers.mockResolvedValue({ users: [ALICE, BOB] }) + mockGetUser.mockImplementation( + async (_token: string, id: string) => + [ALICE, BOB].find((user) => user.id === id || user.email === id) ?? null + ) + fetchMock.mockImplementation(async () => response({ items: [EVENT] })) + vi.stubGlobal('fetch', fetchMock) +}) + +afterEach(() => { + vi.useRealTimers() + vi.unstubAllGlobals() +}) + +describe('Google Calendar company crawl', () => { + it('declares independent read-only Directory and delegated Calendar scopes', () => { + expect(googleCalendarConnectorMeta.auth).toMatchObject({ + requiredScopes: ['https://www.googleapis.com/auth/calendar'], + adminCredentialType: 'service_account', + adminServiceAccountScopes: ['https://www.googleapis.com/auth/admin.directory.user.readonly'], + serviceAccountDelegationScopes: ['https://www.googleapis.com/auth/calendar.events.readonly'], + serviceAccountSubjectFieldId: 'adminEmail', + }) + expect( + googleCalendarConnectorMeta.configFields.find((field) => field.id === 'calendarSelector') + ?.hideInAdminMode + ).toBe(true) + expect( + googleCalendarConnectorMeta.configFields.find((field) => field.id === 'calendarId') + ?.hideInAdminMode + ).not.toBe(true) + expect(googleCalendarConnectorMeta.supportsSeparateContentCredential).toBeUndefined() + }) + + it('isolates two users of the same calendar and event using verified user identities', async () => { + const syncContext = context() + const first = await googleCalendarConnector.listDocuments( + 'directory-token', + {}, + undefined, + syncContext + ) + expect(first.hasMore).toBe(true) + expect(first.documents[0].acl).toEqual(['u:alice@example.com']) + const second = await googleCalendarConnector.listDocuments( + 'directory-token', + {}, + first.nextCursor, + syncContext + ) + expect(second.hasMore).toBe(false) + expect(second.documents[0].acl).toEqual(['u:bob@example.com']) + expect(second.documents[0].externalId).not.toBe(first.documents[0].externalId) + expect(syncContext.getDelegatedAccessToken.mock.calls).toEqual([[ALICE.email], [BOB.email]]) + for (const [, init] of fetchMock.mock.calls) { + expect(init?.headers).not.toMatchObject({ Authorization: 'Bearer directory-token' }) + } + }) + + it('keeps an organizer’s full event separate from a reader’s restricted representation', async () => { + fetchMock.mockImplementation(async (_url, init) => { + const owner = + new Headers(init?.headers).get('Authorization') === `Bearer delegated:${ALICE.email}` + return response({ + items: [ + { + ...EVENT, + summary: owner ? 'Private planning' : 'Busy', + description: owner ? 'Owner-only details' : undefined, + }, + ], + }) + }) + const syncContext = context() + const owner = await googleCalendarConnector.listDocuments( + 'directory-token', + {}, + undefined, + syncContext + ) + const reader = await googleCalendarConnector.listDocuments( + 'directory-token', + {}, + owner.nextCursor, + syncContext + ) + expect(owner.documents[0].content).toContain('Owner-only details') + expect(reader.documents[0].content).not.toContain('Owner-only details') + expect(reader.documents[0].contentHash).not.toBe(owner.documents[0].contentHash) + expect(reader.documents[0].acl).toEqual([`u:${BOB.email}`]) + }) + + it('omits bare free/busy, cancelled, and status entries, while retaining declined recurring meetings', async () => { + fetchMock.mockResolvedValue( + response({ + items: [ + { id: 'busy', start: EVENT.start, end: EVENT.end }, + { ...EVENT, id: 'cancelled', status: 'cancelled' }, + { ...EVENT, id: 'out-of-office', eventType: 'outOfOffice' }, + { ...EVENT, id: 'working-location', eventType: 'workingLocation' }, + { ...EVENT, id: 'focus-time', eventType: 'focusTime' }, + { ...EVENT, id: 'birthday', eventType: 'birthday' }, + { + ...EVENT, + id: 'recurring_20260910', + recurringEventId: 'series', + attendees: [{ email: ALICE.email, self: true, responseStatus: 'declined' }], + }, + { ...EVENT, id: 'all-day', start: { date: '2026-09-10' }, end: { date: '2026-09-11' } }, + ], + }) + ) + const page = await googleCalendarConnector.listDocuments( + 'directory-token', + {}, + undefined, + context() + ) + expect(page.documents).toHaveLength(2) + expect(page.documents[0].content).toContain('Response: declined') + expect(page.documents[0].metadata?.responseStatus).toBe('declined') + expect(page.documents[1].metadata?.isAllDay).toBe(true) + expect(new URL(String(fetchMock.mock.calls[0][0])).searchParams.get('singleEvents')).toBe( + 'true' + ) + expect(new URL(String(fetchMock.mock.calls[0][0])).searchParams.get('eventTypes')).toBe( + 'default' + ) + }) + + it('pins the initial date window during replay, continues empty provider pages, then advances users', async () => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-09-10T10:00:00Z')) + fetchMock.mockImplementation(async (url) => + response( + new URL(String(url)).searchParams.has('pageToken') + ? { items: [EVENT] } + : { items: [], nextPageToken: 'page-two' } + ) + ) + const first = await googleCalendarConnector.listDocuments( + 'directory-token', + {}, + undefined, + context() + ) + expect(first.documents).toEqual([]) + expect(first.hasMore).toBe(true) + const firstWindow = new URL(String(fetchMock.mock.calls[0][0])).searchParams.get('timeMin') + vi.setSystemTime(new Date('2026-09-12T10:00:00Z')) + const replay = await googleCalendarConnector.listDocuments( + 'directory-token', + {}, + first.currentCursor, + context() + ) + expect(replay.currentCursor).toBe(first.currentCursor) + expect(new URL(String(fetchMock.mock.calls[1][0])).searchParams.get('timeMin')).toBe( + firstWindow + ) + const continuation = await googleCalendarConnector.listDocuments( + 'directory-token', + {}, + first.nextCursor, + context() + ) + expect(continuation.documents[0].acl).toEqual([`u:${ALICE.email}`]) + expect(new URL(String(fetchMock.mock.calls[2][0])).searchParams.get('timeMin')).toBe( + firstWindow + ) + const nextUser = await googleCalendarConnector.listDocuments( + 'directory-token', + {}, + continuation.nextCursor, + context() + ) + expect(nextUser.hasMore).toBe(true) + expect(new Headers(fetchMock.mock.calls[3][1]?.headers).get('Authorization')).toBe( + `Bearer delegated:${BOB.email}` + ) + }) + + it('binds deferred hydration to the active page’s verified identity, never an admin token', async () => { + const syncContext = context() + const first = await googleCalendarConnector.listDocuments( + 'directory-token', + {}, + undefined, + syncContext + ) + const id = first.documents[0].externalId + fetchMock.mockResolvedValueOnce(response(EVENT)) + const hydrated = await googleCalendarConnector.getDocument( + 'directory-token', + {}, + id, + syncContext + ) + expect(hydrated?.acl).toEqual([`u:${ALICE.email}`]) + expect(syncContext.getDelegatedAccessToken).toHaveBeenCalledTimes(1) + await expect( + googleCalendarConnector.getDocument('directory-token', {}, id, context()) + ).rejects.toThrow('verified delegated listing identity') + await googleCalendarConnector.listDocuments( + 'directory-token', + {}, + first.nextCursor, + syncContext + ) + await expect( + googleCalendarConnector.getDocument('directory-token', {}, id, syncContext) + ).rejects.toThrow('verified delegated listing identity') + }) + + it('rejects a different event returned during hydration', async () => { + const syncContext = context() + const first = await googleCalendarConnector.listDocuments( + 'directory-token', + {}, + undefined, + syncContext + ) + fetchMock.mockResolvedValueOnce(response({ ...EVENT, id: 'different-event' })) + await expect( + googleCalendarConnector.getDocument( + 'directory-token', + {}, + first.documents[0].externalId, + syncContext + ) + ).rejects.toThrow('different event') + }) + + it('continues past inaccessible configured calendars and preserves later shared calendars and users', async () => { + fetchMock.mockImplementation(async (url, init) => { + const alice = + new Headers(init?.headers).get('Authorization') === `Bearer delegated:${ALICE.email}` + const denied = String(url).includes('/calendars/restricted/') && alice + return response(denied ? { error: { code: 404 } } : { items: [EVENT] }, denied ? 404 : 200) + }) + const config = { calendarId: 'restricted,primary' } + const first = await googleCalendarConnector.listDocuments( + 'directory-token', + config, + undefined, + context() + ) + expect(first.documents).toEqual([]) + expect(first.hasMore).toBe(true) + const second = await googleCalendarConnector.listDocuments( + 'directory-token', + config, + first.nextCursor, + context() + ) + expect(second.documents[0].metadata?.calendarId).toBe('primary') + expect(second.documents[0].acl).toEqual([`u:${ALICE.email}`]) + const third = await googleCalendarConnector.listDocuments( + 'directory-token', + config, + second.nextCursor, + context() + ) + expect(third.documents[0].metadata?.calendarId).toBe('restricted') + expect(third.documents[0].acl).toEqual([`u:${BOB.email}`]) + }) + + it('treats a sole inaccessible calendar as empty for that user and continues to the next user', async () => { + fetchMock.mockResolvedValueOnce(response({ error: { code: 404 } }, 404)) + const first = await googleCalendarConnector.listDocuments( + 'directory-token', + {}, + undefined, + context() + ) + expect(first.documents).toEqual([]) + expect(first.hasMore).toBe(true) + const next = await googleCalendarConnector.listDocuments( + 'directory-token', + {}, + first.nextCursor, + context() + ) + expect(next.documents[0].acl).toEqual([`u:${BOB.email}`]) + }) + + it.each([401, 403])( + 'propagates provider HTTP %s without completing a user’s listing', + async (status) => { + fetchMock.mockResolvedValue(response({ error: { code: status } }, status)) + await expect( + googleCalendarConnector.listDocuments('directory-token', {}, undefined, context()) + ).rejects.toThrow() + } + ) + + it('skips a user who became inactive before their page and never delegates to them', async () => { + mockGetUser.mockResolvedValueOnce({ ...ALICE, active: false }) + const syncContext = context() + const first = await googleCalendarConnector.listDocuments( + 'directory-token', + {}, + undefined, + syncContext + ) + expect(first.documents).toEqual([]) + expect(first.hasMore).toBe(true) + expect(fetchMock).not.toHaveBeenCalled() + expect(syncContext.getDelegatedAccessToken).not.toHaveBeenCalled() + }) + + it('forwards cancellation without accepting an empty successful listing', async () => { + const controller = new AbortController() + const syncContext = { ...context(), signal: controller.signal } + fetchMock.mockImplementation(async (_url, init) => { + expect(init?.signal).toBe(controller.signal) + controller.abort() + return response({ items: [] }) + }) + await expect( + googleCalendarConnector.listDocuments('directory-token', {}, undefined, syncContext) + ).rejects.toThrow() + }) + + it.each([ + {}, + { items: null }, + { items: [{ summary: 'Missing ID' }] }, + { items: [], nextPageToken: 123 }, + ])( + 'rejects malformed provider payloads rather than treating them as empty (%j)', + async (payload) => { + fetchMock.mockResolvedValue(response(payload)) + await expect( + googleCalendarConnector.listDocuments('directory-token', {}, undefined, context()) + ).rejects.toThrow() + } + ) + + it('accepts Google’s typed empty result and continues to the next user', async () => { + fetchMock.mockResolvedValue(response({ kind: 'calendar#events' })) + const first = await googleCalendarConnector.listDocuments( + 'directory-token', + {}, + undefined, + context() + ) + expect(first.documents).toEqual([]) + expect(first.hasMore).toBe(true) + }) + + it('rejects an oversized page before it can grow the working document set', async () => { + fetchMock.mockResolvedValue( + response({ items: Array.from({ length: 251 }, (_, i) => ({ ...EVENT, id: `event-${i}` })) }) + ) + await expect( + googleCalendarConnector.listDocuments('directory-token', {}, undefined, context()) + ).rejects.toThrow() + }) + + it('validates delegation using the selected user’s primary calendar, not the admin or shared-calendar picker', async () => { + const syncContext = context() + const result = await googleCalendarConnector.validateConfig( + 'directory-token', + { + adminEmail: ALICE.email, + userEmails: BOB.email, + calendarId: 'shared-calendar', + }, + syncContext + ) + expect(result).toEqual({ valid: true }) + expect(syncContext.getDelegatedAccessToken).toHaveBeenCalledWith(BOB.email) + expect(String(fetchMock.mock.calls[0][0])).toContain('/calendars/primary/events?') + expect(new Headers(fetchMock.mock.calls[0][1]?.headers).get('Authorization')).toBe( + `Bearer delegated:${BOB.email}` + ) + }) + + it('reports missing Directory identity, invalid credentials, and invalid cursors without a provider fallback', async () => { + expect( + await googleCalendarConnector.validateConfig('directory-token', {}, context()) + ).toMatchObject({ valid: false }) + expect( + await googleCalendarConnector.validateConfig( + 'oauth-token', + { adminEmail: ALICE.email }, + { mirrorsSourceAcls: true } + ) + ).toMatchObject({ valid: false }) + const error = await googleCalendarConnector + .listDocuments('directory-token', {}, 'malformed', context()) + .catch((caught: unknown) => caught) + expect(googleCalendarConnector.isListingCursorInvalidError?.(error)).toBe(true) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('requires a delegated service account and rejects capped company listings', async () => { + await expect( + googleCalendarConnector.listDocuments('oauth-token', {}, undefined, { + mirrorsSourceAcls: true, + }) + ).rejects.toThrow('service account') + await expect( + googleCalendarConnector.listDocuments( + 'directory-token', + { maxEvents: 1 }, + undefined, + context() + ) + ).rejects.toThrow('Max Events') + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/connectors/google-calendar/google-calendar.test.ts b/apps/sim/connectors/google-calendar/google-calendar.test.ts index 748f3007e96..782fd3a66e5 100644 --- a/apps/sim/connectors/google-calendar/google-calendar.test.ts +++ b/apps/sim/connectors/google-calendar/google-calendar.test.ts @@ -135,10 +135,11 @@ describe('Google Calendar Search isolation', () => { }) }) - it('offers member Search without claiming centralized permissions or shared content', () => { + it('offers personal and centrally delegated Search without sharing event representations', () => { expect(googleCalendarConnectorMeta.search).toBe(true) expect(googleCalendarConnectorMeta.permissionScopedListing?.capFieldIds).toEqual(['maxEvents']) - expect(googleCalendarConnectorMeta.mirrorsSourceAcls).toBeUndefined() + expect(googleCalendarConnectorMeta.mirrorsSourceAcls).toBe(true) + expect(googleCalendarConnectorMeta.auth.adminCredentialType).toBe('service_account') expect(googleCalendarConnectorMeta.supportsSeparateContentCredential).toBeUndefined() }) @@ -222,7 +223,7 @@ describe('Google Calendar Search isolation', () => { }) ) const result = await googleCalendarConnector.listDocuments('token', {}, undefined, alice) - expect(result).toEqual({ documents: [], hasMore: false }) + expect(result).toMatchObject({ documents: [], hasMore: false }) }) it('asks Google for meetings only and drops status entries it still returns', async () => { @@ -283,7 +284,7 @@ describe('Google Calendar Search isolation', () => { jsonResponse({ items: [{ ...EVENT, status: 'cancelled', recurringEventId: 'series' }] }) ) const result = await googleCalendarConnector.listDocuments('token', {}, undefined, alice) - expect(result).toEqual({ documents: [], hasMore: false }) + expect(result).toMatchObject({ documents: [], hasMore: false }) }) it('follows empty continuation pages with identical time bounds after a resumed run', async () => { diff --git a/apps/sim/connectors/google-calendar/google-calendar.ts b/apps/sim/connectors/google-calendar/google-calendar.ts index 210e2b5b700..b56cd329196 100644 --- a/apps/sim/connectors/google-calendar/google-calendar.ts +++ b/apps/sim/connectors/google-calendar/google-calendar.ts @@ -1,7 +1,14 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { z } from 'zod' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_EVENTS, googleCalendarConnectorMeta } from '@/connectors/google-calendar/meta' +import { + getGoogleWorkspaceDocument, + InvalidGoogleWorkspaceCursor, + listGoogleWorkspaceDocuments, + validateGoogleWorkspaceConfig, +} from '@/connectors/google-workspace/company-crawl' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { computeContentHash, @@ -12,7 +19,9 @@ import { memberDocumentId, parseDefaultedUnlimitedSafeInteger, parseMultiValue, + parseOptionalUnlimitedSafeInteger, parseTagDate, + readBodyWithLimit, sourceDocumentId, } from '@/connectors/utils' @@ -21,40 +30,70 @@ const logger = createLogger('GoogleCalendarConnector') const CALENDAR_API_BASE = 'https://www.googleapis.com/calendar/v3' const DEFAULT_RANGE_DAYS = 30 const PAGE_SIZE = 250 +const CALENDAR_PAGE_MAX_BYTES = 16 * 1024 * 1024 +const EVENT_FIELDS = + 'id,status,htmlLink,created,updated,summary,description,location,creator(email,displayName),organizer(email,displayName,self),start(date,dateTime,timeZone),end(date,dateTime,timeZone),attendees(email,displayName,responseStatus,self,resource,optional),recurringEventId,eventType' class GoogleCalendarCredentialInvalidError extends Error {} -interface CalendarEventTime { - date?: string - dateTime?: string - timeZone?: string -} - -interface CalendarAttendee { - email?: string - displayName?: string - responseStatus?: string - self?: boolean - resource?: boolean - optional?: boolean -} - -interface CalendarEvent { - id: string - status?: string - htmlLink?: string - created?: string - updated?: string - summary?: string - description?: string - location?: string - creator?: { email?: string; displayName?: string } - organizer?: { email?: string; displayName?: string; self?: boolean } - start?: CalendarEventTime - end?: CalendarEventTime - attendees?: CalendarAttendee[] - recurringEventId?: string - eventType?: string +const calendarEventTimeSchema = z.object({ + date: z.string().optional(), + dateTime: z.string().optional(), + timeZone: z.string().optional(), +}) +const calendarAttendeeSchema = z.object({ + email: z.string().optional(), + displayName: z.string().optional(), + responseStatus: z.string().optional(), + self: z.boolean().optional(), + resource: z.boolean().optional(), + optional: z.boolean().optional(), +}) +const calendarEventSchema = z.object({ + id: z.string().min(1).max(2048), + status: z.string().optional(), + htmlLink: z.string().optional(), + created: z.string().optional(), + updated: z.string().optional(), + summary: z.string().optional(), + description: z.string().optional(), + location: z.string().optional(), + creator: z + .object({ email: z.string().optional(), displayName: z.string().optional() }) + .optional(), + organizer: z + .object({ + email: z.string().optional(), + displayName: z.string().optional(), + self: z.boolean().optional(), + }) + .optional(), + start: calendarEventTimeSchema.optional(), + end: calendarEventTimeSchema.optional(), + attendees: z.array(calendarAttendeeSchema).optional(), + recurringEventId: z.string().optional(), + eventType: z.string().optional(), +}) +const calendarPageSchema = z + .object({ + kind: z.literal('calendar#events').optional(), + items: z.array(calendarEventSchema).max(PAGE_SIZE).optional(), + nextPageToken: z.string().min(1).max(8192).optional(), + }) + .refine((page) => page.items !== undefined || page.kind === 'calendar#events') +type CalendarEventTime = z.infer +type CalendarAttendee = z.infer +type CalendarEvent = z.infer + +/** Malformed or oversized pages must fail the crawl, never reconcile as an empty calendar. */ +async function readCalendarJson(response: Response): Promise { + const body = await readBodyWithLimit(response, CALENDAR_PAGE_MAX_BYTES) + if (!body) throw new Error('Google Calendar response exceeded its size limit') + try { + return JSON.parse(body.toString('utf8')) + } catch { + throw new Error('Google Calendar returned malformed event data') + } } /** @@ -359,7 +398,7 @@ async function eventToDocument( } } -export const googleCalendarConnector: ConnectorConfig = { +const userCalendarConnector: ConnectorConfig = { ...googleCalendarConnectorMeta, isListingScopeUnavailableError: isListingScopeUnavailableError, @@ -414,6 +453,8 @@ export const googleCalendarConnector: ConnectorConfig = { return { documents: [], hasMore: false } } + const currentCursor = JSON.stringify({ calendarIndex, pageToken, timeRange }) + const calendarId = calendarIds[calendarIndex] const { timeMin, timeMax } = timeRange @@ -440,6 +481,7 @@ export const googleCalendarConnector: ConnectorConfig = { maxResults: String(pageSize), timeMin, timeMax, + fields: `kind,nextPageToken,items(${EVENT_FIELDS})`, }) if (searchQuery.trim()) { @@ -463,6 +505,7 @@ export const googleCalendarConnector: ConnectorConfig = { const response = await fetchWithRetry(url, { method: 'GET', + signal: syncContext?.signal instanceof AbortSignal ? syncContext.signal : undefined, headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json', @@ -473,11 +516,9 @@ export const googleCalendarConnector: ConnectorConfig = { if (response.status === 401) { throw new GoogleCalendarCredentialInvalidError('Reconnect your Google Calendar account') } - const errorText = await response.text() logger.error('Failed to list Google Calendar events', { status: response.status, calendarId, - error: errorText, }) const error = listingRequestError('Failed to list Google Calendar events', response.status) /** @@ -500,16 +541,17 @@ export const googleCalendarConnector: ConnectorConfig = { return calendarIndex + 1 < calendarIds.length ? { documents: [], + currentCursor, nextCursor: JSON.stringify({ calendarIndex: calendarIndex + 1, timeRange }), hasMore: true, } - : { documents: [], hasMore: false } + : { documents: [], currentCursor, hasMore: false } } throw error } - const data = await response.json() - const events = (data.items || []) as CalendarEvent[] + const data = calendarPageSchema.parse(await readCalendarJson(response)) + const events = data.items ?? [] const isMultiCalendar = calendarIds.length > 1 const includeAttendees = readIncludeAttendees(sourceConfig) @@ -534,7 +576,7 @@ export const googleCalendarConnector: ConnectorConfig = { const totalFetched = prevFetched + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched - const nextPageToken = (data.nextPageToken as string | undefined) || undefined + const nextPageToken = data.nextPageToken const hasMoreCalendars = calendarIndex + 1 < calendarIds.length const hitLimit = isCapped && totalFetched >= maxEvents @@ -553,12 +595,16 @@ export const googleCalendarConnector: ConnectorConfig = { if (truncatedByCap && syncContext) syncContext.listingCapped = true if (hitLimit) { - return { documents, hasMore: false } + return { documents, currentCursor, hasMore: false } } if (nextPageToken) { + if (nextPageToken === pageToken) { + throw new Error('Google Calendar repeated its continuation token') + } return { documents, + currentCursor, nextCursor: JSON.stringify({ calendarIndex, pageToken: nextPageToken, timeRange }), hasMore: true, } @@ -567,12 +613,13 @@ export const googleCalendarConnector: ConnectorConfig = { if (hasMoreCalendars) { return { documents, + currentCursor, nextCursor: JSON.stringify({ calendarIndex: calendarIndex + 1, timeRange }), hasMore: true, } } - return { documents, hasMore: false } + return { documents, currentCursor, hasMore: false } }, getDocument: async ( @@ -618,10 +665,11 @@ export const googleCalendarConnector: ConnectorConfig = { } if (isPerMemberListing(syncContext) && !calendarIds.includes(calendarId)) return null - const url = `${CALENDAR_API_BASE}/calendars/${encodeURIComponent(calendarId)}/events/${encodeURIComponent(eventId)}` + const url = `${CALENDAR_API_BASE}/calendars/${encodeURIComponent(calendarId)}/events/${encodeURIComponent(eventId)}?fields=${encodeURIComponent(EVENT_FIELDS)}` const response = await fetchWithRetry(url, { method: 'GET', + signal: syncContext?.signal instanceof AbortSignal ? syncContext.signal : undefined, headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json', @@ -636,7 +684,8 @@ export const googleCalendarConnector: ConnectorConfig = { throw new Error(`Failed to get Google Calendar event: ${response.status}`) } - const event = (await response.json()) as CalendarEvent + const event = calendarEventSchema.parse(await readCalendarJson(response)) + if (event.id !== eventId) throw new Error('Google Calendar returned a different event') return eventToDocument( event, @@ -671,6 +720,7 @@ export const googleCalendarConnector: ConnectorConfig = { url, { method: 'GET', + signal: syncContext?.signal instanceof AbortSignal ? syncContext.signal : undefined, headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json', @@ -727,3 +777,72 @@ export const googleCalendarConnector: ConnectorConfig = { return result }, } + +function companySourceConfig(sourceConfig: Record): Record { + const error = + 'Company-wide indexing does not support Max Events. Use calendars, dates, or a search query to limit the source.' + if (parseOptionalUnlimitedSafeInteger(sourceConfig.maxEvents, error) > 0) throw new Error(error) + return { ...sourceConfig, maxEvents: 0 } +} + +/** Company crawling keeps each user's visible event representation and reader grant separate. */ +export const googleCalendarConnector: ConnectorConfig = { + ...userCalendarConnector, + isListingCursorInvalidError: (error) => error instanceof InvalidGoogleWorkspaceCursor, + listDocuments: async (accessToken, sourceConfig, cursor, syncContext) => { + if (syncContext?.mirrorsSourceAcls !== true) { + return userCalendarConnector.listDocuments(accessToken, sourceConfig, cursor, syncContext) + } + return listGoogleWorkspaceDocuments({ + provider: 'google_calendar', + accessToken, + sourceConfig: companySourceConfig(sourceConfig), + cursor, + syncContext, + listUserDocuments: async (...args) => { + try { + return await userCalendarConnector.listDocuments(...args) + } catch (error) { + /** A confirmed inaccessible calendar is absent for this user; auth and quota failures still abort. */ + if (isListingScopeUnavailableError(error)) return { documents: [], hasMore: false } + throw error + } + }, + }) + }, + getDocument: async (accessToken, sourceConfig, externalId, syncContext) => + syncContext?.mirrorsSourceAcls === true + ? getGoogleWorkspaceDocument({ + provider: 'google_calendar', + sourceConfig: companySourceConfig(sourceConfig), + externalId, + syncContext, + getUserDocument: userCalendarConnector.getDocument, + }) + : userCalendarConnector.getDocument(accessToken, sourceConfig, externalId, syncContext), + validateConfig: async (accessToken, sourceConfig, syncContext) => { + if (syncContext?.mirrorsSourceAcls !== true) { + return userCalendarConnector.validateConfig(accessToken, sourceConfig, syncContext) + } + try { + const config = companySourceConfig(sourceConfig) + const delegated = await validateGoogleWorkspaceConfig({ + provider: 'google_calendar', + accessToken, + sourceConfig: config, + syncContext, + }) + /** Shared calendars need not be visible to the sample user; primary verifies the delegated API grant. */ + return userCalendarConnector.validateConfig( + delegated.accessToken, + { ...config, calendarId: 'primary' }, + delegated.syncContext + ) + } catch (error) { + return { + valid: false, + error: getErrorMessage(error, 'Failed to validate Google Calendar service account'), + } + } + }, +} diff --git a/apps/sim/connectors/google-calendar/meta.ts b/apps/sim/connectors/google-calendar/meta.ts index fdcef422a7a..6b3490eb7fc 100644 --- a/apps/sim/connectors/google-calendar/meta.ts +++ b/apps/sim/connectors/google-calendar/meta.ts @@ -16,15 +16,46 @@ export const googleCalendarConnectorMeta: ConnectorMeta = { mode: 'oauth', provider: 'google-calendar', requiredScopes: ['https://www.googleapis.com/auth/calendar'], + adminCredentialType: 'service_account', + serviceAccountScopes: ['https://www.googleapis.com/auth/calendar.events.readonly'], + adminServiceAccountScopes: ['https://www.googleapis.com/auth/admin.directory.user.readonly'], + serviceAccountDelegationScopes: ['https://www.googleapis.com/auth/calendar.events.readonly'], + serviceAccountSubjectFieldId: 'adminEmail', }, permissionScopedListing: { capFieldIds: ['maxEvents'] }, + mirrorsSourceAcls: true, + adminSetupHint: + 'Use a service account with domain-wide delegation to index selected Google Workspace calendars. Each person searches only their own view of events.', configFields: [ + { + id: 'adminEmail', + title: 'Directory administrator email', + showInAdminModeOnly: true, + type: 'short-input', + required: false, + placeholder: 'admin@yourcompany.com', + description: + 'A Google Workspace administrator who can read the user directory. Calendars are read as each selected user.', + }, + { + id: 'userEmails', + title: 'Users', + showInAdminModeOnly: true, + setupGroup: 'options', + type: 'short-input', + multi: true, + required: false, + placeholder: 'All active Google Workspace users', + description: + 'Optional primary email addresses, separated by commas (up to 100). Leave blank to index all active users across this Google Workspace customer.', + }, { id: 'calendarSelector', title: 'Calendars', type: 'selector', selectorKey: 'google.calendar', + hideInAdminMode: true, canonicalParamId: 'calendarId', mode: 'basic', multi: true, @@ -43,6 +74,8 @@ export const googleCalendarConnectorMeta: ConnectorMeta = { required: false, description: 'Calendars to sync from. Use "primary" for your main calendar. Defaults to "primary".', + descriptionInAdminMode: + 'Leave blank or use "primary" for each selected user’s main calendar. Shared calendar IDs apply to each user who can read them.', }, { id: 'dateRange', diff --git a/apps/sim/connectors/google-drive/company-crawl.test.ts b/apps/sim/connectors/google-drive/company-crawl.test.ts new file mode 100644 index 00000000000..1633a255f98 --- /dev/null +++ b/apps/sim/connectors/google-drive/company-crawl.test.ts @@ -0,0 +1,678 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { validateGoogleCompanyConfig } from '@/connectors/google-drive/company-crawl' +import { googleDriveConnector as drive } from '@/connectors/google-drive/google-drive' +import { GoogleDriveApiError } from '@/connectors/google-drive/google-drive-errors' +import { + listGoogleWorkspaceUsers, + selectedGoogleWorkspaceUsers, +} from '@/connectors/google-workspace/users' + +vi.mock('@/components/icons', () => ({ GoogleDriveIcon: () => null })) + +const mockFetch = vi.fn() +const CONFIG = { adminEmail: 'admin@corp.com' } +const USER = (email: string, extra: Record = {}) => ({ + id: email, + primaryEmail: email, + customerId: 'customer-1', + suspended: false, + ...extra, +}) +const FILE = (id: string, email: string, extra: Record = {}) => ({ + id, + name: id, + mimeType: 'text/plain', + modifiedTime: '2026-09-01T00:00:00Z', + permissions: [{ type: 'user', emailAddress: email, role: 'owner' }], + ...extra, +}) +const json = (value: unknown, status = 200) => + new Response(JSON.stringify(value), { status, headers: { 'Content-Type': 'application/json' } }) + +function context() { + return { + mirrorsSourceAcls: true, + getDelegatedAccessToken: vi.fn(async (email: string) => `delegated:${email}`), + } +} + +function fixture(input: { + users?: ReturnType[] + files?: Record[]> + permissions?: Record +}) { + const users = input.users ?? [USER('alice@corp.com'), USER('bob@corp.com')] + const files = input.files ?? { + 'alice@corp.com': [FILE('private-alice', 'alice@corp.com')], + 'bob@corp.com': [FILE('private-bob', 'bob@corp.com')], + } + mockFetch.mockImplementation(async (address: string, init?: RequestInit) => { + const url = new URL(address) + if (url.hostname === 'admin.googleapis.com') { + if (url.pathname.endsWith('/users')) return json({ users }) + const key = decodeURIComponent(url.pathname.split('/').at(-1)!) + return json(users.find((user) => user.id === key) ?? USER(key)) + } + const token = new Headers(init?.headers).get('Authorization')?.replace('Bearer delegated:', '') + const visible = files[token ?? ''] ?? [] + if (url.pathname.endsWith('/drives')) return json({ drives: [] }) + if (url.pathname.endsWith('/files')) return json({ files: visible }) + const fileId = decodeURIComponent(url.pathname.split('/')[4]) + if (url.pathname.endsWith('/permissions')) + return json({ permissions: input.permissions?.[fileId] ?? [] }) + const file = visible.find((candidate) => candidate.id === fileId) + if (!file) return json({ error: { errors: [{ reason: 'notFound' }] } }, 404) + if (url.searchParams.get('alt') === 'media') return new Response(`Body: ${file.id}`) + return json(file) + }) +} + +beforeEach(() => { + mockFetch.mockReset() + vi.stubGlobal('fetch', mockFetch) +}) +afterEach(() => vi.unstubAllGlobals()) + +describe('Google Drive company-wide crawl', () => { + it('indexes private files for multiple users without granting either user access to the other', async () => { + fixture({}) + const ctx = context() + const alice = await drive.listDocuments('directory-token', CONFIG, undefined, ctx, new Date()) + expect(alice.documents.map((doc) => [doc.externalId, doc.acl])).toEqual([ + ['private-alice', ['u:alice@corp.com']], + ]) + const body = await drive.getDocument('directory-token', CONFIG, 'private-alice', ctx) + expect(body?.content).toBe('Body: private-alice') + expect(body?.metadata).not.toHaveProperty('googleDrivePageSubjects') + const bob = await drive.listDocuments('directory-token', CONFIG, alice.nextCursor, ctx) + expect(bob.documents.map((doc) => [doc.externalId, doc.acl])).toEqual([ + ['private-bob', ['u:bob@corp.com']], + ]) + expect(bob.hasMore).toBe(false) + expect(ctx.getDelegatedAccessToken.mock.calls.map(([email]) => email)).toEqual([ + 'alice@corp.com', + 'bob@corp.com', + ]) + for (const [address, init] of mockFetch.mock.calls) { + const url = new URL(address) + const token = new Headers(init.headers).get('Authorization') + expect(token).toEqual( + url.hostname === 'admin.googleapis.com' + ? 'Bearer directory-token' + : expect.stringContaining('Bearer delegated:') + ) + if (url.pathname.endsWith('/files')) { + expect(url.searchParams.get('corpora')).toBe('user') + expect(url.searchParams.get('q')).not.toContain('modifiedTime >') + } + } + }) + + it('retains file IDs for deduplication and resolves a later reader through the full permissions endpoint', async () => { + const acl = [ + { type: 'user', emailAddress: 'alice@corp.com', role: 'owner' }, + { type: 'user', emailAddress: 'bob@corp.com', role: 'reader' }, + ] + fixture({ + files: { + 'alice@corp.com': [FILE('shared', 'alice@corp.com', { permissions: acl })], + 'bob@corp.com': [FILE('shared', 'alice@corp.com', { permissions: undefined })], + }, + permissions: { shared: acl }, + }) + const ctx = context() + const owner = await drive.listDocuments('directory-token', CONFIG, undefined, ctx) + const reader = await drive.listDocuments('directory-token', CONFIG, owner.nextCursor, ctx) + expect(reader.documents[0].externalId).toBe(owner.documents[0].externalId) + expect(reader.documents[0].contentHash).toBe(owner.documents[0].contentHash) + const resolved = await drive.getDocumentAcls!('directory-token', CONFIG, reader.documents, ctx) + expect(resolved.shared).toEqual(owner.documents[0].acl) + expect(resolved.shared).toEqual(['u:alice@corp.com', 'u:bob@corp.com']) + expect(mockFetch.mock.calls.at(-1)?.[1].headers.Authorization).toBe( + 'Bearer delegated:bob@corp.com' + ) + }) + + it('lets a downloadable owner index a shared file after an earlier reader cannot download it', async () => { + fixture({ + files: { + 'alice@corp.com': [ + FILE('shared', 'bob@corp.com', { capabilities: { canDownload: false } }), + ], + 'bob@corp.com': [FILE('shared', 'bob@corp.com', { capabilities: { canDownload: true } })], + }, + }) + const ctx = context() + const reader = await drive.listDocuments('directory-token', CONFIG, undefined, ctx) + expect(reader.documents).toEqual([]) + const owner = await drive.listDocuments('directory-token', CONFIG, reader.nextCursor, ctx) + expect(owner.documents.map((file) => file.externalId)).toEqual(['shared']) + expect((await drive.getDocument('directory-token', CONFIG, 'shared', ctx))?.content).toBe( + 'Body: shared' + ) + const fieldMasks = mockFetch.mock.calls + .filter(([address]) => new URL(address).pathname.endsWith('/files')) + .map(([address]) => new URL(address).searchParams.get('fields')) + expect(fieldMasks.every((fields) => fields?.includes('capabilities(canDownload)'))).toBe(true) + }) + + it('omits a file when no selected user can download it', async () => { + fixture({ + files: { + 'alice@corp.com': [ + FILE('shared', 'bob@corp.com', { capabilities: { canDownload: false } }), + ], + 'bob@corp.com': [FILE('shared', 'bob@corp.com', { capabilities: { canDownload: false } })], + }, + }) + const first = await drive.listDocuments('directory-token', CONFIG, undefined, context()) + const second = await drive.listDocuments('directory-token', CONFIG, first.nextCursor, context()) + expect([...first.documents, ...second.documents]).toEqual([]) + expect(second.hasMore).toBe(false) + }) + + it('checks the shortcut target download capability rather than the shortcut capability', async () => { + fixture({ + files: { + 'alice@corp.com': [ + FILE('shortcut', 'alice@corp.com', { + mimeType: 'application/vnd.google-apps.shortcut', + capabilities: { canDownload: true }, + shortcutDetails: { targetId: 'target' }, + }), + ], + }, + }) + const route = mockFetch.getMockImplementation()! + mockFetch.mockImplementation(async (address: string, init: RequestInit) => { + const url = new URL(address) + if (url.pathname.endsWith('/files/target')) { + expect(url.searchParams.get('fields')).toContain('capabilities(canDownload)') + return json(FILE('target', 'alice@corp.com', { capabilities: { canDownload: false } })) + } + return route(address, init) + }) + const page = await drive.listDocuments('directory-token', CONFIG, undefined, context()) + expect(page.documents).toEqual([]) + }) + + it('lets an owner index a shortcut after the earlier reader cannot access its target', async () => { + const shortcut = FILE('shortcut', 'bob@corp.com', { + mimeType: 'application/vnd.google-apps.shortcut', + capabilities: { canDownload: false }, + shortcutDetails: { targetId: 'target' }, + }) + fixture({ files: { 'alice@corp.com': [shortcut], 'bob@corp.com': [shortcut] } }) + const route = mockFetch.getMockImplementation()! + mockFetch.mockImplementation(async (address: string, init: RequestInit) => { + const url = new URL(address) + if (url.pathname.endsWith('/files/target')) { + if (new Headers(init.headers).get('Authorization') === 'Bearer delegated:alice@corp.com') + return json({ error: { errors: [{ reason: 'notFound' }] } }, 404) + if (url.searchParams.get('alt') === 'media') return new Response('Target content') + return json(FILE('target', 'bob@corp.com', { capabilities: { canDownload: true } })) + } + return route(address, init) + }) + const ctx = context() + const reader = await drive.listDocuments('directory-token', CONFIG, undefined, ctx) + expect(reader.documents).toEqual([]) + const owner = await drive.listDocuments('directory-token', CONFIG, reader.nextCursor, ctx) + expect(owner.documents.map((file) => file.externalId)).toEqual(['shortcut']) + expect((await drive.getDocument('directory-token', CONFIG, 'shortcut', ctx))?.content).toBe( + 'Target content' + ) + }) + + it('checkpoints nested Drive pagination and resumes the exact user page with a fresh context', async () => { + fixture({}) + const route = mockFetch.getMockImplementation()! + mockFetch.mockImplementation(async (address: string, init: RequestInit) => { + const url = new URL(address) + if ( + url.pathname.endsWith('/files') && + new Headers(init.headers).get('Authorization') === 'Bearer delegated:alice@corp.com' + ) { + const second = url.searchParams.get('pageToken') === 'file-page-2' + return json({ + files: [FILE(second ? 'alice-2' : 'alice-1', 'alice@corp.com')], + ...(!second ? { nextPageToken: 'file-page-2' } : {}), + }) + } + return route(address, init) + }) + const first = await drive.listDocuments('directory-token', CONFIG, undefined, context()) + const second = await drive.listDocuments('directory-token', CONFIG, first.nextCursor, context()) + expect(second.documents[0].externalId).toBe('alice-2') + const replayContext = context() + const replay = await drive.listDocuments( + 'directory-token', + CONFIG, + second.currentCursor, + replayContext + ) + expect(replay.documents).toEqual(second.documents) + expect(replayContext.getDelegatedAccessToken).toHaveBeenCalledWith('alice@corp.com') + const third = await drive.listDocuments('directory-token', CONFIG, second.nextCursor, context()) + expect(third.documents[0].externalId).toBe('private-bob') + expect( + mockFetch.mock.calls.filter(([address]) => new URL(address).pathname.endsWith('/users')) + ).toHaveLength(1) + }) + + it('visits untouched shared-drive files, paginates drives and files, and replays the exact scope', async () => { + fixture({ users: [USER('alice@corp.com')] }) + const route = mockFetch.getMockImplementation()! + mockFetch.mockImplementation(async (address: string, init: RequestInit) => { + const url = new URL(address) + if (url.pathname.endsWith('/drives')) { + expect(new Headers(init.headers).get('Authorization')).toBe( + 'Bearer delegated:alice@corp.com' + ) + const token = url.searchParams.get('pageToken') + return json( + token === 'drives-3' + ? { drives: [{ id: 'drive-c' }] } + : token === 'drives-2' + ? { drives: [], nextPageToken: 'drives-3' } + : { drives: [{ id: 'drive-a' }, { id: 'drive-b' }], nextPageToken: 'drives-2' } + ) + } + if (url.pathname.endsWith('/files') && url.searchParams.get('corpora') === 'drive') { + const id = url.searchParams.get('driveId')! + const more = id === 'drive-a' && !url.searchParams.has('pageToken') + return json({ + files: [FILE(`${id}${more ? '-1' : '-2'}`, 'alice@corp.com')], + ...(more ? { nextPageToken: 'files-2' } : {}), + }) + } + return route(address, init) + }) + const pages = [] + let cursor: string | undefined + for (let index = 0; index < 10; index++) { + const page = await drive.listDocuments('directory-token', CONFIG, cursor, context()) + pages.push(page) + if (!page.hasMore) break + cursor = page.nextCursor + } + expect(pages.flatMap((page) => page.documents.map((file) => file.externalId))).toEqual([ + 'private-alice', + 'drive-a-1', + 'drive-a-2', + 'drive-b-2', + 'drive-c-2', + ]) + expect(pages.at(-1)?.hasMore).toBe(false) + const replayContext = context() + const replay = await drive.listDocuments( + 'directory-token', + CONFIG, + pages[2].currentCursor, + replayContext + ) + expect(replay.documents).toEqual(pages[2].documents) + expect(replayContext.getDelegatedAccessToken).toHaveBeenCalledExactlyOnceWith('alice@corp.com') + const params = new URL(mockFetch.mock.calls.at(-1)![0]).searchParams + expect(params.get('corpora')).toBe('drive') + expect(params.get('driveId')).toBe('drive-a') + expect(params.get('pageToken')).toBe('files-2') + expect(pages.every((page) => !page.currentCursor?.includes('delegated:'))).toBe(true) + }) + + it.each(['teamDriveMembershipRequired', 'notFound'])( + 'advances after a listed shared drive becomes inaccessible: %s', + async (reason) => { + fixture({ users: [USER('alice@corp.com')] }) + const route = mockFetch.getMockImplementation()! + mockFetch.mockImplementation(async (address: string, init: RequestInit) => { + const url = new URL(address) + if (url.pathname.endsWith('/drives')) + return json({ drives: [{ id: 'removed' }, { id: 'visible' }] }) + if (url.pathname.endsWith('/files') && url.searchParams.get('corpora') === 'drive') { + return url.searchParams.get('driveId') === 'removed' + ? json({ error: { errors: [{ reason }] } }, reason === 'notFound' ? 404 : 403) + : json({ files: [FILE('still-visible', 'alice@corp.com')] }) + } + return route(address, init) + }) + const personal = await drive.listDocuments('directory-token', CONFIG, undefined, context()) + const removed = await drive.listDocuments( + 'directory-token', + CONFIG, + personal.nextCursor, + context() + ) + expect(removed.documents).toEqual([]) + expect(removed.hasMore).toBe(true) + const visible = await drive.listDocuments( + 'directory-token', + CONFIG, + removed.nextCursor, + context() + ) + expect(visible.documents.map((file) => file.externalId)).toEqual(['still-visible']) + expect(visible.hasMore).toBe(false) + } + ) + + it('continues other selected folders when one folder in a shared drive becomes inaccessible', async () => { + fixture({ users: [USER('alice@corp.com')] }) + const route = mockFetch.getMockImplementation()! + mockFetch.mockImplementation(async (address: string, init: RequestInit) => { + const url = new URL(address) + if (url.pathname.endsWith('/drives')) return json({ drives: [{ id: 'drive-a' }] }) + if (url.pathname.endsWith('/files')) { + if (url.searchParams.get('corpora') !== 'drive') return json({ files: [] }) + return url.searchParams.get('q')?.includes("'removed' in parents") + ? json({ error: { errors: [{ reason: 'notFound' }] } }, 404) + : json({ files: [FILE('visible-folder-file', 'alice@corp.com')] }) + } + return route(address, init) + }) + let cursor: string | undefined + const ids: string[] = [] + for (let index = 0; index < 8; index++) { + const page = await drive.listDocuments( + 'directory-token', + { ...CONFIG, folderId: ['visible', 'removed'] }, + cursor, + context() + ) + ids.push(...page.documents.map((file) => file.externalId)) + if (!page.hasMore) break + cursor = page.nextCursor + } + expect(ids).toEqual(['visible-folder-file']) + }) + + it.each([401, 403])( + 'does not treat a shared-drive authorization failure as an empty drive: %s', + async (status) => { + fixture({ users: [USER('alice@corp.com')] }) + const route = mockFetch.getMockImplementation()! + mockFetch.mockImplementation(async (address: string, init: RequestInit) => { + const url = new URL(address) + if (url.pathname.endsWith('/drives')) return json({ drives: [{ id: 'drive-a' }] }) + if (url.pathname.endsWith('/files') && url.searchParams.get('corpora') === 'drive') + return json({ error: {} }, status) + return route(address, init) + }) + const first = await drive.listDocuments('directory-token', CONFIG, undefined, context()) + await expect( + drive.listDocuments('directory-token', CONFIG, first.nextCursor, context()) + ).rejects.toThrow(GoogleDriveApiError) + } + ) + + it('paginates Directory users, including an empty page, without preloading the company', async () => { + fixture({}) + const route = mockFetch.getMockImplementation()! + mockFetch.mockImplementation(async (address: string, init: RequestInit) => { + const url = new URL(address) + if (url.pathname.endsWith('/users')) { + const token = url.searchParams.get('pageToken') + expect(url.searchParams.get('maxResults')).toBe('100') + return json( + token === 'users-3' + ? { users: [USER('bob@corp.com')] } + : token === 'users-2' + ? { users: [], nextPageToken: 'users-3' } + : { users: [USER('alice@corp.com')], nextPageToken: 'users-2' } + ) + } + return route(address, init) + }) + const one = await drive.listDocuments('directory-token', CONFIG, undefined, context()) + const two = await drive.listDocuments('directory-token', CONFIG, one.nextCursor, context()) + expect(two.documents).toEqual([]) + expect(two.hasMore).toBe(true) + const three = await drive.listDocuments('directory-token', CONFIG, two.nextCursor, context()) + expect(three.documents[0].externalId).toBe('private-bob') + expect(three.hasMore).toBe(false) + }) + + it('delegates only to selected primary emails found in the actual Workspace directory', async () => { + fixture({}) + const ctx = context() + const page = await drive.listDocuments( + 'directory-token', + { ...CONFIG, userEmails: ['Bob@Corp.com'] }, + undefined, + ctx + ) + expect(page.documents.map((doc) => doc.externalId)).toEqual(['private-bob']) + expect(ctx.getDelegatedAccessToken).toHaveBeenCalledExactlyOnceWith('bob@corp.com') + }) + + it.each([{ suspended: true }, { archived: true }, { isGuestUser: true }])( + 'skips inactive users without impersonating them: %j', + async (inactive) => { + fixture({ users: [USER('alice@corp.com', inactive), USER('bob@corp.com')] }) + const ctx = context() + const page = await drive.listDocuments('directory-token', CONFIG, undefined, ctx) + expect(page.documents[0].externalId).toBe('private-bob') + expect(ctx.getDelegatedAccessToken).not.toHaveBeenCalledWith('alice@corp.com') + } + ) + + it('rechecks suspension before resuming a previously listed user', async () => { + fixture({}) + const first = await drive.listDocuments('directory-token', CONFIG, undefined, context()) + fixture({ users: [USER('alice@corp.com'), USER('bob@corp.com', { suspended: true })] }) + const ctx = context() + const second = await drive.listDocuments('directory-token', CONFIG, first.nextCursor, ctx) + expect(second.documents).toEqual([]) + expect(second.hasMore).toBe(false) + expect(ctx.getDelegatedAccessToken).not.toHaveBeenCalled() + }) + + it('propagates delegation revocation rather than completing an empty crawl', async () => { + fixture({}) + const ctx = context() + ctx.getDelegatedAccessToken.mockRejectedValue(new Error('Delegation revoked')) + await expect(drive.listDocuments('directory-token', CONFIG, undefined, ctx)).rejects.toThrow( + 'Delegation revoked' + ) + }) + + it('fails closed if a reader cannot retrieve permissions', async () => { + fixture({ + files: { 'alice@corp.com': [FILE('file', 'alice@corp.com', { permissions: undefined })] }, + }) + const ctx = context() + const page = await drive.listDocuments('directory-token', CONFIG, undefined, ctx) + mockFetch.mockResolvedValue( + json({ error: { errors: [{ reason: 'insufficientFilePermissions' }] } }, 403) + ) + await expect( + drive.getDocumentAcls!('directory-token', CONFIG, page.documents, ctx) + ).resolves.toEqual({}) + }) + + it('returns the current revoked ACL on a later crawl even when content is unchanged', async () => { + fixture({ + files: { + 'alice@corp.com': [ + FILE('same', 'alice@corp.com', { + permissions: [ + { type: 'user', emailAddress: 'alice@corp.com', role: 'owner' }, + { type: 'user', emailAddress: 'bob@corp.com', role: 'reader' }, + ], + }), + ], + }, + }) + const before = await drive.listDocuments('directory-token', CONFIG, undefined, context()) + fixture({ files: { 'alice@corp.com': [FILE('same', 'alice@corp.com')] } }) + const after = await drive.listDocuments( + 'directory-token', + CONFIG, + undefined, + context(), + new Date() + ) + expect(after.documents[0].contentHash).toBe(before.documents[0].contentHash) + expect(after.documents[0].acl).toEqual(['u:alice@corp.com']) + }) + + it('refuses hydration without a verified listing identity and refuses nondelegated central tokens', async () => { + const ctx = context() + await expect(drive.getDocument('directory-token', CONFIG, 'unlisted', ctx)).rejects.toThrow( + 'verified delegated listing identity' + ) + await expect( + drive.listDocuments('ordinary-oauth', CONFIG, undefined, { mirrorsSourceAcls: true }) + ).rejects.toThrow('requires a delegated service account') + expect(mockFetch).not.toHaveBeenCalled() + }) + + it('marks a partial provider search unsafe for deletion reconciliation', async () => { + fixture({}) + const route = mockFetch.getMockImplementation()! + mockFetch.mockImplementation(async (address: string, init: RequestInit) => + new URL(address).pathname.endsWith('/files') + ? json({ files: [], incompleteSearch: true }) + : route(address, init) + ) + const page = await drive.listDocuments('directory-token', CONFIG, undefined, context()) + expect(page.reconciliationSafe).toBe(false) + }) + + it('stops before provider access when cancelled', async () => { + const controller = new AbortController() + controller.abort() + await expect( + drive.listDocuments('directory-token', CONFIG, undefined, { + ...context(), + signal: controller.signal, + }) + ).rejects.toThrow() + expect(mockFetch).not.toHaveBeenCalled() + }) +}) + +describe('Company-wide setup validation', () => { + it('validates only a bounded active user sample by default', async () => { + fixture({}) + await expect(validateGoogleCompanyConfig('directory-token', CONFIG, context())).resolves.toBe( + 'alice@corp.com' + ) + const sample = mockFetch.mock.calls.find(([address]) => + new URL(address).pathname.endsWith('/users') + ) + expect(new URL(sample![0]).searchParams.get('maxResults')).toBe('1') + expect(mockFetch).toHaveBeenCalledTimes(2) + }) + + it.each([ + ['different customer', USER('other@elsewhere.com', { customerId: 'customer-2' })], + ['alias', USER('primary@corp.com', { id: 'alias@corp.com' })], + ['suspended', USER('suspended@corp.com', { suspended: true })], + ['archived', USER('archived@corp.com', { archived: true })], + ['guest', USER('guest@corp.com', { isGuestUser: true })], + ])('rejects a selected user that is %s', async (_reason, user) => { + fixture({ users: [USER('admin@corp.com'), user] }) + await expect( + validateGoogleCompanyConfig( + 'directory-token', + { ...CONFIG, userEmails: [user.id] }, + context() + ) + ).rejects.toThrow() + }) + + it('probes delegated Drive access without requiring the administrator to open selected folders', async () => { + fixture({}) + const route = mockFetch.getMockImplementation()! + mockFetch.mockImplementation(async (address: string, init: RequestInit) => { + const path = new URL(address).pathname + if (path.endsWith('/groups')) return json({ groups: [] }) + if (path.endsWith('/domains')) return json({ domains: [] }) + return route(address, init) + }) + const ctx = context() + await expect( + drive.validateConfig( + 'directory-token', + { ...CONFIG, folderId: 'private-employee-folder' }, + ctx + ) + ).resolves.toEqual({ valid: true }) + expect(ctx.getDelegatedAccessToken).toHaveBeenCalledExactlyOnceWith('alice@corp.com') + const driveRequests = mockFetch.mock.calls.filter( + ([address]) => new URL(address).hostname === 'www.googleapis.com' + ) + expect(driveRequests).toHaveLength(1) + expect(new URL(driveRequests[0][0]).searchParams.get('pageSize')).toBe('1') + expect(driveRequests[0][1].headers.Authorization).toBe('Bearer delegated:alice@corp.com') + }) + + it('falls back to the verified administrator if the filtered sample is inactive or empty', async () => { + fixture({ users: [USER('admin@corp.com')] }) + const route = mockFetch.getMockImplementation()! + mockFetch.mockImplementation(async (address: string, init: RequestInit) => { + const url = new URL(address) + if (url.pathname.endsWith('/users')) { + expect(url.searchParams.get('query')).toBe( + 'isSuspended=false isArchived=false isGuest=false' + ) + return json({ users: [USER('suspended@corp.com', { suspended: true })] }) + } + return route(address, init) + }) + await expect(validateGoogleCompanyConfig('directory-token', CONFIG, context())).resolves.toBe( + 'admin@corp.com' + ) + }) + + it('rejects disabled Drive access during setup', async () => { + fixture({}) + const route = mockFetch.getMockImplementation()! + mockFetch.mockImplementation(async (address: string, init: RequestInit) => { + const path = new URL(address).pathname + if (path.endsWith('/groups')) return json({ groups: [] }) + if (path.endsWith('/domains')) return json({ domains: [] }) + if (new URL(address).hostname === 'www.googleapis.com') + return json({ error: { errors: [{ reason: 'accessNotConfigured' }] } }, 403) + return route(address, init) + }) + await expect(drive.validateConfig('directory-token', CONFIG, context())).resolves.toMatchObject( + { valid: false, error: expect.stringContaining('403') } + ) + }) +}) + +describe('Workspace user enumeration boundaries', () => { + it.each([ + {}, + { users: [{ primaryEmail: 'alice@corp.com' }] }, + { users: [USER('alice@corp.com', { id: 'x'.repeat(257) })] }, + { users: [USER('alice@corp.com', { customerId: 'x'.repeat(257) })] }, + { users: [], nextPageToken: '' }, + { users: [], nextPageToken: 'x'.repeat(8193) }, + ])('rejects malformed directory data: %j', async (body) => { + mockFetch.mockResolvedValue(json(body)) + await expect(listGoogleWorkspaceUsers('token')).rejects.toThrow('malformed') + }) + it('rejects repeated continuation and directory authorization failures', async () => { + mockFetch.mockResolvedValueOnce(json({ users: [], nextPageToken: 'same' })) + await expect(listGoogleWorkspaceUsers('token', 'same')).rejects.toThrow('repeated') + mockFetch.mockResolvedValueOnce(json({ error: { errors: [{ reason: 'forbidden' }] } }, 403)) + await expect(listGoogleWorkspaceUsers('token')).rejects.toBeInstanceOf(GoogleDriveApiError) + }) + it('validates explicit user emails and their bound', () => { + expect(selectedGoogleWorkspaceUsers(' Alice@Corp.com,alice@corp.com ')).toEqual([ + 'alice@corp.com', + ]) + expect(() => selectedGoogleWorkspaceUsers('not-an-email')).toThrow('valid') + expect(() => selectedGoogleWorkspaceUsers(['alice@corp.com', 123])).toThrow() + expect(() => selectedGoogleWorkspaceUsers({ email: 'alice@corp.com' })).toThrow() + expect(() => + selectedGoogleWorkspaceUsers(Array.from({ length: 101 }, (_, i) => `u${i}@corp.com`)) + ).toThrow('100') + }) +}) diff --git a/apps/sim/connectors/google-drive/company-crawl.ts b/apps/sim/connectors/google-drive/company-crawl.ts new file mode 100644 index 00000000000..4a4815bc424 --- /dev/null +++ b/apps/sim/connectors/google-drive/company-crawl.ts @@ -0,0 +1,272 @@ +import { normalizeEmail } from '@sim/utils/string' +import { z } from 'zod' +import { mapWithConcurrency } from '@/lib/core/utils/concurrency' +import { GOOGLE_DRIVE_ADMIN_EMAIL_FIELD_ID } from '@/connectors/google-drive/meta' +import { + GOOGLE_WORKSPACE_DRIVES_PAGE_SIZE, + listGoogleWorkspaceDrives, +} from '@/connectors/google-drive/workspace-drives' +import { + GOOGLE_WORKSPACE_USERS_PAGE_SIZE, + getGoogleWorkspaceUser, + listGoogleWorkspaceUsers, + selectedGoogleWorkspaceUsers, +} from '@/connectors/google-workspace/users' +import type { ConnectorConfig, ExternalDocumentList } from '@/connectors/types' +import { parseOptionalUnlimitedSafeInteger } from '@/connectors/utils' + +const CURSOR_PREFIX = 'gdrive-company:v1:' +const MAX_CURSOR_BYTES = 384 * 1024 +const cursorSchema = z.object({ + users: z + .array( + z.object({ + id: z.string().min(1).max(256), + email: z.string().email().max(254), + customerId: z.string().min(1).max(256), + }) + ) + .max(GOOGLE_WORKSPACE_USERS_PAGE_SIZE), + nextUsersPageToken: z.string().min(1).max(8192).optional(), + scope: z.discriminatedUnion('kind', [ + z.object({ + kind: z.literal('user'), + cursor: z + .string() + .min(1) + .max(256 * 1024) + .optional(), + }), + z.object({ kind: z.literal('drives'), pageToken: z.string().min(1).max(8192) }), + z.object({ + kind: z.literal('drive'), + driveIds: z.array(z.string().min(1).max(256)).min(1).max(GOOGLE_WORKSPACE_DRIVES_PAGE_SIZE), + nextPageToken: z.string().min(1).max(8192).optional(), + cursor: z + .string() + .min(1) + .max(256 * 1024) + .optional(), + }), + ]), +}) +type CompanyCursor = z.infer + +type DelegatedTokenResolver = (subject: string) => Promise + +function delegatedTokenResolver(syncContext: Record): DelegatedTokenResolver { + if (typeof syncContext.getDelegatedAccessToken !== 'function') { + throw new Error('Company-wide Google Drive indexing requires a delegated service account') + } + return syncContext.getDelegatedAccessToken as DelegatedTokenResolver +} + +function cancellationSignal(syncContext?: Record): AbortSignal | undefined { + return syncContext?.signal instanceof AbortSignal ? syncContext.signal : undefined +} + +export class InvalidGoogleCompanyCursor extends Error {} + +function readCursor(cursor: string): CompanyCursor { + try { + if (!cursor.startsWith(CURSOR_PREFIX) || cursor.length > MAX_CURSOR_BYTES) throw new Error() + return cursorSchema.parse( + JSON.parse(Buffer.from(cursor.slice(CURSOR_PREFIX.length), 'base64url').toString('utf8')) + ) + } catch { + throw new InvalidGoogleCompanyCursor( + 'Google Workspace crawl cursor is invalid; restart the sync' + ) + } +} + +function writeCursor(cursor: CompanyCursor): string { + const serialized = `${CURSOR_PREFIX}${Buffer.from(JSON.stringify(cursor)).toString('base64url')}` + if (serialized.length > MAX_CURSOR_BYTES) { + throw new Error('Google Workspace crawl exceeded its continuation-size limit') + } + return serialized +} + +/** Positive caps can stop before later users; central sources must finish their selected corpus. */ +function validateCompanyConfig(sourceConfig: Record): string[] { + if ( + parseOptionalUnlimitedSafeInteger( + sourceConfig.maxFiles, + 'Max Files must be a positive safe integer, or 0 for unlimited' + ) > 0 + ) { + throw new Error( + 'Max Files is not supported for company-wide indexing; narrow Users or Folders instead' + ) + } + return selectedGoogleWorkspaceUsers(sourceConfig.userEmails) +} + +/** Verifies selected identities belong to the same Workspace customer before saving. */ +export async function validateGoogleCompanyConfig( + accessToken: string, + sourceConfig: Record, + syncContext: Record +): Promise { + delegatedTokenResolver(syncContext) + const selected = validateCompanyConfig(sourceConfig) + const adminEmail = sourceConfig[GOOGLE_DRIVE_ADMIN_EMAIL_FIELD_ID] + if (typeof adminEmail !== 'string' || !adminEmail.trim()) { + throw new Error('Enter a Directory administrator email') + } + const administrator = await getGoogleWorkspaceUser(accessToken, normalizeEmail(adminEmail), { + validate: true, + }) + if (!administrator?.active) + throw new Error('The Directory administrator must be an active Google Workspace user') + await mapWithConcurrency(selected, 4, async (email) => { + const user = await getGoogleWorkspaceUser(accessToken, email, { validate: true }) + if (!user || user.customerId !== administrator.customerId || user.email !== email) { + throw new Error(`User "${email}" is not a primary email in this Google Workspace customer`) + } + if (!user.active) + throw new Error(`User "${email}" must be an active, non-guest Google Workspace user`) + }) + if (selected.length > 0) return selected[0] + const sample = await listGoogleWorkspaceUsers(accessToken, undefined, undefined, { + validate: true, + }) + return sample.users.find((user) => user.active)?.email ?? administrator.email +} + +/** + * Visits one user's Drive page per call. A bounded Directory page and the nested + * Drive cursor are durable; delegated tokens and per-page document identities are not. + */ +export async function listGoogleCompanyDocuments(input: { + accessToken: string + sourceConfig: Record + cursor?: string + syncContext: Record + listUserDocuments: ConnectorConfig['listDocuments'] +}): Promise { + const { accessToken, sourceConfig, syncContext, listUserDocuments } = input + const resolveToken = delegatedTokenResolver(syncContext) + syncContext.googleDrivePageAccess = undefined + const selected = validateCompanyConfig(sourceConfig) + const signal = cancellationSignal(syncContext) + signal?.throwIfAborted() + let state: CompanyCursor = input.cursor + ? readCursor(input.cursor) + : { users: [], scope: { kind: 'user' } } + if (state.users.length === 0) { + const page = await listGoogleWorkspaceUsers(accessToken, state.nextUsersPageToken, signal) + state = { + users: page.users + .filter((user) => user.active && (!selected.length || selected.includes(user.email))) + .map(({ id, email, customerId }) => ({ id, email, customerId })), + nextUsersPageToken: page.nextPageToken, + scope: { kind: 'user' }, + } + } + let currentCursor = writeCursor(state) + const pending = state.users[0] + if (!pending) { + return { + documents: [], + currentCursor, + hasMore: Boolean(state.nextUsersPageToken), + nextCursor: state.nextUsersPageToken ? writeCursor(state) : undefined, + } + } + + const user = await getGoogleWorkspaceUser(accessToken, pending.id, { signal }) + if (user && (user.customerId !== pending.customerId || user.id !== pending.id)) { + throw new Error('Google Workspace user no longer belongs to the expected customer') + } + const advanceUser = (): CompanyCursor => ({ + users: state.users.slice(1), + nextUsersPageToken: state.nextUsersPageToken, + scope: { kind: 'user' }, + }) + if (!user?.active || (selected.length > 0 && !selected.includes(user.email))) { + state = advanceUser() + const hasMore = state.users.length > 0 || Boolean(state.nextUsersPageToken) + return { + documents: [], + currentCursor, + hasMore, + nextCursor: hasMore ? writeCursor(state) : undefined, + } + } + + signal?.throwIfAborted() + const userToken = await resolveToken(user.email) + const nextDrives = async (pageToken?: string): Promise => { + const drives = await listGoogleWorkspaceDrives(userToken, pageToken, signal) + if (drives.driveIds.length > 0) { + return { + ...state, + scope: { kind: 'drive', driveIds: drives.driveIds, nextPageToken: drives.nextPageToken }, + } + } + return drives.nextPageToken + ? { ...state, scope: { kind: 'drives', pageToken: drives.nextPageToken } } + : advanceUser() + } + if (state.scope.kind === 'drives') { + state = await nextDrives(state.scope.pageToken) + if (state.scope.kind !== 'drive') { + const hasMore = state.users.length > 0 || Boolean(state.nextUsersPageToken) + return { + documents: [], + currentCursor, + hasMore, + nextCursor: hasMore ? writeCursor(state) : undefined, + } + } + currentCursor = writeCursor(state) + } + syncContext.googleDriveSharedDriveId = + state.scope.kind === 'drive' ? state.scope.driveIds[0] : undefined + const page = await listUserDocuments(userToken, sourceConfig, state.scope.cursor, syncContext) + syncContext.googleDrivePageAccess = { + token: userToken, + externalIds: new Set(page.documents.map((document) => document.externalId)), + } + if (page.hasMore) { + if (!page.nextCursor) throw new Error('Google Drive omitted its continuation token') + state = { ...state, scope: { ...state.scope, cursor: page.nextCursor } } + } else if (state.scope.kind === 'user') { + state = await nextDrives() + } else if (state.scope.driveIds.length > 1) { + state = { + ...state, + scope: { ...state.scope, driveIds: state.scope.driveIds.slice(1), cursor: undefined }, + } + } else { + state = state.scope.nextPageToken ? await nextDrives(state.scope.nextPageToken) : advanceUser() + } + const hasMore = state.users.length > 0 || Boolean(state.nextUsersPageToken) + return { ...page, currentCursor, hasMore, nextCursor: hasMore ? writeCursor(state) : undefined } +} + +/** Hydration and permission reads use the same verified identity as their replayable listing page. */ +export async function googleDriveDocumentToken( + accessToken: string, + externalId: string, + syncContext?: Record +): Promise { + if (syncContext?.mirrorsSourceAcls !== true) return accessToken + const page = syncContext.googleDrivePageAccess + if ( + !page || + typeof page !== 'object' || + !('externalIds' in page) || + !(page.externalIds instanceof Set) || + !page.externalIds.has(externalId) || + !('token' in page) || + typeof page.token !== 'string' || + !page.token + ) { + throw new Error('Google Drive document has no verified delegated listing identity') + } + cancellationSignal(syncContext)?.throwIfAborted() + return page.token +} diff --git a/apps/sim/connectors/google-drive/directory.ts b/apps/sim/connectors/google-drive/directory.ts index eee338d0bbc..92b9ae54c54 100644 --- a/apps/sim/connectors/google-drive/directory.ts +++ b/apps/sim/connectors/google-drive/directory.ts @@ -62,9 +62,7 @@ export async function validateGoogleDirectoryAccess( adminEmail: unknown ): Promise { if (!googleWorkspaceDomain(adminEmail)) { - throw new Error( - 'Enter a Google Workspace administrator in Crawl as to mirror Drive permissions.' - ) + throw new Error('Enter a Directory administrator email to mirror Drive permissions.') } const probe = async (path: string) => @@ -87,7 +85,7 @@ export async function validateGoogleDirectoryAccess( error instanceof GoogleDriveApiError && (error.status === 401 || error.status === 403) && !error.rateLimited - ? ' The Crawl as account must have permission to read Workspace groups, memberships, and domains. Check its administrator privileges and the service account’s delegated Directory scopes.' + ? ' The Directory administrator must have permission to read Workspace users, groups, memberships, and domains. Check its administrator privileges and the service account’s delegated Directory scopes.' : '' throw new Error( `Google Workspace directory access failed: ${getErrorMessage(error)}.${guidance}`, diff --git a/apps/sim/connectors/google-drive/google-drive.test.ts b/apps/sim/connectors/google-drive/google-drive.test.ts index f25ecc6dd08..f538e5b81bd 100644 --- a/apps/sim/connectors/google-drive/google-drive.test.ts +++ b/apps/sim/connectors/google-drive/google-drive.test.ts @@ -8,6 +8,34 @@ const { mockFetch } = vi.hoisted(() => ({ mockFetch: vi.fn() })) vi.mock('@/components/icons', () => ({ GoogleDriveIcon: () => null })) +vi.mock('@/connectors/google-drive/workspace-drives', () => ({ + GOOGLE_WORKSPACE_DRIVES_PAGE_SIZE: 100, + listGoogleWorkspaceDrives: async () => ({ driveIds: [] }), +})) + +/** The file/ACL tests isolate Directory enumeration; company-crawl tests exercise its real HTTP boundary. */ +vi.mock('@/connectors/google-workspace/users', () => ({ + GOOGLE_WORKSPACE_USERS_PAGE_SIZE: 100, + selectedGoogleWorkspaceUsers: () => [], + listGoogleWorkspaceUsers: async () => ({ + users: [{ id: 'admin', email: 'admin@corp.com', customerId: 'customer', active: true }], + }), + getGoogleWorkspaceUser: async () => ({ + id: 'admin', + email: 'admin@corp.com', + customerId: 'customer', + active: true, + }), +})) + +function companyContext(): Record { + return { + mirrorsSourceAcls: true, + getDelegatedAccessToken: async () => 'token', + googleDrivePageAccess: { token: 'token', externalIds: new Set(['drive-file-1']) }, + } +} + afterEach(() => { vi.useRealTimers() vi.unstubAllGlobals() @@ -43,8 +71,11 @@ describe('Google Drive administrator setup', () => { it('requires a delegated administrator only when mirroring source permissions', async () => { await expect( - googleDriveConnector.validateConfig('token', {}, { mirrorsSourceAcls: true }) - ).resolves.toMatchObject({ valid: false, error: expect.stringContaining('Crawl as') }) + googleDriveConnector.validateConfig('token', {}, companyContext()) + ).resolves.toMatchObject({ + valid: false, + error: expect.stringContaining('Directory administrator email'), + }) expect(mockFetch).not.toHaveBeenCalled() mockFetch.mockResolvedValue(jsonResponse({ files: [] })) await expect(googleDriveConnector.validateConfig('token', {})).resolves.toEqual({ valid: true }) @@ -67,7 +98,7 @@ describe('Google Drive administrator setup', () => { googleDriveConnector.validateConfig( 'token', { adminEmail: 'admin@corp.com' }, - { mirrorsSourceAcls: true } + companyContext() ) ).resolves.toMatchObject({ valid: false, @@ -96,7 +127,7 @@ describe('Google Drive administrator setup', () => { googleDriveConnector.validateConfig( 'token', { adminEmail: 'admin@corp.com' }, - { mirrorsSourceAcls: true } + companyContext() ) ).resolves.toEqual({ valid: true }) expect(mockFetch).toHaveBeenCalledTimes(4) @@ -120,7 +151,7 @@ describe('Google Drive administrator setup', () => { googleDriveConnector.validateConfig( 'token', { adminEmail: 'admin@corp.com' }, - { mirrorsSourceAcls: true } + companyContext() ) ).resolves.toEqual({ valid: true }) expect(mockFetch).toHaveBeenCalledTimes(3) @@ -238,7 +269,7 @@ describe('Google Drive recursive folders and raw files', () => { }) ) const config = { folderId: 'root', adminEmail: 'admin@example.com' } - const context = { mirrorsSourceAcls: true } + const context = companyContext() const first = await googleDriveConnector.listDocuments('token', config, undefined, context) const second = await googleDriveConnector.listDocuments( 'token', @@ -995,7 +1026,7 @@ describe('mirroring Drive permissions onto listed documents', () => { }) /** The engine seeds this on every mirroring run; without it a crawl reads no permissions. */ - const MIRRORING = { mirrorsSourceAcls: true } + const MIRRORING = companyContext() async function listWith(file: Record, sourceConfig: Record) { mockFetch.mockResolvedValueOnce(fileListResponse([file])) diff --git a/apps/sim/connectors/google-drive/google-drive.ts b/apps/sim/connectors/google-drive/google-drive.ts index a6100525a0d..fdca1d94b87 100644 --- a/apps/sim/connectors/google-drive/google-drive.ts +++ b/apps/sim/connectors/google-drive/google-drive.ts @@ -12,6 +12,12 @@ import type { MirroredDocumentAcl } from '@/lib/knowledge/access/types' import { OCR_IMAGE_MIME_TYPES } from '@/lib/knowledge/documents/ocr-request-policy' import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { drainGooglePagedList } from '@/lib/oauth/google-pagination' +import { + googleDriveDocumentToken, + InvalidGoogleCompanyCursor, + listGoogleCompanyDocuments, + validateGoogleCompanyConfig, +} from '@/connectors/google-drive/company-crawl' import { googleWorkspaceDomain, openGoogleDirectory, @@ -249,6 +255,7 @@ interface DriveFile { starred?: boolean trashed?: boolean parents?: string[] + capabilities?: { canDownload?: boolean } shortcutDetails?: { targetId: string; targetMimeType?: string; targetResourceKey?: string } /** * Absent for a file on a shared drive, and for any file the impersonated @@ -539,9 +546,8 @@ function driveAclContext( * The file's mirrored ACL from its listing, or undefined when the listing * cannot speak for it and {@link resolveDriveAcls} must. * - * Drive leaves `permissions` unpopulated for a file on a shared drive, and for - * any file the requesting user cannot share. Those go to `permissions.list`, - * the one endpoint that answers for every file. + * Drive can omit `permissions` from a listing, including on shared drives. + * Those files need `permissions.list`, which may also refuse the requesting user. */ function fileAcl(file: DriveFile, context: DriveAclContext | null): string[] | undefined { if (!context || !file.permissions) return undefined @@ -558,8 +564,7 @@ const MAX_PERMISSION_PAGES = 50 const DRIVE_PERMISSION_FIELDS = 'id,type,emailAddress,domain,role,allowFileDiscovery,deleted' /** - * A file's full permission list, from the one endpoint that serves it for every - * file — including those on a shared drive, whose listing carries none. + * A file's full permission list, when the requesting user may read it. * * Throws rather than returning a partial list: a file mirrored under the * permissions that happened to arrive is a file whose missing grants nobody @@ -604,9 +609,8 @@ async function listFilePermissions( * The ACLs of files whose listing could not describe them — every file on a * shared drive, whose listing carries no permissions at all. * - * A file whose permissions cannot be read is omitted, which leaves it readable - * by nobody until a run can read them: the failure is logged per file and the - * rest of the batch still resolves. + * Omitted files have unresolved permissions. The sync engine keeps them hidden + * unless another observation verified their ACL during the same crawl. */ async function resolveDriveAcls( accessToken: string, @@ -621,17 +625,21 @@ async function resolveDriveAcls( await mapWithConcurrency(documents, PERMISSION_FETCH_CONCURRENCY, async (document) => { const fileId = document.externalId try { - if (document.metadata?.shortcutTargetId) { - const file = await readDriveFile(accessToken, fileId, undefined, true) + const documentToken = await googleDriveDocumentToken(accessToken, fileId, syncContext) + if ( + document.metadata?.shortcutTargetId || + document.metadata?.originalMimeType === SHORTCUT_MIME_TYPE + ) { + const file = await readDriveFile(documentToken, fileId, undefined, true) if (file.trashed) { acls[fileId] = [] return } if (file.mimeType === SHORTCUT_MIME_TYPE) { - const target = await readShortcutTarget(accessToken, file, true) + const target = await readShortcutTarget(documentToken, file, true) acls[fileId] = target && isSupportedFile(target.file) - ? await shortcutAcl(accessToken, file, target.file, context, target.resourceKey) + ? await shortcutAcl(documentToken, file, target.file, context, target.resourceKey) : [] return } @@ -641,10 +649,10 @@ async function resolveDriveAcls( return } } - const permissions = await listFilePermissions(accessToken, fileId) + const permissions = await listFilePermissions(documentToken, fileId) acls[fileId] = driveFileAcl({ ...context, permissions }) } catch (error) { - logger.warn("Could not read a file's permissions; it stays readable by nobody", { + logger.warn("Could not verify a file's permissions with the current account", { fileId, ...googleDriveErrorLogFields(error), }) @@ -723,7 +731,7 @@ async function readDriveFile( resourceKey?: string, permissions = false ): Promise { - const fields = `${DRIVE_FILE_FIELDS}${permissions ? `,permissions(${DRIVE_PERMISSION_FIELDS})` : ''}` + const fields = `${DRIVE_FILE_FIELDS}${permissions ? `,permissions(${DRIVE_PERMISSION_FIELDS}),capabilities(canDownload)` : ''}` const response = await fetchGoogleDriveWithRetry( `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true`, { method: 'GET', headers: driveRequestHeaders(accessToken, fileId, resourceKey) } @@ -811,6 +819,14 @@ async function listedFileToDocument( ): Promise { if (file.trashed || file.mimeType === FOLDER_MIME_TYPE) return null const context = driveAclContext(sourceConfig, syncContext) + const companyWide = syncContext?.mirrorsSourceAcls === true + /** A download-restricted reader must not reserve a file before a later owner can index it. */ + if ( + companyWide && + file.mimeType !== SHORTCUT_MIME_TYPE && + file.capabilities?.canDownload === false + ) + return null let target: DriveFile | undefined let acl: MirroredDocumentAcl | undefined = fileAcl(file, context) if (file.mimeType === SHORTCUT_MIME_TYPE) { @@ -824,10 +840,12 @@ async function listedFileToDocument( 'Could not resolve shortcut target; deferring to content hydration', googleDriveErrorLogFields(error) ) - return fileToStub(file, []) + return fileToStub(file, context ? undefined : []) } - if (!resolved) return isPerMemberListing(syncContext) ? null : unavailableShortcut(file) + if (!resolved) + return isPerMemberListing(syncContext) || companyWide ? null : unavailableShortcut(file) target = resolved.file + if (companyWide && target.capabilities?.canDownload === false) return null if (context) { try { acl = await shortcutAcl(accessToken, file, target, context, resolved.resourceKey) @@ -836,7 +854,7 @@ async function listedFileToDocument( 'Could not verify shortcut and target permissions', googleDriveErrorLogFields(error) ) - acl = [] + acl = undefined } } } @@ -1003,171 +1021,186 @@ function writeTraversal(state: FolderTraversal): string { return cursor } -export const googleDriveConnector: ConnectorConfig = { - isCredentialInvalidError: (error) => error instanceof GoogleDriveApiError && error.status === 401, - ...googleDriveConnectorMeta, - - listDocuments: async ( - accessToken: string, - sourceConfig: Record, - cursor?: string, - syncContext?: Record, - lastSyncAt?: Date - ): Promise => { - const roots = [...new Set(parseMultiValue(sourceConfig.folderId))] - const traversal: FolderTraversal | undefined = roots.length - ? cursor - ? readTraversal(cursor, roots) - : { pending: roots.map((id) => ({ id, depth: 0 })), totalFetched: 0 } - : undefined - const folder = traversal?.pending.pop() - /** Folder moves affect descendants without changing their modified timestamps. */ - const query = buildQuery( - folder ? { ...sourceConfig, folderId: folder.id } : sourceConfig, - folder ? undefined : lastSyncAt, - Boolean(folder) - ) - const pageSize = 100 +const listGoogleDriveDocuments: ConnectorConfig['listDocuments'] = async ( + accessToken: string, + sourceConfig: Record, + cursor?: string, + syncContext?: Record, + lastSyncAt?: Date +): Promise => { + const roots = [...new Set(parseMultiValue(sourceConfig.folderId))] + const traversal: FolderTraversal | undefined = roots.length + ? cursor + ? readTraversal(cursor, roots) + : { pending: roots.map((id) => ({ id, depth: 0 })), totalFetched: 0 } + : undefined + const folder = traversal?.pending.pop() + /** Folder moves affect descendants without changing their modified timestamps. */ + const query = buildQuery( + folder ? { ...sourceConfig, folderId: folder.id } : sourceConfig, + folder ? undefined : lastSyncAt, + Boolean(folder) + ) + const pageSize = 100 - const maxFiles = parseMaxFiles(sourceConfig.maxFiles) - const previouslyFetched = - traversal?.totalFetched ?? (syncContext?.totalDocsFetched as number) ?? 0 + const maxFiles = parseMaxFiles(sourceConfig.maxFiles) + const previouslyFetched = + traversal?.totalFetched ?? (syncContext?.totalDocsFetched as number) ?? 0 - if (maxFiles > 0 && previouslyFetched >= maxFiles) { - return { documents: [], hasMore: false } - } + if (maxFiles > 0 && previouslyFetched >= maxFiles) { + return { documents: [], hasMore: false } + } - const remaining = maxFiles > 0 ? maxFiles - previouslyFetched : 0 - const effectivePageSize = maxFiles > 0 ? Math.min(pageSize, remaining) : pageSize + const remaining = maxFiles > 0 ? maxFiles - previouslyFetched : 0 + const effectivePageSize = maxFiles > 0 ? Math.min(pageSize, remaining) : pageSize - const aclContext = driveAclContext(sourceConfig, syncContext) - const queryParams = new URLSearchParams({ - q: query, - pageSize: String(effectivePageSize), - orderBy: 'modifiedTime desc', - /** - * Permissions ride along only where the run mirrors them. Every other - * crawl would pull a permission array per file and discard it. - */ - fields: `kind,nextPageToken,incompleteSearch,files(${DRIVE_FILE_FIELDS}${ - aclContext ? `,permissions(${DRIVE_PERMISSION_FIELDS})` : '' - })`, - supportsAllDrives: 'true', - includeItemsFromAllDrives: 'true', - }) + const aclContext = driveAclContext(sourceConfig, syncContext) + const sharedDriveId = + syncContext?.mirrorsSourceAcls === true && + typeof syncContext.googleDriveSharedDriveId === 'string' + ? syncContext.googleDriveSharedDriveId + : undefined + const queryParams = new URLSearchParams({ + q: query, + pageSize: String(effectivePageSize), + orderBy: 'modifiedTime desc', + /** + * Permissions ride along only where the run mirrors them. Every other + * crawl would pull a permission array per file and discard it. + */ + fields: `kind,nextPageToken,incompleteSearch,files(${DRIVE_FILE_FIELDS}${ + aclContext ? `,permissions(${DRIVE_PERMISSION_FIELDS}),capabilities(canDownload)` : '' + })`, + supportsAllDrives: 'true', + includeItemsFromAllDrives: 'true', + ...(syncContext?.mirrorsSourceAcls === true + ? sharedDriveId + ? { corpora: 'drive', driveId: sharedDriveId } + : { corpora: 'user' } + : {}), + }) - const pageToken = folder ? folder.pageToken : cursor - if (pageToken) { - queryParams.set('pageToken', pageToken) - } + const pageToken = folder ? folder.pageToken : cursor + if (pageToken) { + queryParams.set('pageToken', pageToken) + } - const url = `https://www.googleapis.com/drive/v3/files?${queryParams.toString()}` + const url = `https://www.googleapis.com/drive/v3/files?${queryParams.toString()}` - logger.info('Listing Google Drive files', { query, cursor: cursor ?? 'initial' }) + logger.info('Listing Google Drive files', { query, cursor: cursor ?? 'initial' }) - let response: Response - try { - response = await fetchGoogleDriveWithRetry(url, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) - } catch (error) { - if ( - traversal && - isPerMemberListing(syncContext) && - error instanceof GoogleDriveApiError && - (error.kind === 'not_found' || error.kind === 'permission') - ) { - return { - documents: [], - hasMore: traversal.pending.length > 0, - nextCursor: traversal.pending.length ? writeTraversal(traversal) : undefined, - } + let response: Response + try { + response = await fetchGoogleDriveWithRetry(url, { + method: 'GET', + signal: syncContext?.signal instanceof AbortSignal ? syncContext.signal : undefined, + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + }, + }) + } catch (error) { + if ( + (traversal || sharedDriveId) && + (isPerMemberListing(syncContext) || syncContext?.mirrorsSourceAcls === true) && + error instanceof GoogleDriveApiError && + (error.kind === 'not_found' || error.kind === 'permission') + ) { + return { + documents: [], + hasMore: Boolean(traversal?.pending.length), + nextCursor: traversal?.pending.length ? writeTraversal(traversal) : undefined, } - logger.error('Failed to list Google Drive files', googleDriveErrorLogFields(error)) - throw error } + logger.error('Failed to list Google Drive files', googleDriveErrorLogFields(error)) + throw error + } - const data = parseDriveFileListResponse(await readDriveJson(response, DRIVE_PAGE_MAX_BYTES)) - const files = data.files + const data = parseDriveFileListResponse(await readDriveJson(response, DRIVE_PAGE_MAX_BYTES)) + const files = data.files - /** - * Drive sets `incompleteSearch` when it could not search every corpus (it - * arises with the `allDrives` scope enabled by `includeItemsFromAllDrives`). - * A partial listing drops still-existing files, so reconciliation must be - * suppressed to avoid hard-deleting valid documents. - */ - const incompleteSearch = data.incompleteSearch === true + /** + * Drive sets `incompleteSearch` when it could not finish searching the requested + * corpus, especially when searching multiple shared drives with `allDrives`. + * A partial listing drops still-existing files, so reconciliation must be + * suppressed to avoid hard-deleting valid documents. + */ + const incompleteSearch = data.incompleteSearch === true - if (traversal && folder) { - if (data.nextPageToken) { - if (data.nextPageToken === folder.pageToken) { - throw new Error('Google Drive repeated a folder continuation token') - } - traversal.pending.push({ ...folder, pageToken: data.nextPageToken }) + if (traversal && folder) { + if (data.nextPageToken) { + if (data.nextPageToken === folder.pageToken) { + throw new Error('Google Drive repeated a folder continuation token') } - const children = [ - ...new Set( - files.filter((file) => file.mimeType === FOLDER_MIME_TYPE).map((file) => file.id) - ), - ] - for (const id of children) { - /** An explicitly selected descendant is walked as its own root. */ - if (roots.includes(id)) continue - if (folder.depth >= MAX_FOLDER_DEPTH) { - throw new Error('Google Drive folder traversal exceeded its nesting-depth limit') - } - traversal.pending.push({ id, depth: folder.depth + 1 }) + traversal.pending.push({ ...folder, pageToken: data.nextPageToken }) + } + const children = [ + ...new Set(files.filter((file) => file.mimeType === FOLDER_MIME_TYPE).map((file) => file.id)), + ] + for (const id of children) { + /** An explicitly selected descendant is walked as its own root. */ + if (roots.includes(id)) continue + if (folder.depth >= MAX_FOLDER_DEPTH) { + throw new Error('Google Drive folder traversal exceeded its nesting-depth limit') } + traversal.pending.push({ id, depth: folder.depth + 1 }) } + } - const resolved = await mapWithConcurrency(files, SHORTCUT_FETCH_CONCURRENCY, (file) => - listedFileToDocument(accessToken, sourceConfig, file, syncContext) - ) - const pageDocuments = resolved.filter((doc): doc is ExternalDocument => doc !== null) + const resolved = await mapWithConcurrency(files, SHORTCUT_FETCH_CONCURRENCY, (file) => + listedFileToDocument(accessToken, sourceConfig, file, syncContext) + ) + const pageDocuments = resolved.filter((doc): doc is ExternalDocument => doc !== null) - const page = takeIndexableWithinCap( - pageDocuments, - isSkippedDocument, - maxFiles, - previouslyFetched - ) + const page = takeIndexableWithinCap(pageDocuments, isSkippedDocument, maxFiles, previouslyFetched) - const totalFetched = previouslyFetched + page.indexableCount - if (syncContext) syncContext.totalDocsFetched = totalFetched - if (traversal) traversal.totalFetched = totalFetched - const hitLimit = page.capReached + const totalFetched = previouslyFetched + page.indexableCount + if (syncContext) syncContext.totalDocsFetched = totalFetched + if (traversal) traversal.totalFetched = totalFetched + const hitLimit = page.capReached - const nextPageToken = traversal - ? traversal.pending.length - ? writeTraversal(traversal) - : undefined - : data.nextPageToken + const nextPageToken = traversal + ? traversal.pending.length + ? writeTraversal(traversal) + : undefined + : data.nextPageToken - /** - * Suppress deletion reconciliation only when the listing really is partial. - * Drive omits `nextPageToken` once the end of the list is reached, so hitting - * `maxFiles` on the final page still represents the full source set and must - * stay reconcilable — otherwise a capped source can never drop deleted files. - */ - if ( - syncContext && - ((hitLimit && (Boolean(nextPageToken) || page.documents.length < pageDocuments.length)) || - incompleteSearch) - ) { - syncContext.listingCapped = true - } + /** + * Suppress deletion reconciliation only when the listing really is partial. + * Drive omits `nextPageToken` once the end of the list is reached, so hitting + * `maxFiles` on the final page still represents the full source set and must + * stay reconcilable — otherwise a capped source can never drop deleted files. + */ + if ( + syncContext && + ((hitLimit && (Boolean(nextPageToken) || page.documents.length < pageDocuments.length)) || + incompleteSearch) + ) { + syncContext.listingCapped = true + } - return { - documents: page.documents, - nextCursor: hitLimit ? undefined : nextPageToken, - hasMore: hitLimit ? false : Boolean(nextPageToken), - reconciliationSafe: incompleteSearch ? false : undefined, - } - }, + return { + documents: page.documents, + nextCursor: hitLimit ? undefined : nextPageToken, + hasMore: hitLimit ? false : Boolean(nextPageToken), + reconciliationSafe: incompleteSearch ? false : undefined, + } +} + +export const googleDriveConnector: ConnectorConfig = { + isCredentialInvalidError: (error) => error instanceof GoogleDriveApiError && error.status === 401, + ...googleDriveConnectorMeta, + + listDocuments: (accessToken, sourceConfig, cursor, syncContext, lastSyncAt) => + syncContext?.mirrorsSourceAcls === true + ? listGoogleCompanyDocuments({ + accessToken, + sourceConfig, + cursor, + syncContext, + listUserDocuments: listGoogleDriveDocuments, + }) + : listGoogleDriveDocuments(accessToken, sourceConfig, cursor, syncContext, lastSyncAt), openDirectory: async (accessToken, sourceConfig) => openGoogleDirectory( @@ -1182,8 +1215,10 @@ export const googleDriveConnector: ConnectorConfig = { getDocument: async ( accessToken: string, sourceConfig: Record, - externalId: string + externalId: string, + syncContext?: Record ): Promise => { + accessToken = await googleDriveDocumentToken(accessToken, externalId, syncContext) let file: DriveFile try { file = await readDriveFile(accessToken, externalId) @@ -1255,10 +1290,20 @@ export const googleDriveConnector: ConnectorConfig = { try { parseMaxFiles(sourceConfig.maxFiles) if (syncContext?.mirrorsSourceAcls === true) { + const sampleUser = await validateGoogleCompanyConfig(accessToken, sourceConfig, syncContext) await validateGoogleDirectoryAccess( accessToken, sourceConfig[GOOGLE_DRIVE_ADMIN_EMAIL_FIELD_ID] ) + if (sampleUser && typeof syncContext.getDelegatedAccessToken === 'function') { + const sampleToken = await syncContext.getDelegatedAccessToken(sampleUser) + await fetchGoogleDriveWithRetry( + 'https://www.googleapis.com/drive/v3/files?pageSize=1&fields=files(id)&corpora=user&supportsAllDrives=true&includeItemsFromAllDrives=true', + { headers: { Authorization: `Bearer ${sampleToken}`, Accept: 'application/json' } }, + VALIDATE_RETRY_OPTIONS + ) + } + return { valid: true } } if (folderIds.length > 0) { @@ -1449,5 +1494,7 @@ export const googleDriveConnector: ConnectorConfig = { isChangeCursorInvalidError: (error) => error instanceof InvalidDriveListingCursor || isDriveChangeCursorInvalidError(error), isListingCursorInvalidError: (error) => - error instanceof InvalidDriveListingCursor || isDriveChangeCursorInvalidError(error), + error instanceof InvalidDriveListingCursor || + error instanceof InvalidGoogleCompanyCursor || + isDriveChangeCursorInvalidError(error), } diff --git a/apps/sim/connectors/google-drive/meta.ts b/apps/sim/connectors/google-drive/meta.ts index 93e3b95a3cc..503cee2d70e 100644 --- a/apps/sim/connectors/google-drive/meta.ts +++ b/apps/sim/connectors/google-drive/meta.ts @@ -1,7 +1,7 @@ import { GoogleDriveIcon } from '@/components/icons' import type { ConnectorMeta } from '@/connectors/types' -/** The config field naming the administrator a service account crawls as. */ +/** Directory administrator for company-wide indexing; delegated identity for member content. */ export const GOOGLE_DRIVE_ADMIN_EMAIL_FIELD_ID = 'adminEmail' /** The config field saying how far open shares are searchable. */ export const GOOGLE_DRIVE_OPEN_SHARING_FIELD_ID = 'openSharing' @@ -21,15 +21,17 @@ export const googleDriveConnectorMeta: ConnectorMeta = { requiredScopes: ['https://www.googleapis.com/auth/drive'], adminCredentialType: 'service_account', /** - * Delegated crawls need read access only. The token acts as the configured - * administrator and sees files that account can access; delegation alone - * does not discover every user's files. Directory scopes resolve group grants. + * The administrator enumerates Workspace users and resolves directory grants. + * Each user's content token has only the Drive read scope. */ - serviceAccountScopes: [ + serviceAccountScopes: ['https://www.googleapis.com/auth/drive.readonly'], + adminServiceAccountScopes: [ 'https://www.googleapis.com/auth/drive.readonly', + 'https://www.googleapis.com/auth/admin.directory.user.readonly', 'https://www.googleapis.com/auth/admin.directory.group.readonly', 'https://www.googleapis.com/auth/admin.directory.domain.readonly', ], + serviceAccountDelegationScopes: ['https://www.googleapis.com/auth/drive.readonly'], serviceAccountSubjectFieldId: GOOGLE_DRIVE_ADMIN_EMAIL_FIELD_ID, }, @@ -41,17 +43,32 @@ export const googleDriveConnectorMeta: ConnectorMeta = { /** `files.list` reports each file's own permissions, so one crawl can mirror them. */ mirrorsSourceAcls: true, adminSetupHint: - 'Use a service account with domain-wide delegation and Google Workspace Directory access. Only files the administrator in Crawl as can access are indexed.', + 'Use a service account with domain-wide delegation and Google Workspace Directory access to index selected employees’ Drives with their existing permissions.', configFields: [ { id: GOOGLE_DRIVE_ADMIN_EMAIL_FIELD_ID, title: 'Crawl as', + titleInAdminMode: 'Directory administrator email', + descriptionInAdminMode: + 'A Google Workspace administrator who can read users, groups, memberships, and domains. Used for directory access; files are read as each selected user.', type: 'short-input', required: false, placeholder: 'admin@yourcompany.com', description: - 'A Google Workspace administrator the service account acts as. Only files this account can access are indexed. Required to mirror Drive permissions; leave blank for your own Google account or files shared directly with the service account.', + 'The Google Workspace user this service account acts as when fetching content for connected members.', + }, + { + id: 'userEmails', + title: 'Users', + showInAdminModeOnly: true, + setupGroup: 'options', + type: 'short-input', + multi: true, + required: false, + placeholder: 'All active Google Workspace users', + description: + 'Optional primary email addresses, separated by commas (up to 100). Leave blank to index all active users across this Google Workspace customer.', }, { id: GOOGLE_DRIVE_OPEN_SHARING_FIELD_ID, diff --git a/apps/sim/connectors/google-drive/shortcuts.test.ts b/apps/sim/connectors/google-drive/shortcuts.test.ts index b1c110184b6..16cebd03556 100644 --- a/apps/sim/connectors/google-drive/shortcuts.test.ts +++ b/apps/sim/connectors/google-drive/shortcuts.test.ts @@ -4,6 +4,34 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { fetchMock } = vi.hoisted(() => ({ fetchMock: vi.fn() })) vi.mock('@/components/icons', () => ({ GoogleDriveIcon: () => null })) +vi.mock('@/connectors/google-drive/workspace-drives', () => ({ + GOOGLE_WORKSPACE_DRIVES_PAGE_SIZE: 100, + listGoogleWorkspaceDrives: async () => ({ driveIds: [] }), +})) + +/** The file/ACL tests isolate Directory enumeration; company-crawl tests exercise its real HTTP boundary. */ +vi.mock('@/connectors/google-workspace/users', () => ({ + GOOGLE_WORKSPACE_USERS_PAGE_SIZE: 100, + selectedGoogleWorkspaceUsers: () => [], + listGoogleWorkspaceUsers: async () => ({ + users: [{ id: 'admin', email: 'admin@corp.com', customerId: 'customer', active: true }], + }), + getGoogleWorkspaceUser: async () => ({ + id: 'admin', + email: 'admin@corp.com', + customerId: 'customer', + active: true, + }), +})) + +function companyContext(): Record { + return { + mirrorsSourceAcls: true, + getDelegatedAccessToken: async () => 'token', + googleDrivePageAccess: { token: 'token', externalIds: new Set(['shortcut']) }, + } +} + import { googleDriveConnector as drive } from '@/connectors/google-drive/google-drive' import { GoogleDriveApiError } from '@/connectors/google-drive/google-drive-errors' import { CONNECTOR_MAX_FILE_BYTES } from '@/connectors/utils' @@ -206,7 +234,7 @@ describe('Drive file shortcuts', () => { 'token', { adminEmail: 'admin@fixture.test' }, undefined, - { mirrorsSourceAcls: true } + companyContext() ) expect(page.documents[0].acl).toEqual({ acl: ['u:alice@fixture.test'], @@ -229,7 +257,7 @@ describe('Drive file shortcuts', () => { 'token', { adminEmail: 'admin@fixture.test' }, page.documents, - { mirrorsSourceAcls: true } + companyContext() ) expect(acls.shortcut).toEqual({ acl: ['u:alice@fixture.test'], @@ -237,13 +265,16 @@ describe('Drive file shortcuts', () => { }) fetchMock.mockResolvedValueOnce(json(alias)).mockResolvedValueOnce(denied()) expect( - await drive.getDocumentAcls!('token', { adminEmail: 'admin@fixture.test' }, page.documents, { - mirrorsSourceAcls: true, - }) + await drive.getDocumentAcls!( + 'token', + { adminEmail: 'admin@fixture.test' }, + page.documents, + companyContext() + ) ).toEqual({ shortcut: [] }) }) - it('fetches target permissions with its resource key and fails closed on permission lookup errors', async () => { + it('keeps denied target permissions unresolved without granting shortcut-only access', async () => { fetchMock .mockResolvedValueOnce( json({ @@ -261,15 +292,46 @@ describe('Drive file shortcuts', () => { 'token', { adminEmail: 'admin@fixture.test', openSharing: 'anyone' }, undefined, - { mirrorsSourceAcls: true } + companyContext() ) - expect(page.documents[0].acl).toEqual([]) + expect(page.documents[0].acl).toBeUndefined() expect(urlAt(2).pathname).toBe('/drive/v3/files/target/permissions') expect(new Headers(fetchMock.mock.calls[2][1].headers).get('X-Goog-Drive-Resource-Keys')).toBe( 'target/key' ) }) + it('rechecks both permissions when target metadata failed before a target ID was recorded', async () => { + const alias = shortcut({ + permissions: [{ type: 'anyone', role: 'reader', allowFileDiscovery: true }], + }) + const context = companyContext() + const config = { adminEmail: 'admin@fixture.test', openSharing: 'anyone' } + fetchMock + .mockResolvedValueOnce(json({ files: [alias] })) + .mockResolvedValueOnce(denied('invalid', 400)) + const page = await drive.listDocuments('token', config, undefined, context) + expect(page.documents[0].acl).toBeUndefined() + expect(page.documents[0].metadata).toMatchObject({ originalMimeType: shortcutMime }) + expect(page.documents[0].metadata?.shortcutTargetId).toBeUndefined() + + fetchMock.mockResolvedValueOnce(json(alias)).mockResolvedValueOnce(denied('invalid', 400)) + expect(await drive.getDocumentAcls!('token', config, page.documents, context)).toEqual({}) + expect(urlAt(2).pathname).toBe('/drive/v3/files/shortcut') + expect(urlAt(3).pathname).toBe('/drive/v3/files/target') + + fetchMock.mockResolvedValueOnce(json(alias)).mockResolvedValueOnce( + json( + file({ + permissions: [{ type: 'user', emailAddress: 'alice@fixture.test', role: 'reader' }], + }) + ) + ) + expect(await drive.getDocumentAcls!('token', config, page.documents, context)).toEqual({ + shortcut: { acl: ['pub'], requirements: [['u:alice@fixture.test']] }, + }) + }) + it('checks target size at listing and caps target bytes while downloading', async () => { fetchMock .mockResolvedValueOnce(json({ files: [shortcut()] })) diff --git a/apps/sim/connectors/google-drive/workspace-drives.test.ts b/apps/sim/connectors/google-drive/workspace-drives.test.ts new file mode 100644 index 00000000000..79e1a8ebcf5 --- /dev/null +++ b/apps/sim/connectors/google-drive/workspace-drives.test.ts @@ -0,0 +1,143 @@ +/** + * @vitest-environment node + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { GoogleDriveApiError } from '@/connectors/google-drive/google-drive-errors' +import { listGoogleWorkspaceDrives } from '@/connectors/google-drive/workspace-drives' + +const mockFetch = vi.fn() +const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }) + +beforeEach(() => { + mockFetch.mockReset() + vi.stubGlobal('fetch', mockFetch) +}) + +afterEach(() => { + vi.unstubAllGlobals() + vi.useRealTimers() +}) + +describe('Google Workspace shared-drive enumeration', () => { + it('returns one bounded page using only the delegated user token', async () => { + const controller = new AbortController() + mockFetch.mockResolvedValueOnce( + json({ drives: [{ id: 'drive-a' }, { id: 'drive-b' }], nextPageToken: 'next-page' }) + ) + await expect( + listGoogleWorkspaceDrives('delegated-user-token', undefined, controller.signal) + ).resolves.toEqual({ driveIds: ['drive-a', 'drive-b'], nextPageToken: 'next-page' }) + expect(mockFetch).toHaveBeenCalledTimes(1) + const [address, init] = mockFetch.mock.calls[0] + const url = new URL(address) + expect(url.origin + url.pathname).toBe('https://www.googleapis.com/drive/v3/drives') + expect(url.searchParams.get('pageSize')).toBe('100') + expect(url.searchParams.get('fields')).toBe('kind,nextPageToken,drives(id)') + expect(url.searchParams.has('useDomainAdminAccess')).toBe(false) + expect(url.searchParams.has('pageToken')).toBe(false) + expect(init.headers.Authorization).toBe('Bearer delegated-user-token') + expect(init.signal).toBe(controller.signal) + }) + + it('follows only the supplied page token and preserves empty intermediate pages', async () => { + mockFetch.mockResolvedValueOnce(json({ drives: [], nextPageToken: 'page-3' })) + await expect(listGoogleWorkspaceDrives('token', 'page-2')).resolves.toEqual({ + driveIds: [], + nextPageToken: 'page-3', + }) + expect(mockFetch).toHaveBeenCalledTimes(1) + expect(new URL(mockFetch.mock.calls[0][0]).searchParams.get('pageToken')).toBe('page-2') + }) + + it.each([{ drives: [] }, { kind: 'drive#driveList' }])( + 'recognizes an authoritative empty final page: %j', + async (body) => { + mockFetch.mockResolvedValueOnce(json(body)) + await expect(listGoogleWorkspaceDrives('token')).resolves.toEqual({ driveIds: [] }) + } + ) + + it('deduplicates repeated drive IDs within one page', async () => { + mockFetch.mockResolvedValueOnce(json({ drives: [{ id: 'drive-a' }, { id: 'drive-a' }] })) + await expect(listGoogleWorkspaceDrives('token')).resolves.toEqual({ driveIds: ['drive-a'] }) + }) + + it.each([ + {}, + { kind: 'drive#fileList' }, + { drives: null }, + { drives: [{}] }, + { drives: [{ id: 1 }] }, + { drives: [{ id: '' }] }, + { drives: [{ id: 'd'.repeat(257) }] }, + { drives: [], nextPageToken: '' }, + { drives: [], nextPageToken: 't'.repeat(8193) }, + { drives: Array.from({ length: 101 }, (_, index) => ({ id: `drive-${index}` })) }, + ])('rejects malformed or oversized pages', async (body) => { + mockFetch.mockResolvedValueOnce(json(body)) + await expect(listGoogleWorkspaceDrives('token')).rejects.toThrow('malformed') + }) + + it('rejects malformed JSON without exposing the provider body', async () => { + mockFetch.mockResolvedValueOnce(new Response('not-json-private-provider-details')) + await expect(listGoogleWorkspaceDrives('token')).rejects.toThrow( + 'Google Drive returned malformed shared-drive metadata' + ) + }) + + it('caps the response body before parsing it', async () => { + mockFetch.mockResolvedValueOnce(new Response('x'.repeat(1024 * 1024 + 1))) + await expect(listGoogleWorkspaceDrives('token')).rejects.toThrow('size limit') + }) + + it('rejects a repeated provider cursor', async () => { + mockFetch.mockResolvedValueOnce(json({ drives: [], nextPageToken: 'same-page' })) + await expect(listGoogleWorkspaceDrives('token', 'same-page')).rejects.toThrow('repeated') + }) + + it.each(['', 't'.repeat(8193)])( + 'rejects invalid input cursors before provider access', + async (cursor) => { + await expect(listGoogleWorkspaceDrives('token', cursor)).rejects.toThrow('invalid') + expect(mockFetch).not.toHaveBeenCalled() + } + ) + + it.each([401, 403, 404])( + 'preserves provider HTTP %i rather than reporting an empty drive list', + async (status) => { + mockFetch.mockResolvedValueOnce( + json({ error: { errors: [{ reason: 'forbidden' }] } }, status) + ) + await expect(listGoogleWorkspaceDrives('token')).rejects.toMatchObject< + Partial + >({ + status, + }) + expect(mockFetch).toHaveBeenCalledTimes(1) + } + ) + + it('uses the shared provider retry for transient failures', async () => { + vi.useFakeTimers() + mockFetch + .mockResolvedValueOnce(json({ error: { errors: [{ reason: 'backendError' }] } }, 503)) + .mockResolvedValueOnce(json({ drives: [{ id: 'drive-a' }] })) + const pending = listGoogleWorkspaceDrives('token') + const assertion = expect(pending).resolves.toEqual({ driveIds: ['drive-a'] }) + await vi.runAllTimersAsync() + await assertion + expect(mockFetch).toHaveBeenCalledTimes(2) + }) + + it('stops before provider access when cancelled', async () => { + const controller = new AbortController() + controller.abort() + await expect(listGoogleWorkspaceDrives('token', undefined, controller.signal)).rejects.toThrow() + expect(mockFetch).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/connectors/google-drive/workspace-drives.ts b/apps/sim/connectors/google-drive/workspace-drives.ts new file mode 100644 index 00000000000..6722a552edf --- /dev/null +++ b/apps/sim/connectors/google-drive/workspace-drives.ts @@ -0,0 +1,64 @@ +import { z } from 'zod' +import { fetchGoogleDriveWithRetry } from '@/connectors/google-drive/google-drive-errors' +import { readBodyWithLimit } from '@/connectors/utils' + +export const GOOGLE_WORKSPACE_DRIVES_PAGE_SIZE = 100 +const SHARED_DRIVES_PAGE_MAX_BYTES = 1024 * 1024 +const pageTokenSchema = z.string().min(1).max(8192) +const drivePageSchema = z + .object({ + kind: z.literal('drive#driveList').optional(), + drives: z + .array(z.object({ id: z.string().min(1).max(256) })) + .max(GOOGLE_WORKSPACE_DRIVES_PAGE_SIZE) + .optional(), + nextPageToken: pageTokenSchema.optional(), + }) + .refine((page) => page.drives !== undefined || page.kind === 'drive#driveList') + +export interface GoogleWorkspaceDrivePage { + driveIds: string[] + nextPageToken?: string +} + +/** Lists one page of the delegated user's shared drives without domain-admin expansion. */ +export async function listGoogleWorkspaceDrives( + accessToken: string, + pageToken?: string, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + if (pageToken !== undefined && !pageTokenSchema.safeParse(pageToken).success) { + throw new Error('Google Drive shared-drive continuation token is invalid') + } + const params = new URLSearchParams({ + pageSize: String(GOOGLE_WORKSPACE_DRIVES_PAGE_SIZE), + fields: 'kind,nextPageToken,drives(id)', + }) + if (pageToken) params.set('pageToken', pageToken) + const response = await fetchGoogleDriveWithRetry( + `https://www.googleapis.com/drive/v3/drives?${params}`, + { + headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, + signal, + } + ) + const body = await readBodyWithLimit(response, SHARED_DRIVES_PAGE_MAX_BYTES) + if (!body) throw new Error('Google Drive shared-drive metadata exceeded its size limit') + let data: unknown + try { + data = JSON.parse(body.toString('utf8')) + } catch { + throw new Error('Google Drive returned malformed shared-drive metadata') + } + const parsed = drivePageSchema.safeParse(data) + if (!parsed.success) throw new Error('Google Drive returned malformed shared-drive metadata') + const page = parsed.data + if (page.nextPageToken && page.nextPageToken === pageToken) { + throw new Error('Google Drive repeated a shared-drive continuation token') + } + return { + driveIds: [...new Set((page.drives ?? []).map((drive) => drive.id))], + ...(page.nextPageToken ? { nextPageToken: page.nextPageToken } : {}), + } +} diff --git a/apps/sim/connectors/google-workspace/company-crawl.test.ts b/apps/sim/connectors/google-workspace/company-crawl.test.ts new file mode 100644 index 00000000000..1e34062abfc --- /dev/null +++ b/apps/sim/connectors/google-workspace/company-crawl.test.ts @@ -0,0 +1,464 @@ +/** @vitest-environment node */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + getGoogleWorkspaceDocument, + InvalidGoogleWorkspaceCursor, + listGoogleWorkspaceDocuments, + validateGoogleWorkspaceConfig, +} from '@/connectors/google-workspace/company-crawl' +import type { ConnectorConfig, ExternalDocument } from '@/connectors/types' +import { memberDocumentId } from '@/connectors/utils' + +const mockFetch = vi.fn() +const USER = ( + id: string, + primaryEmail = `${id}@corp.com`, + extra: Record = {} +) => ({ + id, + primaryEmail, + customerId: 'customer-1', + suspended: false, + ...extra, +}) +const json = (value: unknown, status = 200) => + new Response(JSON.stringify(value), { status, headers: { 'Content-Type': 'application/json' } }) +function directory(users = [USER('alice'), USER('bob')]) { + mockFetch.mockImplementation(async (address: string) => { + const url = new URL(address) + if (url.pathname.endsWith('/users')) return json({ users }) + const key = decodeURIComponent(url.pathname.split('/').at(-1)!) + const user = + key === 'admin@corp.com' + ? USER('admin') + : users.find((item) => item.id === key || item.primaryEmail === key) + return user ? json(user) : json({ error: { errors: [{ reason: 'notFound' }] } }, 404) + }) +} +function context() { + return { + mirrorsSourceAcls: true, + getDelegatedAccessToken: vi.fn(async (email: string) => `delegated:${email}`), + } +} +const CONFIG = { adminEmail: 'admin@corp.com' } +function document(ctx?: Record, id = 'shared-provider-id'): ExternalDocument { + return { + externalId: memberDocumentId(id, ctx), + title: 'Title', + content: '', + contentHash: 'revision-1', + contentDeferred: true, + mimeType: 'text/plain', + acl: ['pub'], + } +} +const listUserDocuments = vi.fn() +function list(syncContext: Record, cursor?: string, sourceConfig = CONFIG) { + return listGoogleWorkspaceDocuments({ + provider: 'gmail', + accessToken: 'directory-token', + sourceConfig, + syncContext, + cursor, + listUserDocuments, + }) +} + +beforeEach(() => { + mockFetch.mockReset() + vi.stubGlobal('fetch', mockFetch) + directory() + listUserDocuments.mockReset().mockImplementation(async (_token, _config, _cursor, ctx) => ({ + documents: [document(ctx)], + hasMore: false, + })) +}) +afterEach(() => { + vi.unstubAllGlobals() + vi.useRealTimers() +}) + +describe('Google Workspace per-user central crawl', () => { + it('isolates same-ID content and caches for each user, with only that user in its ACL', async () => { + const ctx = context() + listUserDocuments.mockImplementation(async (_token, _config, _cursor, child) => { + expect(child?.cachedLabels).toBeUndefined() + expect(child?.getDelegatedAccessToken).toBeUndefined() + expect(child?.mirrorsSourceAcls).toBeUndefined() + if (child) child.cachedLabels = ['Private mailbox cache'] + return { documents: [document(child)], hasMore: false } + }) + const alice = await list(ctx) + const bob = await list(ctx, alice.nextCursor) + expect(alice.documents[0].externalId).not.toBe(bob.documents[0].externalId) + expect(alice.documents[0].acl).toEqual(['u:alice@corp.com']) + expect(bob.documents[0].acl).toEqual(['u:bob@corp.com']) + expect(bob.hasMore).toBe(false) + expect(listUserDocuments.mock.calls.map(([token]) => token)).toEqual([ + 'delegated:alice@corp.com', + 'delegated:bob@corp.com', + ]) + expect(alice.currentCursor).not.toContain('delegated') + }) + + it('replays the pinned provider page in a fresh process before advancing its nested cursor', async () => { + listUserDocuments.mockImplementation(async (_token, _config, cursor, child) => ({ + documents: [document(child)], + currentCursor: cursor ?? 'fixed-window:first', + nextCursor: 'fixed-window:second', + hasMore: true, + })) + const first = await list(context()) + mockFetch.mockClear() + const replay = await list(context(), first.currentCursor) + expect(mockFetch).toHaveBeenCalledTimes(1) + expect(mockFetch.mock.calls[0][0]).toContain('/users/alice?') + expect(listUserDocuments.mock.calls[1][2]).toBe('fixed-window:first') + expect(replay.documents).toEqual(first.documents) + expect(replay.currentCursor).toBe(first.currentCursor) + expect(replay.nextCursor).toBe(first.nextCursor) + }) + + it('preserves only the active user cache across provider pages', async () => { + listUserDocuments + .mockImplementationOnce(async (_token, _config, _cursor, child) => { + if (child) child.labels = ['Alice label'] + return { documents: [document(child)], nextCursor: 'page-2', hasMore: true } + }) + .mockImplementationOnce(async (_token, _config, cursor, child) => { + expect(cursor).toBe('page-2') + expect(child?.labels).toEqual(['Alice label']) + return { documents: [document(child, 'second')], hasMore: false } + }) + const ctx = context() + const first = await list(ctx) + await list(ctx, first.nextCursor) + expect(ctx.getDelegatedAccessToken).toHaveBeenCalledTimes(2) + }) + + it('hydrates only the active page using its token and preserves the provider revision', async () => { + const ctx = context() + const first = await list(ctx) + const externalId = first.documents[0].externalId + const hydrate = vi + .fn() + .mockImplementation(async (_token, _config, id, child) => ({ + ...document(child), + externalId: id, + content: 'Private body', + contentHash: 'revision-2', + contentDeferred: false, + })) + const input = { + provider: 'gmail' as const, + sourceConfig: CONFIG, + externalId, + syncContext: ctx, + getUserDocument: hydrate, + } + await expect(getGoogleWorkspaceDocument(input)).resolves.toMatchObject({ + content: 'Private body', + contentHash: 'revision-2', + acl: ['u:alice@corp.com'], + }) + expect(hydrate.mock.calls[0][0]).toBe('delegated:alice@corp.com') + await expect(getGoogleWorkspaceDocument({ ...input, externalId: 'arbitrary' })).rejects.toThrow( + 'verified delegated listing identity' + ) + await expect( + getGoogleWorkspaceDocument({ ...input, provider: 'google_calendar' }) + ).rejects.toThrow('verified delegated listing identity') + await expect(getGoogleWorkspaceDocument({ ...input, syncContext: context() })).rejects.toThrow( + 'verified delegated listing identity' + ) + await list(ctx, first.nextCursor) + await expect(getGoogleWorkspaceDocument(input)).rejects.toThrow( + 'verified delegated listing identity' + ) + }) + + it('clears old hydration authority before a failed subsequent list', async () => { + const ctx = context() + const first = await list(ctx) + listUserDocuments.mockRejectedValueOnce(new Error('Provider quota exhausted')) + await expect(list(ctx, first.nextCursor)).rejects.toThrow('quota') + await expect( + getGoogleWorkspaceDocument({ + provider: 'gmail', + sourceConfig: CONFIG, + externalId: first.documents[0].externalId, + syncContext: ctx, + getUserDocument: vi.fn(), + }) + ).rejects.toThrow('verified delegated listing identity') + }) + + it.each([ + ['suspended', USER('alice', 'alice@corp.com', { suspended: true })], + ['archived', USER('alice', 'alice@corp.com', { archived: true })], + ['guest', USER('alice', 'alice@corp.com', { isGuestUser: true })], + ['deleted', null], + ])('advances past a user now %s without delegation', async (_status, replacement) => { + const first = await list(context()) + const ctx = context() + mockFetch.mockResolvedValueOnce(replacement ? json(replacement) : json({ error: {} }, 404)) + const skipped = await list(ctx, first.currentCursor) + expect(skipped.documents).toEqual([]) + expect(skipped.hasMore).toBe(true) + expect(ctx.getDelegatedAccessToken).not.toHaveBeenCalled() + const next = await list(ctx, skipped.nextCursor) + expect(next.documents[0].acl).toEqual(['u:bob@corp.com']) + }) + + it.each([ + USER('different-id'), + USER('alice', 'alice@corp.com', { customerId: 'other-customer' }), + ])('fails closed on changed immutable identity or customer', async (changed) => { + const first = await list(context()) + mockFetch.mockResolvedValueOnce(json(changed)) + const ctx = context() + await expect(list(ctx, first.currentCursor)).rejects.toThrow('expected customer') + expect(ctx.getDelegatedAccessToken).not.toHaveBeenCalled() + }) + + it('uses the current primary email after a rename without changing its stable document ID', async () => { + const first = await list(context()) + mockFetch.mockResolvedValueOnce(json(USER('alice', 'renamed@corp.com'))) + const ctx = context() + const renamed = await list(ctx, first.currentCursor) + expect(renamed.documents[0].externalId).toBe(first.documents[0].externalId) + expect(renamed.documents[0].acl).toEqual(['u:renamed@corp.com']) + expect(ctx.getDelegatedAccessToken).toHaveBeenCalledWith('renamed@corp.com') + }) + + it('applies primary-user filters both to discovery and to resumed identities', async () => { + const config = { ...CONFIG, userEmails: ['ALICE@corp.com'] } + const first = await list(context(), undefined, config) + expect(first.hasMore).toBe(false) + mockFetch.mockResolvedValueOnce(json(USER('alice', 'renamed@corp.com'))) + const ctx = context() + const next = await list(ctx, first.currentCursor, config) + expect(next.documents).toEqual([]) + expect(next.hasMore).toBe(false) + expect(ctx.getDelegatedAccessToken).not.toHaveBeenCalled() + }) + + it('preserves an empty intermediate Directory page and visits secondary domains', async () => { + mockFetch.mockResolvedValueOnce(json({ users: [], nextPageToken: 'users-2' })) + const ctx = context() + const empty = await list(ctx) + expect(empty.hasMore).toBe(true) + mockFetch + .mockResolvedValueOnce(json({ users: [USER('secondary', 'person@secondary.com')] })) + .mockResolvedValueOnce(json(USER('secondary', 'person@secondary.com'))) + const next = await list(ctx, empty.nextCursor) + expect(next.documents[0].acl).toEqual(['u:person@secondary.com']) + expect(new URL(mockFetch.mock.calls[1][0]).searchParams.get('pageToken')).toBe('users-2') + }) + + it('rejects missing, looping or oversized provider cursors before publishing page authority', async () => { + for (const page of [ + { hasMore: true }, + { hasMore: true, currentCursor: 'same', nextCursor: 'same' }, + { hasMore: true, nextCursor: 'x'.repeat(256 * 1024 + 1) }, + ]) { + listUserDocuments.mockImplementationOnce(async (_token, _config, _cursor, child) => ({ + documents: [document(child)], + ...page, + })) + await expect(list(context())).rejects.toThrow() + } + }) + + it('rejects oversized, malformed, and cross-provider durable cursors', async () => { + const first = await list(context()) + for (const cursor of ['invalid', 'google-workspace:v1:invalid', 'x'.repeat(384 * 1024 + 1)]) { + await expect(list(context(), cursor)).rejects.toBeInstanceOf(InvalidGoogleWorkspaceCursor) + } + await expect( + listGoogleWorkspaceDocuments({ + provider: 'google_calendar', + accessToken: 'directory-token', + sourceConfig: CONFIG, + syncContext: context(), + cursor: first.currentCursor, + listUserDocuments, + }) + ).rejects.toBeInstanceOf(InvalidGoogleWorkspaceCursor) + }) + + it('refuses an unscoped provider document or a mismatched hydration result', async () => { + listUserDocuments.mockResolvedValueOnce({ documents: [document()], hasMore: false }) + await expect(list(context())).rejects.toThrow('verified listing user') + const ctx = context() + const first = await list(ctx) + await expect( + getGoogleWorkspaceDocument({ + provider: 'gmail', + sourceConfig: CONFIG, + externalId: first.documents[0].externalId, + syncContext: ctx, + getUserDocument: vi.fn().mockResolvedValue(document()), + }) + ).rejects.toThrow('different document') + }) + + it('never falls back to an administrator token or accepts source-config delegation', async () => { + await expect(list({})).rejects.toThrow('delegated Google Workspace') + await expect(list({ getDelegatedAccessToken: vi.fn() })).rejects.toThrow( + 'delegated Google Workspace' + ) + expect(mockFetch).not.toHaveBeenCalled() + }) + + it('forwards cancellation and stops before delegation after Directory cancellation', async () => { + const controller = new AbortController() + const ctx = { ...context(), signal: controller.signal } + controller.abort() + await expect(list(ctx)).rejects.toThrow() + expect(mockFetch).not.toHaveBeenCalled() + expect(ctx.getDelegatedAccessToken).not.toHaveBeenCalled() + }) + + it('stops when cancellation arrives after the Directory read or delegated token mint', async () => { + const first = await list(context()) + const controller = new AbortController() + const ctx = { ...context(), signal: controller.signal } + mockFetch.mockImplementationOnce(async () => { + controller.abort() + return json(USER('alice')) + }) + await expect(list(ctx, first.currentCursor)).rejects.toThrow() + expect(ctx.getDelegatedAccessToken).not.toHaveBeenCalled() + + const controller2 = new AbortController() + const ctx2 = { + mirrorsSourceAcls: true, + signal: controller2.signal, + getDelegatedAccessToken: vi.fn(async () => { + controller2.abort() + return 'delegated-token' + }), + } + const calls = listUserDocuments.mock.calls.length + await expect(list(ctx2, first.currentCursor)).rejects.toThrow() + expect(listUserDocuments).toHaveBeenCalledTimes(calls) + }) + + it.each([401, 403])( + 'propagates Directory authorization failure %s without declaring completion', + async (status) => { + mockFetch.mockResolvedValueOnce( + json({ error: { errors: [{ reason: 'forbidden' }] } }, status) + ) + const ctx = context() + await expect(list(ctx)).rejects.toThrow() + expect(ctx.getDelegatedAccessToken).not.toHaveBeenCalled() + expect(listUserDocuments).not.toHaveBeenCalled() + } + ) + + it('bounds the saved Directory page and retains only the remaining users', async () => { + const users = Array.from({ length: 100 }, (_, i) => USER(`employee-${i}`)) + directory(users) + const first = await list(context()) + const decoded = JSON.parse( + Buffer.from(first.nextCursor!.split(':').at(-1)!, 'base64url').toString() + ) + expect(decoded.users).toHaveLength(99) + expect(decoded.users[0].id).toBe('employee-1') + expect(JSON.stringify(decoded)).not.toContain('delegated') + expect(JSON.stringify(decoded)).not.toContain('directory-token') + const url = new URL(mockFetch.mock.calls[0][0]) + expect(url.searchParams.get('maxResults')).toBe('100') + expect(url.searchParams.get('fields')).toBe( + 'kind,nextPageToken,users(id,primaryEmail,customerId,suspended,archived,isGuestUser)' + ) + }) + + it('preserves null for a document deleted before hydration', async () => { + const ctx = context() + const first = await list(ctx) + const input = { + provider: 'gmail' as const, + sourceConfig: CONFIG, + externalId: first.documents[0].externalId, + syncContext: ctx, + getUserDocument: vi.fn().mockResolvedValue(null), + } + await expect(getGoogleWorkspaceDocument(input)).resolves.toBeNull() + }) + + it('requests restart for an expired Directory continuation without hiding initial request errors', async () => { + mockFetch.mockResolvedValueOnce(json({ users: [], nextPageToken: 'expired-users' })) + const first = await list(context()) + mockFetch.mockResolvedValueOnce(json({ error: { errors: [{ reason: 'invalid' }] } }, 400)) + await expect(list(context(), first.nextCursor)).rejects.toBeInstanceOf( + InvalidGoogleWorkspaceCursor + ) + mockFetch.mockResolvedValueOnce(json({ error: { errors: [{ reason: 'invalid' }] } }, 400)) + await expect(list(context())).rejects.not.toBeInstanceOf(InvalidGoogleWorkspaceCursor) + }) + + it('propagates provider truncation so partial results cannot reconcile deletions', async () => { + listUserDocuments.mockImplementation(async (_token, _config, _cursor, child) => { + if (child) child.listingCapped = true + return { documents: [], hasMore: false } + }) + const ctx: Record = context() + await list(ctx) + expect(ctx.listingCapped).toBe(true) + }) +}) + +describe('Google Workspace central validation', () => { + it('validates selected primary users and rechecks the sample immutable ID before delegation', async () => { + const ctx = context() + const validated = await validateGoogleWorkspaceConfig({ + provider: 'gmail', + accessToken: 'directory-token', + sourceConfig: { ...CONFIG, userEmails: 'bob@corp.com' }, + syncContext: ctx, + }) + expect(validated.user.id).toBe('bob') + expect(validated.accessToken).toBe('delegated:bob@corp.com') + expect(validated.syncContext.memberId).toBe('google-workspace:customer-1:bob') + expect(mockFetch.mock.calls.map(([url]) => new URL(url).pathname.split('/').at(-1))).toEqual([ + 'admin%40corp.com', + 'bob%40corp.com', + 'bob', + ]) + }) + + it('selects a bounded active sample when Users is blank', async () => { + const validated = await validateGoogleWorkspaceConfig({ + provider: 'google_calendar', + accessToken: 'directory-token', + sourceConfig: CONFIG, + syncContext: context(), + }) + expect(validated.user.email).toBe('alice@corp.com') + const url = new URL(mockFetch.mock.calls[1][0]) + expect(url.searchParams.get('maxResults')).toBe('1') + expect(url.searchParams.get('customer')).toBe('my_customer') + }) + + it.each([ + USER('bob', 'primary@corp.com'), + USER('bob', 'bob@corp.com', { customerId: 'other' }), + USER('bob', 'bob@corp.com', { suspended: true }), + ])('rejects aliases, other customers and inactive selected users', async (user) => { + mockFetch.mockResolvedValueOnce(json(USER('admin'))).mockResolvedValueOnce(json(user)) + const ctx = context() + await expect( + validateGoogleWorkspaceConfig({ + provider: 'gmail', + accessToken: 'directory-token', + sourceConfig: { ...CONFIG, userEmails: 'bob@corp.com' }, + syncContext: ctx, + }) + ).rejects.toThrow() + expect(ctx.getDelegatedAccessToken).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/connectors/google-workspace/company-crawl.ts b/apps/sim/connectors/google-workspace/company-crawl.ts new file mode 100644 index 00000000000..cb7a3c7b0b1 --- /dev/null +++ b/apps/sim/connectors/google-workspace/company-crawl.ts @@ -0,0 +1,311 @@ +import { normalizeEmail } from '@sim/utils/string' +import { z } from 'zod' +import { mapWithConcurrency } from '@/lib/core/utils/concurrency' +import { + GOOGLE_WORKSPACE_USERS_PAGE_SIZE, + type GoogleWorkspaceUser, + getGoogleWorkspaceUser, + listGoogleWorkspaceUsers, + selectedGoogleWorkspaceUsers, +} from '@/connectors/google-workspace/users' +import { ConnectorSourceError } from '@/connectors/source-error' +import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import { PER_MEMBER_LISTING_CONTEXT, sourceDocumentId } from '@/connectors/utils' + +type GoogleWorkspaceProvider = 'gmail' | 'google_calendar' +const CURSOR_PREFIX = 'google-workspace:v1:' +const MAX_CURSOR_BYTES = 384 * 1024 +const MAX_PROVIDER_CURSOR_BYTES = 256 * 1024 +const MAX_PAGE_DOCUMENTS = 2500 +const cursorSchema = z.object({ + provider: z.enum(['gmail', 'google_calendar']), + users: z + .array( + z.object({ + id: z.string().min(1).max(256), + email: z.string().email().max(254), + customerId: z.string().min(1).max(256), + }) + ) + .max(GOOGLE_WORKSPACE_USERS_PAGE_SIZE), + nextUsersPageToken: z.string().min(1).max(8192).optional(), + providerCursor: z.string().min(1).max(MAX_PROVIDER_CURSOR_BYTES).optional(), +}) +type CompanyCursor = z.infer + +interface GoogleWorkspaceCrawlInput { + provider: GoogleWorkspaceProvider + accessToken: string + sourceConfig: Record + syncContext: Record +} + +interface DelegatedUser { + user: GoogleWorkspaceUser + accessToken: string + syncContext: Record +} + +interface PageAccess extends DelegatedUser { + provider: GoogleWorkspaceProvider + externalIds: Set +} + +/** Only the active page carries hydration authority; entries disappear with their sync context. */ +const pageAccess = new WeakMap, PageAccess>() + +export class InvalidGoogleWorkspaceCursor extends Error {} + +function readCursor(cursor: string, provider: GoogleWorkspaceProvider): CompanyCursor { + try { + if (!cursor.startsWith(CURSOR_PREFIX) || cursor.length > MAX_CURSOR_BYTES) throw new Error() + const state = cursorSchema.parse( + JSON.parse(Buffer.from(cursor.slice(CURSOR_PREFIX.length), 'base64url').toString('utf8')) + ) + if (state.provider !== provider || (!state.users.length && state.providerCursor)) + throw new Error() + return state + } catch { + throw new InvalidGoogleWorkspaceCursor( + 'Google Workspace crawl cursor is invalid; restart the sync' + ) + } +} + +function writeCursor(state: CompanyCursor): string { + cursorSchema.parse(state) + const cursor = `${CURSOR_PREFIX}${Buffer.from(JSON.stringify(state)).toString('base64url')}` + if (cursor.length > MAX_CURSOR_BYTES) { + throw new Error('Google Workspace crawl exceeded its continuation-size limit') + } + return cursor +} + +function signalFrom(context: Record): AbortSignal | undefined { + return context.signal instanceof AbortSignal ? context.signal : undefined +} + +function tokenResolver(context: Record): (subject: string) => Promise { + if (context.mirrorsSourceAcls !== true || typeof context.getDelegatedAccessToken !== 'function') { + throw new Error('Company-wide indexing requires a delegated Google Workspace service account') + } + return context.getDelegatedAccessToken as (subject: string) => Promise +} + +function userContext( + user: GoogleWorkspaceUser, + context: Record +): Record { + return { + ...PER_MEMBER_LISTING_CONTEXT, + memberId: `google-workspace:${user.customerId}:${user.id}`, + signal: signalFrom(context), + } +} + +async function delegate( + user: GoogleWorkspaceUser, + context: Record +): Promise { + signalFrom(context)?.throwIfAborted() + const token = await tokenResolver(context)(user.email) + signalFrom(context)?.throwIfAborted() + if (typeof token !== 'string' || !token) + throw new Error('Google Workspace delegation returned no access token') + return token +} + +function assertIdentity(user: GoogleWorkspaceUser, expected: CompanyCursor['users'][number]): void { + if (user.id !== expected.id || user.customerId !== expected.customerId) { + throw new Error('Google Workspace user no longer belongs to the expected customer') + } +} + +function ownerDocument(document: ExternalDocument, access: DelegatedUser): ExternalDocument { + if (!sourceDocumentId(document.externalId, access.syncContext)) { + throw new Error('Google Workspace document does not belong to its verified listing user') + } + return { ...document, acl: [`u:${access.user.email}`] } +} + +/** Validates directory access and returns one revalidated identity for a provider-specific probe. */ +export async function validateGoogleWorkspaceConfig( + input: GoogleWorkspaceCrawlInput +): Promise { + const { accessToken, sourceConfig, syncContext } = input + tokenResolver(syncContext) + const selected = selectedGoogleWorkspaceUsers(sourceConfig.userEmails) + const adminEmail = sourceConfig.adminEmail + if (typeof adminEmail !== 'string' || !adminEmail.trim()) + throw new Error('Enter a Directory administrator email') + const signal = signalFrom(syncContext) + const administrator = await getGoogleWorkspaceUser(accessToken, normalizeEmail(adminEmail), { + validate: true, + signal, + }) + if (!administrator?.active) + throw new Error('The Directory administrator must be an active Google Workspace user') + const selectedUsers = await mapWithConcurrency(selected, 4, async (email) => { + const user = await getGoogleWorkspaceUser(accessToken, email, { validate: true, signal }) + if (!user || user.customerId !== administrator.customerId || user.email !== email) { + throw new Error(`User "${email}" is not a primary email in this Google Workspace customer`) + } + if (!user.active) + throw new Error(`User "${email}" must be an active, non-guest Google Workspace user`) + return user + }) + const sample = + selectedUsers[0] ?? + (await listGoogleWorkspaceUsers(accessToken, undefined, signal, { validate: true })).users.find( + (user) => user.active + ) ?? + administrator + const user = await getGoogleWorkspaceUser(accessToken, sample.id, { validate: true, signal }) + if (!user?.active) throw new Error('The selected Google Workspace user is no longer active') + assertIdentity(user, sample) + if ( + user.customerId !== administrator.customerId || + (selected.length && !selected.includes(user.email)) + ) { + throw new Error('The selected user is outside this Google Workspace configuration') + } + return { + user, + accessToken: await delegate(user, syncContext), + syncContext: userContext(user, syncContext), + } +} + +/** Visits one user's provider page per call, with a bounded, replayable directory checkpoint. */ +export async function listGoogleWorkspaceDocuments( + input: GoogleWorkspaceCrawlInput & { + cursor?: string + listUserDocuments: ConnectorConfig['listDocuments'] + } +): Promise { + const { provider, accessToken, sourceConfig, syncContext, listUserDocuments } = input + const previousAccess = pageAccess.get(syncContext) + pageAccess.delete(syncContext) + tokenResolver(syncContext) + const selected = selectedGoogleWorkspaceUsers(sourceConfig.userEmails) + const signal = signalFrom(syncContext) + signal?.throwIfAborted() + let state: CompanyCursor = input.cursor + ? readCursor(input.cursor, provider) + : { provider, users: [] } + if (!state.users.length) { + const page = await listGoogleWorkspaceUsers( + accessToken, + state.nextUsersPageToken, + signal + ).catch((error: unknown) => { + /** Directory continuation tokens expire; retry a rejected fixed-query continuation from the start. */ + if ( + state.nextUsersPageToken && + error instanceof ConnectorSourceError && + error.status === 400 + ) { + throw new InvalidGoogleWorkspaceCursor( + 'Google Workspace rejected its user continuation token; restart the sync' + ) + } + throw error + }) + state = { + provider, + users: page.users + .filter((user) => user.active && (!selected.length || selected.includes(user.email))) + .map(({ id, email, customerId }) => ({ id, email, customerId })), + nextUsersPageToken: page.nextPageToken, + } + } + const currentCursor = writeCursor(state) + const pending = state.users[0] + const advance = (): CompanyCursor => ({ + provider, + users: state.users.slice(1), + nextUsersPageToken: state.nextUsersPageToken, + }) + const emptyPage = (next: CompanyCursor): ExternalDocumentList => { + const hasMore = Boolean(next.users.length || next.nextUsersPageToken) + return { + documents: [], + currentCursor, + hasMore, + nextCursor: hasMore ? writeCursor(next) : undefined, + } + } + if (!pending) return emptyPage(state) + const user = await getGoogleWorkspaceUser(accessToken, pending.id, { signal }) + if (user) assertIdentity(user, pending) + if (!user?.active || (selected.length && !selected.includes(user.email))) + return emptyPage(advance()) + const access: PageAccess = { + provider, + user, + accessToken: await delegate(user, syncContext), + syncContext: + previousAccess?.provider === provider && + previousAccess.user.id === user.id && + previousAccess.user.customerId === user.customerId && + previousAccess.user.email === user.email + ? previousAccess.syncContext + : userContext(user, syncContext), + externalIds: new Set(), + } + access.syncContext.signal = signal + const page = await listUserDocuments( + access.accessToken, + sourceConfig, + state.providerCursor, + access.syncContext + ) + signal?.throwIfAborted() + if (access.syncContext.listingCapped === true) syncContext.listingCapped = true + if (page.documents.length > MAX_PAGE_DOCUMENTS) + throw new Error('Google Workspace provider returned an oversized document page') + const documents = page.documents.map((document) => ownerDocument(document, access)) + const replay = { ...state, providerCursor: page.currentCursor ?? state.providerCursor } + if (page.hasMore && (!page.nextCursor || page.nextCursor === replay.providerCursor)) { + throw new Error('Google Workspace provider omitted or repeated its continuation token') + } + const next = page.hasMore ? { ...state, providerCursor: page.nextCursor } : advance() + const hasMore = Boolean(next.users.length || next.nextUsersPageToken) + const result = { + ...page, + documents, + currentCursor: writeCursor(replay), + hasMore, + nextCursor: hasMore ? writeCursor(next) : undefined, + } + access.externalIds = new Set(documents.map((document) => document.externalId)) + pageAccess.set(syncContext, access) + return result +} + +/** Refuses cross-user or out-of-page hydration; no administrator-token fallback is possible. */ +export async function getGoogleWorkspaceDocument(input: { + provider: GoogleWorkspaceProvider + sourceConfig: Record + externalId: string + syncContext: Record + getUserDocument: ConnectorConfig['getDocument'] +}): Promise { + const { provider, sourceConfig, externalId, syncContext, getUserDocument } = input + const access = pageAccess.get(syncContext) + if (!access || access.provider !== provider || !access.externalIds.has(externalId)) { + throw new Error('Google Workspace document has no verified delegated listing identity') + } + signalFrom(syncContext)?.throwIfAborted() + const document = await getUserDocument( + access.accessToken, + sourceConfig, + externalId, + access.syncContext + ) + signalFrom(syncContext)?.throwIfAborted() + if (!document) return null + if (document.externalId !== externalId) + throw new Error('Google Workspace returned a different document during hydration') + return ownerDocument(document, access) +} diff --git a/apps/sim/connectors/google-workspace/users.ts b/apps/sim/connectors/google-workspace/users.ts new file mode 100644 index 00000000000..762b43b1b2c --- /dev/null +++ b/apps/sim/connectors/google-workspace/users.ts @@ -0,0 +1,164 @@ +import { isPlainRecord } from '@sim/utils/object' +import { normalizeEmail } from '@sim/utils/string' +import { z } from 'zod' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { + fetchGoogleDriveWithRetry, + GoogleDriveApiError, +} from '@/connectors/google-drive/google-drive-errors' +import { parseMultiValue, readBodyWithLimit } from '@/connectors/utils' + +const DIRECTORY_USERS_URL = 'https://admin.googleapis.com/admin/directory/v1/users' +export const GOOGLE_WORKSPACE_USERS_PAGE_SIZE = 100 +const DIRECTORY_PAGE_MAX_BYTES = 1024 * 1024 +const emailSchema = z.string().email().max(254) + +export interface GoogleWorkspaceUser { + id: string + email: string + customerId: string + active: boolean +} + +export interface GoogleWorkspaceUserPage { + users: GoogleWorkspaceUser[] + nextPageToken?: string +} + +/** Empty means every active user in the administrator's Workspace customer. */ +export function selectedGoogleWorkspaceUsers(value: unknown): string[] { + if ( + value !== undefined && + value !== null && + typeof value !== 'string' && + (!Array.isArray(value) || value.some((email) => typeof email !== 'string')) + ) { + throw new Error('Users must contain Google Workspace email addresses') + } + const emails = [...new Set(parseMultiValue(value).map(normalizeEmail))] + if (emails.length > GOOGLE_WORKSPACE_USERS_PAGE_SIZE) { + throw new Error('Select at most 100 Google Workspace user emails, or leave blank for everyone') + } + if (emails.some((email) => !emailSchema.safeParse(email).success)) { + throw new Error('Users must contain valid Google Workspace email addresses') + } + return emails +} + +function parseUser(value: unknown): GoogleWorkspaceUser { + if ( + !isPlainRecord(value) || + typeof value.id !== 'string' || + !value.id || + value.id.length > 256 || + typeof value.customerId !== 'string' || + !value.customerId || + value.customerId.length > 256 || + typeof value.primaryEmail !== 'string' || + !emailSchema.safeParse(value.primaryEmail).success || + typeof value.suspended !== 'boolean' || + (value.archived !== undefined && typeof value.archived !== 'boolean') || + (value.isGuestUser !== undefined && typeof value.isGuestUser !== 'boolean') + ) { + throw new Error('Google Workspace returned malformed user metadata') + } + return { + id: value.id, + email: normalizeEmail(value.primaryEmail), + customerId: value.customerId, + active: !value.suspended && value.archived !== true && value.isGuestUser !== true, + } +} + +async function readDirectoryJson(response: Response): Promise { + const body = await readBodyWithLimit(response, DIRECTORY_PAGE_MAX_BYTES) + if (!body) throw new Error('Google Workspace user metadata exceeded its size limit') + try { + return JSON.parse(body.toString('utf8')) + } catch { + throw new Error('Google Workspace returned malformed user metadata') + } +} + +const USER_FIELDS = 'id,primaryEmail,customerId,suspended,archived,isGuestUser' + +/** One provider page only; the connector checkpoint advances through the directory. */ +export async function listGoogleWorkspaceUsers( + accessToken: string, + pageToken?: string, + signal?: AbortSignal, + options: { validate?: boolean } = {} +): Promise { + signal?.throwIfAborted() + if (pageToken !== undefined && (!pageToken || pageToken.length > 8192)) { + throw new Error('Google Workspace user continuation token is invalid') + } + const params = new URLSearchParams({ + customer: 'my_customer', + query: 'isGuest=false', + maxResults: String(options.validate ? 1 : GOOGLE_WORKSPACE_USERS_PAGE_SIZE), + orderBy: 'email', + projection: 'basic', + viewType: 'admin_view', + fields: `kind,nextPageToken,users(${USER_FIELDS})`, + }) + if (options.validate) params.set('query', 'isSuspended=false isArchived=false isGuest=false') + if (pageToken) params.set('pageToken', pageToken) + const response = await fetchGoogleDriveWithRetry( + `${DIRECTORY_USERS_URL}?${params}`, + { + headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, + signal, + }, + options.validate ? VALIDATE_RETRY_OPTIONS : undefined + ) + const data = await readDirectoryJson(response) + if ( + !isPlainRecord(data) || + (data.users === undefined && data.kind !== 'admin#directory#users') || + (data.users !== undefined && !Array.isArray(data.users)) || + (data.nextPageToken !== undefined && + (typeof data.nextPageToken !== 'string' || + !data.nextPageToken || + data.nextPageToken.length > 8192)) + ) { + throw new Error('Google Workspace returned malformed user-list metadata') + } + const users = (data.users ?? []).map(parseUser) + if (users.length > GOOGLE_WORKSPACE_USERS_PAGE_SIZE) { + throw new Error('Google Workspace returned an oversized user page') + } + const nextPageToken = typeof data.nextPageToken === 'string' ? data.nextPageToken : undefined + if (nextPageToken && nextPageToken === pageToken) { + throw new Error('Google Workspace repeated a user continuation token') + } + return { users, nextPageToken } +} + +/** Rechecks the current user before delegation, including when resuming a saved page. */ +export async function getGoogleWorkspaceUser( + accessToken: string, + userKey: string, + options: { validate?: boolean; signal?: AbortSignal } = {} +): Promise { + options.signal?.throwIfAborted() + const params = new URLSearchParams({ + projection: 'basic', + viewType: 'admin_view', + fields: USER_FIELDS, + }) + try { + const response = await fetchGoogleDriveWithRetry( + `${DIRECTORY_USERS_URL}/${encodeURIComponent(userKey)}?${params}`, + { + headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, + signal: options.signal, + }, + options.validate ? VALIDATE_RETRY_OPTIONS : undefined + ) + return parseUser(await readDirectoryJson(response)) + } catch (error) { + if (error instanceof GoogleDriveApiError && error.kind === 'not_found') return null + throw error + } +} diff --git a/apps/sim/connectors/types.ts b/apps/sim/connectors/types.ts index d15113c8aa1..d6392a19b1f 100644 --- a/apps/sim/connectors/types.ts +++ b/apps/sim/connectors/types.ts @@ -30,6 +30,10 @@ export type ConnectorAuthConfig = * genuinely coincide, which is the common case. */ serviceAccountScopes?: string[] + /** Scope override for central crawls that read the source directory and mirror permissions. */ + adminServiceAccountScopes?: string[] + /** Fixed scopes for per-user tokens minted during a delegated service-account crawl. */ + serviceAccountDelegationScopes?: string[] /** * The config field naming the person a service-account credential acts * as, through domain-wide delegation. @@ -266,6 +270,8 @@ export interface SyncResult { export interface ConnectorConfigField { id: string title: string + /** Label when configuring a central crawl that mirrors source permissions. */ + titleInAdminMode?: string type: 'short-input' | 'dropdown' | 'selector' placeholder?: string required?: boolean @@ -274,8 +280,14 @@ export interface ConnectorConfigField { /** Secondary Search setup controls, shown in the shared More options disclosure. */ setupGroup?: 'options' description?: string + /** Setup guidance specific to a central crawl that mirrors source permissions. */ + descriptionInAdminMode?: string /** Excludes settings unused by member crawls and account-local selectors that need a manual sibling. */ hideInMemberMode?: true + /** Excludes account-local controls that cannot describe a company-wide source. */ + hideInAdminMode?: true + /** Only applies to a central crawl that mirrors source permissions. */ + showInAdminModeOnly?: true options?: { label: string; id: string }[] /** Selector key from the selector registry (used when type is 'selector') */ diff --git a/apps/sim/lib/knowledge/access/predicate.postgres.test.ts b/apps/sim/lib/knowledge/access/predicate.postgres.test.ts index 5137d8404ef..286d8199904 100644 --- a/apps/sim/lib/knowledge/access/predicate.postgres.test.ts +++ b/apps/sim/lib/knowledge/access/predicate.postgres.test.ts @@ -18,6 +18,9 @@ vi.mock('@/connectors/registry.server', () => ({ CONNECTOR_REGISTRY: {} })) const { drizzle } = await import('drizzle-orm/postgres-js') const schema = await import('@sim/db/schema') const { persistDocumentAcls } = await import('@/lib/knowledge/connectors/sync-persistence') +const { mergeMirroredAcls, hideUnlistedDocuments } = await import( + '@/lib/knowledge/connectors/mirrored-acls' +) const { PgDialect } = await import('drizzle-orm/pg-core') const { knowledgeAccessCondition } = await import('@/lib/knowledge/access/predicate') const { confluencePageAcl } = await import('@/lib/knowledge/access/confluence-permissions') @@ -353,6 +356,133 @@ describe.runIf(Boolean(databaseUrl))('knowledge ACLs in PostgreSQL', () => { expect(stored).toEqual({ shape: 'array', acl_requirements: [[space], [page]] }) }) + it.each([ + { + name: 'owner then reader', + sequence: ['owner', 'unknown'], + expected: ['u:alice@corp.com', 'u:bob@corp.com'], + }, + { + name: 'reader then owner', + sequence: ['unknown', 'owner'], + expected: ['u:alice@corp.com', 'u:bob@corp.com'], + }, + { + name: 'resumed duplicate reader', + sequence: ['owner', 'unknown', 'unknown'], + expected: ['u:alice@corp.com', 'u:bob@corp.com'], + }, + { + name: 'unknown in next generation', + sequence: ['owner', 'next-generation', 'unknown'], + expected: [], + }, + { + name: 'explicit revocation then unknown', + sequence: ['owner', 'empty', 'unknown'], + expected: [], + }, + { name: 'malformed ACL then unknown', sequence: ['owner', 'invalid', 'unknown'], expected: [] }, + { + name: 'changed valid ACL then unknown', + sequence: ['owner', 'restricted', 'unknown'], + expected: ['u:alice@corp.com'], + }, + { name: 'unlisted cleanup', sequence: ['owner', 'unlisted'], expected: [] }, + ])('preserves only verified current-generation ACLs: $name', async ({ sequence, expected }) => { + await connection.unsafe( + "INSERT INTO document(id, external_id, connector_id, acl) VALUES ('shared', 'shared-file', 'admin', '{}')" + ) + const executor = drizzle(connection, { schema }) + const [clock] = await connection.unsafe('SELECT statement_timestamp()::text AS start') + let generationStartedAt = new Date(clock.start) + let lastVerifiedAt: string | undefined + for (const step of sequence) { + if (step === 'next-generation') { + await connection.unsafe( + "UPDATE document SET acl_verified_at = statement_timestamp() - interval '2 minutes' WHERE id = 'shared'" + ) + const [nextClock] = await connection.unsafe('SELECT statement_timestamp()::text AS start') + generationStartedAt = new Date(nextClock.start) + continue + } + const acl = + step === 'owner' + ? ['u:alice@corp.com', 'u:bob@corp.com'] + : step === 'restricted' + ? ['u:alice@corp.com'] + : step === 'empty' + ? [] + : step === 'invalid' + ? ['invalid'] + : undefined + const merged = mergeMirroredAcls( + step === 'unlisted' + ? [] + : [ + { + externalId: 'shared-file', + title: 'Shared', + content: '', + mimeType: 'text/plain', + contentHash: 'same', + acl, + }, + ], + {} + ) + if (step === 'unlisted') hideUnlistedDocuments(merged.acls, ['shared-file']) + await persistDocumentAcls('admin', merged.acls, executor, { + unresolvedExternalIds: merged.unresolvedExternalIds, + generationStartedAt, + }) + const [stored] = await connection.unsafe( + "SELECT to_jsonb(acl) AS acl, acl_verified_at FROM document WHERE id = 'shared'" + ) + if (step === 'owner' || step === 'restricted') lastVerifiedAt = stored.acl_verified_at + if (step === 'unknown' && lastVerifiedAt && stored.acl.length > 0) { + expect(stored.acl_verified_at).toEqual(lastVerifiedAt) + } + } + expect(await readable(['u:alice@corp.com'], 'shared')).toBe( + expected.includes('u:alice@corp.com') + ) + expect(await readable(['u:bob@corp.com'], 'shared')).toBe(expected.includes('u:bob@corp.com')) + const [stored] = await connection.unsafe( + "SELECT to_jsonb(acl) AS acl, acl_verified_at FROM document WHERE id = 'shared'" + ) + expect(stored.acl).toEqual(expected) + if (expected.length === 0) expect(stored.acl_verified_at).toBeNull() + }) + + it.each([ + { name: 'unverified', offsetMs: null, preserved: false }, + { name: 'before generation', offsetMs: -1, preserved: false }, + { name: 'at generation boundary', offsetMs: 0, preserved: true }, + { name: 'within generation', offsetMs: 1, preserved: true }, + ])('guards unresolved SQL writes against $name evidence', async ({ offsetMs, preserved }) => { + const [clock] = await connection.unsafe('SELECT statement_timestamp()::text AS start') + const generationStartedAt = new Date(new Date(clock.start).getTime() - 60_000) + const verifiedAt = + offsetMs === null ? null : new Date(generationStartedAt.getTime() + offsetMs).toISOString() + await connection.unsafe( + "INSERT INTO document(id, external_id, connector_id, acl, acl_verified_at) VALUES ('boundary', 'file', 'admin', '{u:alice@corp.com}', $1::timestamptz AT TIME ZONE 'UTC')", + [verifiedAt] + ) + const executor = drizzle(connection, { schema }) + const result = await persistDocumentAcls('admin', new Map([['file', []]]), executor, { + unresolvedExternalIds: new Set(['file']), + generationStartedAt, + }) + expect(result.updated).toBe(preserved ? 0 : 1) + const [stored] = await connection.unsafe( + "SELECT to_jsonb(acl) AS acl, acl_verified_at FROM document WHERE id = 'boundary'" + ) + expect(stored.acl).toEqual(preserved ? ['u:alice@corp.com'] : []) + if (!preserved) expect(stored.acl_verified_at).toBeNull() + expect(await readable(['u:alice@corp.com'], 'boundary')).toBe(preserved) + }) + it('applies the same gate to direct document and joined chunk reads', async () => { await putDocument('joined', ['u:alice@corp.com'], [['g:confluence:tenant:space']]) await connection.unsafe("INSERT INTO embedding VALUES ('chunk', 'joined', 'protected content')") diff --git a/apps/sim/lib/knowledge/application/connectors.test.ts b/apps/sim/lib/knowledge/application/connectors.test.ts index 38d33c26036..c2183e73dda 100644 --- a/apps/sim/lib/knowledge/application/connectors.test.ts +++ b/apps/sim/lib/knowledge/application/connectors.test.ts @@ -95,6 +95,8 @@ vi.mock('@/lib/credentials/application/organization-credentials', () => ({ vi.mock('@/lib/oauth/credential-service', () => ({ resolveCredentialTokenBundle: mocks.resolveTokenBundle, + resolveOAuthAccountId: vi.fn(async () => null), + getServiceAccountToken: vi.fn(), })) vi.mock('@/lib/api-key/crypto', () => ({ decryptApiKey: mocks.decryptApiKey })) @@ -124,6 +126,12 @@ vi.mock('@/connectors/registry.server', () => ({ provider: 'google-drive', adminCredentialType: 'service_account', serviceAccountScopes: ['https://www.googleapis.com/auth/drive.readonly'], + adminServiceAccountScopes: [ + 'https://www.googleapis.com/auth/drive.readonly', + 'https://www.googleapis.com/auth/admin.directory.user.readonly', + 'https://www.googleapis.com/auth/admin.directory.group.readonly', + 'https://www.googleapis.com/auth/admin.directory.domain.readonly', + ], serviceAccountSubjectFieldId: 'adminEmail', }, validateConfig: mocks.validateConnectorConfig, @@ -331,32 +339,38 @@ describe('knowledge connector application use cases', () => { ) }) - it('mints delegated Drive tokens for an eligible canonical service account', async () => { - mocks.resolveTokenIdentity.mockResolvedValueOnce({ kind: 'service_account' }) - await expect( - resolveConnectorCredentialAccessToken({ - principal: { kind: 'session', userId: 'admin', sessionId: 'session' }, - credentialId: 'credential-1', - workspaceId: 'workspace-a', - actingUserId: 'admin', - requestId: 'request', - auth: googleDriveConnectorMeta.auth, - accessMode: 'admin', - sourceConfig: { adminEmail: 'Admin@corp.com' }, - }) - ).resolves.toEqual({ accessToken: 'access-token' }) - expect(mocks.resolveTokenBundle).toHaveBeenCalledWith( - 'credential-1', - 'admin', - 'request', - [ - 'https://www.googleapis.com/auth/drive.readonly', - 'https://www.googleapis.com/auth/admin.directory.group.readonly', - 'https://www.googleapis.com/auth/admin.directory.domain.readonly', - ], - 'admin@corp.com' - ) - }) + it.each(['workspace', 'members', 'admin'] as const)( + 'mints only the Drive scopes needed by an eligible service account in %s mode', + async (accessMode) => { + mocks.resolveTokenIdentity.mockResolvedValueOnce({ kind: 'service_account' }) + await expect( + resolveConnectorCredentialAccessToken({ + principal: { kind: 'session', userId: 'admin', sessionId: 'session' }, + credentialId: 'credential-1', + workspaceId: 'workspace-a', + actingUserId: 'admin', + requestId: 'request', + auth: googleDriveConnectorMeta.auth, + accessMode, + sourceConfig: { adminEmail: 'Admin@corp.com' }, + }) + ).resolves.toEqual({ accessToken: 'access-token' }) + expect(mocks.resolveTokenBundle).toHaveBeenCalledWith( + 'credential-1', + 'admin', + 'request', + accessMode === 'admin' + ? [ + 'https://www.googleapis.com/auth/drive.readonly', + 'https://www.googleapis.com/auth/admin.directory.user.readonly', + 'https://www.googleapis.com/auth/admin.directory.group.readonly', + 'https://www.googleapis.com/auth/admin.directory.domain.readonly', + ] + : ['https://www.googleapis.com/auth/drive.readonly'], + 'admin@corp.com' + ) + } + ) it('rejects an old central Drive OAuth source during settings validation before provider access', async () => { const connector = { @@ -608,6 +622,7 @@ describe('knowledge connector application use cases', () => { connectorType: string credentialId: string encryptedApiKey: null + accessMode: 'workspace' }, sourceConfig: Record ) => Promise @@ -625,6 +640,7 @@ describe('knowledge connector application use cases', () => { connectorType: 'confluence', credentialId: 'credential-1', encryptedApiKey: null, + accessMode: 'workspace', }, { space: 'ENG' } ) @@ -736,6 +752,7 @@ describe('knowledge connector application use cases', () => { connectorType: string credentialId: string encryptedApiKey: null + accessMode: 'workspace' }, sourceConfig: Record ) => Promise @@ -757,6 +774,7 @@ describe('knowledge connector application use cases', () => { connectorType: 'confluence', credentialId: 'credential-1', encryptedApiKey: null, + accessMode: 'workspace', }, { space: 'ENG' } ) @@ -1423,7 +1441,7 @@ describe('organization connector credential authorization', () => { principal.userId, 'request', googleDriveConnectorMeta.auth.mode === 'oauth' - ? googleDriveConnectorMeta.auth.serviceAccountScopes + ? googleDriveConnectorMeta.auth.adminServiceAccountScopes : undefined, 'admin@corp.com' ) @@ -1471,31 +1489,48 @@ describe('organization connector credential authorization', () => { ) }) - it('uses the same organization credential policy during config validation', async () => { - await expect( - validateConnectorSourceConfig({ - principal, - organizationId: 'org', - actingUserId: principal.userId, - requestId: 'request', - sourceConfig: input.sourceConfig, - connector: { - connectorType: 'google_drive', - credentialId: credential.id, - encryptedApiKey: null, - accessMode: 'admin', - } as Parameters[0]['connector'], - }) - ).resolves.toBeNull() - expect(mocks.authorizeOrganizationCredentialUse).toHaveBeenCalledWith( - expect.objectContaining({ principal, organizationId: 'org' }) - ) - expect(mocks.validateConnectorConfig).toHaveBeenCalledWith( - 'organization-token', - input.sourceConfig, - { mirrorsSourceAcls: true } - ) - }) + it.each(['workspace', 'members', 'admin'] as const)( + 'uses the saved %s mode and organization credential policy during config validation', + async (accessMode) => { + await expect( + validateConnectorSourceConfig({ + principal, + organizationId: 'org', + actingUserId: principal.userId, + requestId: 'request', + sourceConfig: input.sourceConfig, + connector: { + connectorType: 'google_drive', + credentialId: credential.id, + encryptedApiKey: null, + accessMode, + } as Parameters[0]['connector'], + }) + ).resolves.toBeNull() + expect(mocks.authorizeOrganizationCredentialUse).toHaveBeenCalledWith( + expect.objectContaining({ principal, organizationId: 'org' }) + ) + expect(mocks.validateConnectorConfig).toHaveBeenCalledWith( + 'organization-token', + input.sourceConfig, + expect.objectContaining({ mirrorsSourceAcls: accessMode === 'admin' }) + ) + expect(mocks.resolveTokenBundle).toHaveBeenCalledWith( + credential.id, + principal.userId, + 'request', + accessMode === 'admin' + ? [ + 'https://www.googleapis.com/auth/drive.readonly', + 'https://www.googleapis.com/auth/admin.directory.user.readonly', + 'https://www.googleapis.com/auth/admin.directory.group.readonly', + 'https://www.googleapis.com/auth/admin.directory.domain.readonly', + ] + : ['https://www.googleapis.com/auth/drive.readonly'], + 'admin@corp.com' + ) + } + ) it('propagates canonical organization credential refusals before token resolution', async () => { const rejection = new OrchestrationError('not_found', 'Credential not found') diff --git a/apps/sim/lib/knowledge/application/connectors.ts b/apps/sim/lib/knowledge/application/connectors.ts index c8fd8d39ca8..5eb12fbef2c 100644 --- a/apps/sim/lib/knowledge/application/connectors.ts +++ b/apps/sim/lib/knowledge/application/connectors.ts @@ -48,6 +48,7 @@ import { prepareGitHubInstallationSource } from '@/lib/knowledge/application/git import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { type ConnectorAccessMode, + isConnectorAccessMode, mirrorsSourceAcls, } from '@/lib/knowledge/connectors/access-modes' import { @@ -302,6 +303,7 @@ export async function resolveConnectorCredentialAccessToken(input: { if (!identity) return null const resolved = await resolveConnectorAccessToken({ auth: input.auth, + accessMode: input.accessMode, connector: { credentialId: input.credentialId, encryptedApiKey: null }, userId: identity.kind === 'oauth' ? identity.userId : input.actingUserId, requestId: input.requestId, @@ -319,6 +321,10 @@ export async function validateConnectorSourceConfig(input: { actingUserId: string requestId: string }): Promise { + const accessMode = input.connector.accessMode + if (!isConnectorAccessMode(accessMode)) { + return { message: 'Unsupported connector access mode', errorCode: 'validation' } + } const { CONNECTOR_REGISTRY } = await import('@/connectors/registry.server') const connectorConfig = CONNECTOR_REGISTRY[input.connector.connectorType] if (!connectorConfig) { @@ -394,6 +400,7 @@ export async function validateConnectorSourceConfig(input: { const resolved = await resolveConnectorAccessToken({ auth: connectorConfig.auth, + accessMode, connector: input.connector, userId: tokenUserId, requestId: input.requestId, diff --git a/apps/sim/lib/knowledge/connectors/access-token.test.ts b/apps/sim/lib/knowledge/connectors/access-token.test.ts index dc1e341a3c6..4fac4d3b56c 100644 --- a/apps/sim/lib/knowledge/connectors/access-token.test.ts +++ b/apps/sim/lib/knowledge/connectors/access-token.test.ts @@ -3,22 +3,35 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockDecryptApiKey, mockResolveTokenBundle } = vi.hoisted(() => ({ +const { + mockDecryptApiKey, + mockResolveTokenBundle, + mockResolveOAuthAccountId, + mockGetServiceAccountToken, +} = vi.hoisted(() => ({ mockDecryptApiKey: vi.fn(), mockResolveTokenBundle: vi.fn(), + mockResolveOAuthAccountId: vi.fn(), + mockGetServiceAccountToken: vi.fn(), })) vi.mock('@/lib/api-key/crypto', () => ({ decryptApiKey: mockDecryptApiKey })) vi.mock('@/lib/oauth/credential-service', () => ({ resolveCredentialTokenBundle: mockResolveTokenBundle, + resolveOAuthAccountId: mockResolveOAuthAccountId, + getServiceAccountToken: mockGetServiceAccountToken, })) +import type { ConnectorAccessMode } from '@/lib/knowledge/connectors/access-modes' import { connectorServiceAccountScopes, connectorServiceAccountSubject, resolveConnectorAccessToken, syncContextForToken, } from '@/lib/knowledge/connectors/access-token' +import { isConnectorCredentialTypeAllowed } from '@/connectors/auth' +import { gmailConnectorMeta } from '@/connectors/gmail/meta' +import { googleCalendarConnectorMeta } from '@/connectors/google-calendar/meta' import type { ConnectorAuthConfig } from '@/connectors/types' const OAUTH_AUTH: ConnectorAuthConfig = { @@ -268,6 +281,244 @@ describe('connectorServiceAccountSubject', () => { }) }) +describe('delegated connector access', () => { + const driveScope = 'https://www.googleapis.com/auth/drive.readonly' + const delegationAuth: ConnectorAuthConfig = { + ...OAUTH_AUTH, + serviceAccountScopes: [driveScope], + adminServiceAccountScopes: [ + driveScope, + 'https://www.googleapis.com/auth/admin.directory.user.readonly', + ], + serviceAccountDelegationScopes: [driveScope], + serviceAccountSubjectFieldId: 'adminEmail', + } + + function resolve( + auth: ConnectorAuthConfig = delegationAuth, + accessMode: ConnectorAccessMode = 'admin' + ) { + return resolveConnectorAccessToken({ + auth, + accessMode, + connector: credentialConnector('service-credential'), + userId: 'actor', + requestId: 'request', + sourceConfig: { adminEmail: 'admin@example.com' }, + }) + } + + beforeEach(() => { + vi.clearAllMocks() + mockResolveTokenBundle.mockResolvedValue({ accessToken: 'directory-token' }) + mockResolveOAuthAccountId.mockResolvedValue({ + credentialType: 'service_account', + providerId: 'google-service-account', + credentialId: 'service-credential', + }) + mockGetServiceAccountToken.mockResolvedValue('user-drive-token') + }) + + it('keeps directory scopes on the admin token and delegates only the declared Drive scope', async () => { + const token = await resolve() + expect(mockResolveTokenBundle).toHaveBeenCalledWith( + 'service-credential', + 'actor', + 'request', + delegationAuth.adminServiceAccountScopes, + 'admin@example.com' + ) + await expect(token?.getDelegatedAccessToken?.(' Employee@Example.com ')).resolves.toBe( + 'user-drive-token' + ) + expect(mockGetServiceAccountToken).toHaveBeenCalledWith( + 'service-credential', + [driveScope], + 'employee@example.com' + ) + expect(mockResolveTokenBundle).toHaveBeenCalledTimes(1) + expect(syncContextForToken(token!)).toEqual({ + getDelegatedAccessToken: token?.getDelegatedAccessToken, + }) + expect(JSON.stringify(syncContextForToken(token!))).toBe('{}') + }) + + it.each([ + null, + { credentialType: 'oauth', credentialId: 'service-credential', providerId: 'google-drive' }, + { + credentialType: 'service_account', + credentialId: 'service-credential', + providerId: 'atlassian-service-account', + }, + { + credentialType: 'managed_oauth', + credentialId: 'service-credential', + providerId: 'google-service-account', + }, + ])('does not give other credential identities a delegated capability: %j', async (identity) => { + mockResolveOAuthAccountId.mockResolvedValue(identity) + expect(await resolve()).toEqual({ accessToken: 'directory-token' }) + expect(mockGetServiceAccountToken).not.toHaveBeenCalled() + }) + + it('does not resolve a delegation identity for ordinary connector auth', async () => { + expect(await resolve(OAUTH_AUTH)).toEqual({ accessToken: 'directory-token' }) + expect(mockResolveOAuthAccountId).not.toHaveBeenCalled() + }) + + it.each(['members', 'workspace'] as const)( + 'uses only content scopes without delegation capability for %s crawls', + async (accessMode) => { + expect(await resolve(delegationAuth, accessMode)).toEqual({ accessToken: 'directory-token' }) + expect(mockResolveTokenBundle).toHaveBeenCalledWith( + 'service-credential', + 'actor', + 'request', + [driveScope], + 'admin@example.com' + ) + expect(mockResolveOAuthAccountId).not.toHaveBeenCalled() + expect(mockGetServiceAccountToken).not.toHaveBeenCalled() + } + ) + + it.each(['', ' ', 'not-an-email', 'user@example.com\nother@example.com'])( + 'rejects an invalid delegated subject: %j', + async (subject) => { + const token = await resolve() + await expect(token?.getDelegatedAccessToken?.(subject)).rejects.toThrow( + 'valid Workspace user email' + ) + expect(mockGetServiceAccountToken).not.toHaveBeenCalled() + } + ) + + it('captures its scope grant and propagates later credential revocation without OAuth fallback', async () => { + const scopes = [driveScope] + const token = await resolve({ ...delegationAuth, serviceAccountDelegationScopes: scopes }) + scopes.push('https://www.googleapis.com/auth/drive') + await token?.getDelegatedAccessToken?.('employee@example.com') + expect(mockGetServiceAccountToken).toHaveBeenLastCalledWith( + 'service-credential', + [driveScope], + 'employee@example.com' + ) + mockGetServiceAccountToken.mockRejectedValueOnce(new Error('Service account is unavailable')) + await expect(token?.getDelegatedAccessToken?.('employee@example.com')).rejects.toThrow( + 'Service account is unavailable' + ) + expect(mockResolveTokenBundle).toHaveBeenCalledTimes(1) + }) +}) + +describe.each([ + { + name: 'Gmail', + auth: gmailConnectorMeta.auth, + contentScope: 'https://www.googleapis.com/auth/gmail.readonly', + }, + { + name: 'Google Calendar', + auth: googleCalendarConnectorMeta.auth, + contentScope: 'https://www.googleapis.com/auth/calendar.events.readonly', + }, +])('$name declared company authentication', ({ auth, contentScope }) => { + const directoryScope = 'https://www.googleapis.com/auth/admin.directory.user.readonly' + const resolve = (accessMode: ConnectorAccessMode) => + resolveConnectorAccessToken({ + auth, + accessMode, + connector: credentialConnector('google-service'), + userId: 'actor', + requestId: 'request', + sourceConfig: { + adminEmail: ' Directory.Admin@Example.com ', + scopes: ['untrusted.write'], + userEmail: 'unverified@example.com', + }, + }) + + beforeEach(() => { + vi.clearAllMocks() + mockResolveTokenBundle.mockResolvedValue({ accessToken: 'directory-token' }) + mockResolveOAuthAccountId.mockResolvedValue({ + credentialType: 'service_account', + providerId: 'google-service-account', + credentialId: 'google-service', + }) + mockGetServiceAccountToken.mockResolvedValue('delegated-read-token') + }) + + it('mints Directory and per-user tokens from actual connector metadata with separate fixed scope sets', async () => { + const token = await resolve('admin') + expect(mockResolveTokenBundle).toHaveBeenCalledExactlyOnceWith( + 'google-service', + 'actor', + 'request', + [directoryScope], + 'directory.admin@example.com' + ) + expect(token?.getDelegatedAccessToken).toBeTypeOf('function') + await expect(token?.getDelegatedAccessToken?.(' Employee@Example.com ')).resolves.toBe( + 'delegated-read-token' + ) + expect(mockGetServiceAccountToken).toHaveBeenCalledExactlyOnceWith( + 'google-service', + [contentScope], + 'employee@example.com' + ) + expect(syncContextForToken(token!)).toEqual({ + getDelegatedAccessToken: token?.getDelegatedAccessToken, + }) + expect(JSON.stringify(syncContextForToken(token!))).toBe('{}') + expect(isConnectorCredentialTypeAllowed(auth, 'admin', 'oauth')).toBe(false) + expect(isConnectorCredentialTypeAllowed(auth, 'admin', 'service_account')).toBe(true) + }) + + it.each(['members', 'workspace'] as const)( + 'keeps %s OAuth available without a company delegation capability', + async (accessMode) => { + const token = await resolve(accessMode) + expect(token).toEqual({ accessToken: 'directory-token' }) + expect(mockResolveTokenBundle).toHaveBeenCalledExactlyOnceWith( + 'google-service', + 'actor', + 'request', + [contentScope], + 'directory.admin@example.com' + ) + expect(isConnectorCredentialTypeAllowed(auth, accessMode, 'oauth')).toBe(true) + expect(mockResolveOAuthAccountId).not.toHaveBeenCalled() + expect(mockGetServiceAccountToken).not.toHaveBeenCalled() + } + ) + + it('does not expose delegation when a selected credential resolves to ordinary OAuth', async () => { + mockResolveOAuthAccountId.mockResolvedValue({ + credentialType: 'oauth', + providerId: auth.mode === 'oauth' ? auth.provider : '', + credentialId: 'google-service', + }) + expect(await resolve('admin')).toEqual({ accessToken: 'directory-token' }) + expect(mockGetServiceAccountToken).not.toHaveBeenCalled() + }) + + it('propagates service-account revocation without using an OAuth or Directory token fallback', async () => { + const token = await resolve('admin') + mockGetServiceAccountToken.mockRejectedValueOnce(new Error('Service account is unavailable')) + await expect(token?.getDelegatedAccessToken?.('employee@example.com')).rejects.toThrow( + 'Service account is unavailable' + ) + expect(mockResolveTokenBundle).toHaveBeenCalledOnce() + expect(mockGetServiceAccountToken).toHaveBeenCalledExactlyOnceWith( + 'google-service', + [contentScope], + 'employee@example.com' + ) + }) +}) + describe('impersonation on the connector path', () => { const withSubject: ConnectorAuthConfig = { ...OAUTH_AUTH, diff --git a/apps/sim/lib/knowledge/connectors/access-token.ts b/apps/sim/lib/knowledge/connectors/access-token.ts index 0620bc7976d..82b4996fab3 100644 --- a/apps/sim/lib/knowledge/connectors/access-token.ts +++ b/apps/sim/lib/knowledge/connectors/access-token.ts @@ -1,8 +1,14 @@ -import { normalizeEmail } from '@sim/utils/string' +import { isValidEmailSyntax, normalizeEmail } from '@sim/utils/string' import { decryptApiKey } from '@/lib/api-key/crypto' import { resourceScopeFromOwner } from '@/lib/core/resource-scope' import { resolveCredentialTokenIdentity } from '@/lib/credentials/access' -import { resolveCredentialTokenBundle } from '@/lib/oauth/credential-service' +import type { ConnectorAccessMode } from '@/lib/knowledge/connectors/access-modes' +import { + getServiceAccountToken, + resolveCredentialTokenBundle, + resolveOAuthAccountId, +} from '@/lib/oauth/credential-service' +import { GOOGLE_SERVICE_ACCOUNT_PROVIDER_ID } from '@/lib/oauth/types' import { getConnectorApiKeyConfig } from '@/connectors/auth' import type { ConnectorAuthConfig } from '@/connectors/types' @@ -23,6 +29,8 @@ export interface ConnectorAccessToken { cloudId?: string /** The trusted site domain belonging to the credential's cloud id. */ domain?: string + /** Server-only capability bound to this credential and the connector's declared scopes. */ + getDelegatedAccessToken?: (subject: string) => Promise } /** @@ -33,8 +41,14 @@ export interface ConnectorAccessToken { * screen needs no `serviceAccountScopes` of its own; declaring one is how a * connector says the two sets differ. */ -export function connectorServiceAccountScopes(auth: ConnectorAuthConfig): string[] | undefined { +export function connectorServiceAccountScopes( + auth: ConnectorAuthConfig, + accessMode: ConnectorAccessMode = 'workspace' +): string[] | undefined { if (auth.mode !== 'oauth') return undefined + if (accessMode === 'admin' && auth.adminServiceAccountScopes) { + return auth.adminServiceAccountScopes + } return auth.serviceAccountScopes ?? auth.requiredScopes } @@ -67,11 +81,12 @@ export function connectorServiceAccountSubject( * account mints its own token and ignores the argument entirely. * * Returns `null` when an OAuth credential has no resolvable token, which is a - * reconnect prompt rather than a fault. Throws only when the connector row and - * its declared auth mode disagree, which is a bug or a corrupted row. + * reconnect prompt rather than a fault. Credential and token-exchange failures + * propagate to the caller. */ export async function resolveConnectorAccessToken(params: { auth: ConnectorAuthConfig + accessMode?: ConnectorAccessMode connector: { credentialId: string | null; encryptedApiKey: string | null } userId: string requestId: string @@ -120,14 +135,39 @@ export async function resolveConnectorAccessToken(params: { connector.credentialId, userId, requestId, - connectorServiceAccountScopes(auth), + connectorServiceAccountScopes(auth, params.accessMode), subject, ...(githubRepositoryScope ? [{ githubRepositoryScope }] : []) ) if (!bundle?.accessToken) return null + let getDelegatedAccessToken: ConnectorAccessToken['getDelegatedAccessToken'] + if ( + params.accessMode === 'admin' && + auth.mode === 'oauth' && + auth.serviceAccountDelegationScopes?.length + ) { + const identity = await resolveOAuthAccountId(connector.credentialId) + if ( + identity?.credentialType === 'service_account' && + identity.providerId === GOOGLE_SERVICE_ACCOUNT_PROVIDER_ID && + identity.credentialId + ) { + const credentialId = identity.credentialId + const scopes = [...auth.serviceAccountDelegationScopes] + getDelegatedAccessToken = async (subject) => { + const email = normalizeEmail(subject) + if (!isValidEmailSyntax(email)) { + throw new Error('A valid Workspace user email is required for delegated access') + } + return getServiceAccountToken(credentialId, scopes, email) + } + } + } + return { accessToken: bundle.accessToken, + ...(getDelegatedAccessToken ? { getDelegatedAccessToken } : {}), ...(bundle.cloudId ? { cloudId: bundle.cloudId, ...(bundle.domain ? { domain: bundle.domain } : {}) } : {}), @@ -166,10 +206,15 @@ export async function resolveConnectorTokenUserId(input: { * way, so a connector behaves identically on all of them. */ export function syncContextForToken(token: ConnectorAccessToken): Record { - return token.cloudId - ? { - cloudId: token.cloudId, - ...(token.domain ? { credentialDomain: token.domain } : {}), - } - : {} + return { + ...(token.cloudId + ? { + cloudId: token.cloudId, + ...(token.domain ? { credentialDomain: token.domain } : {}), + } + : {}), + ...(token.getDelegatedAccessToken + ? { getDelegatedAccessToken: token.getDelegatedAccessToken } + : {}), + } } diff --git a/apps/sim/lib/knowledge/connectors/external-group-sync.test.ts b/apps/sim/lib/knowledge/connectors/external-group-sync.test.ts index 587b1fd8f6e..8dd68d764c0 100644 --- a/apps/sim/lib/knowledge/connectors/external-group-sync.test.ts +++ b/apps/sim/lib/knowledge/connectors/external-group-sync.test.ts @@ -198,7 +198,7 @@ describe('refreshConnectorDirectory', () => { await expect(refreshConnectorDirectory('connector-1', 'req-1')).resolves.toBe('skipped') expect(mockResolveToken).toHaveBeenCalledWith( - expect.objectContaining({ userId: 'credential-owner' }) + expect.objectContaining({ userId: 'credential-owner', accessMode: 'admin' }) ) }) diff --git a/apps/sim/lib/knowledge/connectors/external-group-sync.ts b/apps/sim/lib/knowledge/connectors/external-group-sync.ts index 836be5c6ec5..02e6e7fad4b 100644 --- a/apps/sim/lib/knowledge/connectors/external-group-sync.ts +++ b/apps/sim/lib/knowledge/connectors/external-group-sync.ts @@ -457,6 +457,7 @@ export async function refreshConnectorDirectory( const sourceConfig = connector.sourceConfig as Record const token = await resolveConnectorAccessToken({ auth: connectorConfig.auth, + accessMode: 'admin', connector, userId: credentialUserId, requestId, diff --git a/apps/sim/lib/knowledge/connectors/listing-checkpoint.test.ts b/apps/sim/lib/knowledge/connectors/listing-checkpoint.test.ts index aab32d1f187..bbe904f130b 100644 --- a/apps/sim/lib/knowledge/connectors/listing-checkpoint.test.ts +++ b/apps/sim/lib/knowledge/connectors/listing-checkpoint.test.ts @@ -206,17 +206,22 @@ describe('durable connector listing checkpoints', () => { it('restarts an expired provider cursor once with a new generation', async () => { const f = fixture({ ...checkpoint(), cursor: 'expired', listedCount: 700 }) const error = new Error('expired') + const databaseTime = new Date('2026-09-08T10:00:00Z') + const getGenerationStartedAt = vi.fn(async () => databaseTime) f.listDocuments .mockRejectedValueOnce(error) .mockResolvedValueOnce({ documents: [doc], hasMore: false }) const result = await runResumableListing({ ...f.input, + getGenerationStartedAt, connectorConfig: { listDocuments: f.listDocuments, isListingCursorInvalidError: (value) => value === error, }, }) expect(result.generationId).not.toBe('cycle-1') + expect(result.startedAt).toBe(databaseTime.toISOString()) + expect(getGenerationStartedAt).toHaveBeenCalledOnce() expect(result).toMatchObject({ complete: true, listedCount: 1 }) expect(f.listDocuments.mock.calls[1][2]).toBeUndefined() expect(f.processPage.mock.calls[0][1].generationId).toBe(result.generationId) diff --git a/apps/sim/lib/knowledge/connectors/listing-checkpoint.ts b/apps/sim/lib/knowledge/connectors/listing-checkpoint.ts index 8c730fdb8dc..f44c1feade5 100644 --- a/apps/sim/lib/knowledge/connectors/listing-checkpoint.ts +++ b/apps/sim/lib/knowledge/connectors/listing-checkpoint.ts @@ -89,6 +89,8 @@ export async function runResumableListing(input: { maxPages?: number beforePage: () => Promise getAccessToken: (page: number) => Promise + /** Uses the persistence clock when generation timestamps gate stored permission evidence. */ + getGenerationStartedAt?: () => Promise /** False retains this page's cursor after a bounded amount of durable work. */ processPage: ( documents: ExternalDocument[], @@ -124,7 +126,7 @@ export async function runResumableListing(input: { checkpoint = { ...checkpoint, generationId: generateId(), - startedAt: new Date().toISOString(), + startedAt: ((await input.getGenerationStartedAt?.()) ?? new Date()).toISOString(), cursor: null, complete: false, listedCount: 0, diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.integration.test.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.integration.test.ts index 2e5a58f49de..140e5c4e7d3 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.integration.test.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.integration.test.ts @@ -304,6 +304,7 @@ describe('member engine with a dedicated content credential', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() + dbChainMockFns.execute.mockImplementation(async () => [{ startedAt: new Date().toISOString() }]) }) it.each([undefined, 'organization'])( @@ -438,6 +439,7 @@ describe('member engine with a dedicated content credential', () => { expect(result.membersClaimed).toBe(0) expect(mocks.list).toHaveBeenCalledTimes(1) expect(mocks.list.mock.calls[0][0]).toBe('service-token') + expect(mocks.token).toHaveBeenCalledWith(expect.objectContaining({ accessMode: 'members' })) expect(mocks.observe).not.toHaveBeenCalled() }) diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts index 6a5ce0c6f7a..186e56e8d72 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -1334,6 +1334,7 @@ async function syncDedicatedMemberContent(input: { const resolveToken = async () => { const token = await resolveConnectorAccessToken({ auth: connectorConfig.auth, + accessMode: 'members', connector, userId, requestId: run.runId, diff --git a/apps/sim/lib/knowledge/connectors/mirrored-acls.test.ts b/apps/sim/lib/knowledge/connectors/mirrored-acls.test.ts index df8766d9a5f..43d96a6a955 100644 --- a/apps/sim/lib/knowledge/connectors/mirrored-acls.test.ts +++ b/apps/sim/lib/knowledge/connectors/mirrored-acls.test.ts @@ -21,6 +21,10 @@ function doc(externalId: string, acl?: readonly string[]): ExternalDocument { } describe('unansweredByListing', () => { + it('requests each unresolved ID once and honors inline answers from any duplicate', () => { + expect(unansweredByListing([doc('a'), doc('a', []), doc('b'), doc('b')])).toEqual([doc('b')]) + }) + it('names exactly the documents the listing left without an ACL', () => { expect( unansweredByListing([doc('a', ['u:alice@corp.com']), doc('b'), doc('c', []), doc('d')]).map( @@ -31,6 +35,15 @@ describe('unansweredByListing', () => { }) describe('mergeMirroredAcls', () => { + it.each([{ acl: [] }, { acl: ['invalid'] }])( + 'keeps an inline answer over fetched grants for an unresolved duplicate', + ({ acl }) => { + const result = mergeMirroredAcls([doc('a', acl), doc('a')], { a: ['pub'] }) + expect(result.acls.get('a')).toEqual(acl) + expect(result.unresolvedExternalIds.size).toBe(0) + } + ) + it("keeps the listing's answer where it gave one", () => { const { acls, unattributed } = mergeMirroredAcls([doc('a', ['u:alice@corp.com'])], { a: ['u:bob@corp.com'], @@ -50,11 +63,7 @@ describe('mergeMirroredAcls', () => { expect(unattributed).toBe(0) }) - /** - * A document nobody answered for is hidden and counted — never skipped, - * because skipping would leave it under an ACL this run did not verify. - */ - it('hides and counts a document neither source answered for', () => { + it('marks a document neither source answered for as unresolved', () => { const { acls, unattributed } = mergeMirroredAcls([doc('a'), doc('b')], { a: ['pub'] }) expect(acls.get('b')).toEqual([]) @@ -68,6 +77,29 @@ describe('mergeMirroredAcls', () => { expect(unattributed).toBe(0) }) + it.each([ + { docs: [doc('a', ['pub']), doc('a')], expected: ['pub'] }, + { docs: [doc('a'), doc('a', ['pub'])], expected: ['pub'] }, + { docs: [doc('a', ['pub']), doc('a', []), doc('a')], expected: [] }, + { docs: [doc('a', ['pub']), doc('a', ['invalid']), doc('a')], expected: ['invalid'] }, + ])( + 'retains the latest explicit same-page answer when a duplicate is unresolved', + ({ docs, expected }) => { + const result = mergeMirroredAcls(docs, {}) + expect(result.acls.get('a')).toEqual(expected) + expect(result.unresolvedExternalIds.size).toBe(0) + } + ) + + it('distinguishes unresolved files from explicit empty responses', () => { + const result = mergeMirroredAcls( + [doc('unknown'), doc('inline-empty', []), doc('fetched-empty')], + { 'fetched-empty': [] } + ) + expect([...result.unresolvedExternalIds]).toEqual(['unknown']) + expect(result.unattributed).toBe(1) + }) + it('answers for every listed document, in listing order', () => { const { acls } = mergeMirroredAcls([doc('z', ['pub']), doc('a')], {}) diff --git a/apps/sim/lib/knowledge/connectors/mirrored-acls.ts b/apps/sim/lib/knowledge/connectors/mirrored-acls.ts index 9f49539b264..bf64ab83e70 100644 --- a/apps/sim/lib/knowledge/connectors/mirrored-acls.ts +++ b/apps/sim/lib/knowledge/connectors/mirrored-acls.ts @@ -3,15 +3,24 @@ import type { MirroredDocumentAcl } from '@/lib/knowledge/access/types' import type { ExternalDocument } from '@/connectors/types' export interface MirroredAcls { - /** Every listed document's ACL, keyed by external id; readable by nobody where neither source answered. */ + /** Each listed document's ACL; unresolved entries are distinguished before persistence. */ acls: Map /** Listed documents neither the listing nor the fetch could speak for. */ + unresolvedExternalIds: ReadonlySet unattributed: number } /** The listed documents whose ACL the listing left unset. */ export function unansweredByListing(externalDocs: readonly ExternalDocument[]): ExternalDocument[] { - return externalDocs.filter((doc) => !doc.acl) + const answered = new Set( + externalDocs.filter((doc) => doc.acl !== undefined).map((doc) => doc.externalId) + ) + const requested = new Set() + return externalDocs.filter((doc) => { + if (answered.has(doc.externalId) || requested.has(doc.externalId)) return false + requested.add(doc.externalId) + return true + }) } /** @@ -21,23 +30,26 @@ export function unansweredByListing(externalDocs: readonly ExternalDocument[]): * * The listing's answer wins where it exists, because it is the cheaper one and * was taken from the same page the document came from. A document neither - * answered for is readable by nobody rather than skipped — leaving its previous - * ACL in place would keep serving it under permissions this run failed to - * verify — and is counted, because a connector that declares it mirrors ACLs is - * promising an answer for everything it lists. + * answered for is marked unresolved. Persistence can retain another verified + * observation from the same crawl, but must hide older, unverified grants. + * Explicit answers, including empty or malformed ACLs, replace earlier answers. */ export function mergeMirroredAcls( externalDocs: readonly ExternalDocument[], fetched: Readonly> ): MirroredAcls { const acls = new Map() - let unattributed = 0 + const unresolvedExternalIds = new Set() for (const doc of externalDocs) { - const acl = doc.acl ?? fetched[doc.externalId] - if (!acl) unattributed += 1 + if (doc.acl !== undefined) acls.set(doc.externalId, doc.acl) + } + for (const doc of externalDocs) { + if (acls.has(doc.externalId)) continue + const acl = fetched[doc.externalId] acls.set(doc.externalId, acl ?? EMPTY_ACL) + if (acl === undefined) unresolvedExternalIds.add(doc.externalId) } - return { acls, unattributed } + return { acls, unresolvedExternalIds, unattributed: unresolvedExternalIds.size } } /** diff --git a/apps/sim/lib/knowledge/connectors/sync-content-pass.test.ts b/apps/sim/lib/knowledge/connectors/sync-content-pass.test.ts index f05abaea968..cec28ccf473 100644 --- a/apps/sim/lib/knowledge/connectors/sync-content-pass.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-content-pass.test.ts @@ -1,5 +1,10 @@ /** @vitest-environment node */ -import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' +import { + dbChainMockFns, + queueTableRows, + resetDbChainMock as resetDatabaseMock, + schemaMock, +} from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import type { ConnectorAccessMode } from '@/lib/knowledge/connectors/access-modes' @@ -13,6 +18,11 @@ import { stillHoldsSyncLock } from '@/lib/knowledge/connectors/sync-lock' import { confluenceConnector } from '@/connectors/confluence/confluence' import type { ExternalDocument, SyncResult } from '@/connectors/types' +function resetDbChainMock() { + resetDatabaseMock() + dbChainMockFns.execute.mockImplementation(async () => [{ startedAt: new Date().toISOString() }]) +} + const mocks = vi.hoisted(() => ({ upload: vi.fn(), deleteFile: vi.fn(), @@ -272,12 +282,16 @@ async function runPass( readCurrent?: boolean forceRehydrate?: boolean checkpoint?: ListingCheckpoint + databaseTime?: Date getDocument?: () => Promise } = {} ) { vi.clearAllMocks() resetDbChainMock() vi.setSystemTime(new Date(Date.now() + 60_000)) + if (options.databaseTime) { + dbChainMockFns.execute.mockResolvedValue([{ startedAt: options.databaseTime.toISOString() }]) + } for (let index = 0; index < 16; index++) { queueTableRows(schemaMock.knowledgeConnector, [ { @@ -367,6 +381,25 @@ function contentWrite(): Record { } describe('content pass checkpoint intent', () => { + it('uses the database clock for a new generation despite a different worker clock', async () => { + const databaseTime = new Date('2026-09-08T10:00:00Z') + sourceBody = { value: '

Current content

' } + const { pass } = await runPass({ databaseTime, access: 'admin' }) + expect(pass.checkpoint.startedAt).toBe(databaseTime.toISOString()) + expect(mocks.onPage).toHaveBeenCalledWith(expect.any(Array), databaseTime) + }) + + it('passes the durable generation start to ACL updates when resuming with a later worker clock', async () => { + const checkpoint = beginListingCheckpoint({ + fingerprint: 'a'.repeat(64), + generationId: 'previous-run', + startedAt: new Date('2026-09-08T11:00:00Z'), + }) + sourceBody = { value: '

Current content

' } + await runPass({ checkpoint, access: 'admin' }) + expect(mocks.onPage).toHaveBeenCalledWith(expect.any(Array), new Date(checkpoint.startedAt)) + }) + it.each([ { name: 'full', incrementalSince: undefined }, { name: 'incremental', incrementalSince: new Date('2026-09-07T12:00:00Z') }, diff --git a/apps/sim/lib/knowledge/connectors/sync-content-pass.ts b/apps/sim/lib/knowledge/connectors/sync-content-pass.ts index 94c1e338cb5..2031682028f 100644 --- a/apps/sim/lib/knowledge/connectors/sync-content-pass.ts +++ b/apps/sim/lib/knowledge/connectors/sync-content-pass.ts @@ -57,7 +57,7 @@ interface ContentPassInput { forceRehydrate: boolean fullSync?: boolean deadlineAt: number - onPage?: (documents: ExternalDocument[]) => Promise + onPage?: (documents: ExternalDocument[], generationStartedAt: Date) => Promise } /** One durable content cycle shared by content-owned and member-visibility connectors. */ @@ -67,23 +67,33 @@ export async function runConnectorContentPass(input: ContentPassInput) { await assertSyncLeaseHeldInTx(tx, input.connectorId, input.lease) return fn(tx) }) + const readGenerationStartedAt = async (tx: DbOrTx): Promise => { + const [clock] = await tx.execute<{ startedAt: string }>( + sql`SELECT statement_timestamp()::text AS "startedAt"` + ) + const startedAt = new Date(clock?.startedAt ?? '') + if (!Number.isFinite(startedAt.getTime())) + throw new Error('Could not read the sync database clock') + return startedAt + } let checkpoint = readListingCheckpoint(input.connector.listingCheckpoint, input.fingerprint) /** A Full resync must revisit documents before an ordinary listing's saved cursor. */ if (!checkpoint || (input.forceRehydrate && !checkpoint.forceRehydrate)) { - checkpoint = beginListingCheckpoint({ - fingerprint: input.fingerprint, - generationId: input.runId, - startedAt: new Date(), - incrementalSince: input.lastSyncAt, - forceRehydrate: input.forceRehydrate, - fullSync: input.fullSync, - }) - await withLease((tx) => - tx + checkpoint = await withLease(async (tx) => { + const next = beginListingCheckpoint({ + fingerprint: input.fingerprint, + generationId: input.runId, + startedAt: await readGenerationStartedAt(tx), + incrementalSince: input.lastSyncAt, + forceRehydrate: input.forceRehydrate, + fullSync: input.fullSync, + }) + await tx .update(knowledgeConnector) - .set({ listingCheckpoint: checkpoint }) + .set({ listingCheckpoint: next }) .where(input.lease.stillHeld()) - ) + return next + }) } let hydratedCount = 0 checkpoint = await runResumableListing({ @@ -93,6 +103,7 @@ export async function runConnectorContentPass(input: ContentPassInput) { checkpoint, deadlineAt: input.deadlineAt, beforePage: input.lease.beatIfDue, + getGenerationStartedAt: () => withLease(readGenerationStartedAt), getAccessToken: input.getAccessToken, processPage: async (documents, cycle) => { const externalIds = documents.map((item) => item.externalId) @@ -161,7 +172,7 @@ export async function runConnectorContentPass(input: ContentPassInput) { }, }) if (!finished) return false - await input.onPage?.(documents) + await input.onPage?.(documents, startedAt) await withLease(async (tx) => { const verified = externalIds.filter((id) => !state.failedExternalIds.has(id)) for (let offset = 0; offset < verified.length; offset += 500) { diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts index 026f12e3cad..61a6fdbe07b 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts @@ -9,11 +9,12 @@ import { hasMockCondition, type MockCondition, queueTableRows, - resetDbChainMock, + resetDbChainMock as resetDatabaseMock, schemaMock, } from '@sim/testing' import { generateShortId } from '@sim/utils/id' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import * as connectorTokens from '@/lib/knowledge/connectors/access-token' import { executeSync, isConnectorRunnableStatus } from '@/lib/knowledge/connectors/sync-engine' import { classifySuspectListing, @@ -27,6 +28,11 @@ import { } from '@/lib/knowledge/connectors/sync-primitives' import type { ExternalDocument } from '@/connectors/types' +function resetDbChainMock() { + resetDatabaseMock() + dbChainMockFns.execute.mockImplementation(async () => [{ startedAt: new Date().toISOString() }]) +} + vi.mock('drizzle-orm', () => drizzleOrmMock) const { mockProcessDocumentsWithQueue, mockUploadFile } = vi.hoisted(() => ({ mockProcessDocumentsWithQueue: vi.fn(), @@ -2634,6 +2640,24 @@ describe('executeSync heartbeats during the listing phase', () => { dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'c-1', accessMode: 'workspace' }]) } + it.each(['workspace', 'admin'] as const)( + 'uses the locked source mode %s when resolving its token', + async (accessMode) => { + primeSyncUpToListing() + dbChainMockFns.returning.mockReset() + dbChainMockFns.returning.mockResolvedValueOnce([{ ...CONNECTOR, accessMode }]) + const resolveToken = vi + .spyOn(connectorTokens, 'resolveConnectorAccessToken') + .mockRejectedValueOnce(new Error('Stop at token resolution')) + try { + await executeSync('c-1', { billingAttribution: { workspaceId: 'ws-1' } as never }) + expect(resolveToken).toHaveBeenCalledWith(expect.objectContaining({ accessMode })) + } finally { + resolveToken.mockRestore() + } + } + ) + it('beats between pages and abandons the run when the lock was reclaimed', async () => { const { executeSync } = await import('@/lib/knowledge/connectors/sync-engine') const { SYNC_LOCK_HEARTBEAT_INTERVAL_MS } = await import( diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.ts b/apps/sim/lib/knowledge/connectors/sync-engine.ts index cf079e9c5c3..5cfb6a2d08a 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.ts @@ -19,6 +19,7 @@ import { resourceScopeFields, resourceScopeFromOwner } from '@/lib/core/resource import { EMPTY_ACL } from '@/lib/knowledge/access/tokens' import { CONTENT_ENGINE_ACCESS_MODES, + type ContentEngineAccessMode, effectiveConnectorSyncIntervalMinutes, isContentEngineAccessMode, mirrorsSourceAcls, @@ -105,11 +106,9 @@ export { * documents would let a revoked grant stay readable until somebody happened to * edit the file. * - * A listed document the connector could not speak for gets an empty ACL, which - * hides it. That is the safe direction and it is visible — a connector - * declaring {@link ConnectorMeta.mirrorsSourceAcls} is promising an ACL for - * every document it lists, so a missing one is a bug in the connector rather - * than an expected state to paper over. + * An unresolved observation cannot erase another identity's verified ACL from + * this crawl. Older unverified grants are hidden, while explicit source answers + * always replace existing permissions. */ async function applySourceMirroredAcls(input: { connectorId: string @@ -121,6 +120,7 @@ async function applySourceMirroredAcls(input: { /** External ids of every live document the connector owns, listed this run or not. */ ownedExternalIds: readonly (string | null)[] lease?: SyncRunLease + generationStartedAt: Date }): Promise { const { connectorId, connectorConfig, externalDocs } = input @@ -140,7 +140,8 @@ async function applySourceMirroredAcls(input: { input.syncContext ) : {} - const { acls, unattributed } = mergeMirroredAcls(externalDocs, fetched) + const { acls, unattributed, unresolvedExternalIds } = mergeMirroredAcls(externalDocs, fetched) + const evidence = { unresolvedExternalIds, generationStartedAt: input.generationStartedAt } const listed = acls.size /** * A document this run did not list has no ACL this run can vouch for, so it @@ -153,9 +154,9 @@ async function applySourceMirroredAcls(input: { const written = input.lease ? await db.transaction(async (tx) => { await assertSyncLeaseHeldInTx(tx, connectorId, input.lease!) - return persistDocumentAcls(connectorId, acls, tx) + return persistDocumentAcls(connectorId, acls, tx, evidence) }) - : await persistDocumentAcls(connectorId, acls) + : await persistDocumentAcls(connectorId, acls, db, evidence) logger.info('Mirrored source permissions onto connector documents', { connectorId, listed, @@ -164,10 +165,13 @@ async function applySourceMirroredAcls(input: { ...(unattributed > 0 ? { unattributed } : {}), }) if (unattributed > 0) { - logger.error('Connector listed documents without an ACL; they are readable by nobody', { - connectorId, - unattributed, - }) + logger.warn( + 'Connector listed documents without an ACL; only current-crawl evidence is retained', + { + connectorId, + unattributed, + } + ) } } @@ -641,11 +645,13 @@ async function resolveAccessToken( connector: { credentialId: string | null; encryptedApiKey: string | null }, connectorConfig: { auth: ConnectorAuthConfig }, userId: string, - sourceConfig: Record + sourceConfig: Record, + accessMode: ContentEngineAccessMode ): Promise { const requestId = `sync-${connector.credentialId}` const resolved = await resolveConnectorAccessToken({ auth: connectorConfig.auth, + accessMode, connector, userId, requestId, @@ -880,6 +886,7 @@ export async function executeSync( if (!isContentEngineAccessMode(connector.accessMode)) { throw new Error(`Connector ${connectorId} left the content engine's modes while locked`) } + const accessMode = connector.accessMode const mirrored = mirrorsSourceAcls(connector.accessMode) const sourceConfig = connector.sourceConfig as Record const syncStartedAt = new Date() @@ -914,7 +921,8 @@ export async function executeSync( connector, connectorConfig, credentialUserId, - sourceConfig + sourceConfig, + accessMode ) /** Re-resolves the token for every OAuth call after the first, so a long run outlives a short-lived token. */ const refreshOAuthToken = async (): Promise => { @@ -923,7 +931,8 @@ export async function executeSync( connector, connectorConfig, credentialUserId, - sourceConfig + sourceConfig, + accessMode ) } } @@ -1091,7 +1100,7 @@ export async function executeSync( fullSync: options.fullSync, deadlineAt: syncStartedAt.getTime() + (CONNECTOR_SYNC_MAX_DURATION_SECONDS - 300) * 1000, onPage: mirrored - ? async (externalDocs) => { + ? async (externalDocs, generationStartedAt) => { await directoryRefreshed await applySourceMirroredAcls({ connectorId, @@ -1100,6 +1109,7 @@ export async function executeSync( syncContext, accessToken: credentialToken.accessToken, externalDocs, + generationStartedAt, ownedExternalIds: [], lease, }) diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts index d18f3dbed70..34c9e6a2423 100644 --- a/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.test.ts @@ -70,7 +70,10 @@ describe('persistDocumentAcls', () => { expect(dbChainMockFns.set).toHaveBeenCalledWith({ acl: ['u:alice@corp.com'], aclRequirements: [], - aclVerifiedAt: expect.any(Date), + aclVerifiedAt: expect.objectContaining({ + strings: ["statement_timestamp() AT TIME ZONE 'UTC'"], + values: [], + }), }) }) @@ -108,12 +111,18 @@ describe('persistDocumentAcls', () => { expect(dbChainMockFns.set).toHaveBeenNthCalledWith(1, { acl: ['u:alice@corp.com'], aclRequirements: [], - aclVerifiedAt: expect.any(Date), + aclVerifiedAt: expect.objectContaining({ + strings: ["statement_timestamp() AT TIME ZONE 'UTC'"], + values: [], + }), }) expect(dbChainMockFns.set).toHaveBeenNthCalledWith(2, { acl: ['u:bob@corp.com'], aclRequirements: [], - aclVerifiedAt: expect.any(Date), + aclVerifiedAt: expect.objectContaining({ + strings: ["statement_timestamp() AT TIME ZONE 'UTC'"], + values: [], + }), }) }) @@ -132,7 +141,10 @@ describe('persistDocumentAcls', () => { expect(dbChainMockFns.set).toHaveBeenCalledWith({ acl: ['u:alice@corp.com', 'u:bob@corp.com'], aclRequirements: [], - aclVerifiedAt: expect.any(Date), + aclVerifiedAt: expect.objectContaining({ + strings: ["statement_timestamp() AT TIME ZONE 'UTC'"], + values: [], + }), }) }) @@ -164,12 +176,18 @@ describe('persistDocumentAcls', () => { expect(dbChainMockFns.set).toHaveBeenNthCalledWith(1, { acl: ['u:alice@corp.com'], aclRequirements: [['u:alice@corp.com'], []], - aclVerifiedAt: expect.any(Date), + aclVerifiedAt: expect.objectContaining({ + strings: ["statement_timestamp() AT TIME ZONE 'UTC'"], + values: [], + }), }) expect(dbChainMockFns.set).toHaveBeenNthCalledWith(2, { acl: ['u:alice@corp.com'], aclRequirements: [['u:alice@corp.com'], ['g:confluence:site:team']], - aclVerifiedAt: expect.any(Date), + aclVerifiedAt: expect.objectContaining({ + strings: ["statement_timestamp() AT TIME ZONE 'UTC'"], + values: [], + }), }) }) diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.ts index a090c7d011f..0dbdbe4935b 100644 --- a/apps/sim/lib/knowledge/connectors/sync-persistence.ts +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.ts @@ -3,7 +3,7 @@ import { document, embedding, knowledgeBase, knowledgeConnector } from '@sim/db/ import { createLogger } from '@sim/logger' import { chunkArray } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' -import { and, eq, exists, inArray, isNull, sql } from 'drizzle-orm' +import { and, eq, exists, inArray, isNull, lt, or, sql } from 'drizzle-orm' import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' import type { DbOrTx } from '@/lib/db/types' import { textArrayLiteral } from '@/lib/knowledge/access/predicate' @@ -94,18 +94,23 @@ export interface DocumentAclWriteResult { * Permission-only changes must not trigger re-embedding. Unchanged ACLs still refresh * their evidence timestamp; failed fetches cannot extend it. Malformed or oversized * ACLs are stored as unreadable so the previous grant cannot survive failed verification. + * An unresolved duplicate may retain evidence verified during this durable crawl, + * without refreshing its timestamp; explicit empty ACLs always revoke access. */ export async function persistDocumentAcls( connectorId: string, acls: ReadonlyMap, - executor: DbOrTx = db + executor: DbOrTx = db, + evidence?: { + unresolvedExternalIds: ReadonlySet + generationStartedAt: Date + } ): Promise { const byAcl = new Map< string, - { acl: string[]; requirements: string[][]; externalIds: string[] } + { acl: string[]; requirements: string[][]; externalIds: string[]; unresolved: boolean } >() let rejected = 0 - const verifiedAt = new Date() for (const [externalId, value] of acls) { const validation = validateMirroredDocumentAcl(value) @@ -127,23 +132,37 @@ export async function persistDocumentAcls( validation.valid && validation.requirements.length > 0 ? [acl, ...validation.requirements] : [] - const key = JSON.stringify([acl, requirements]) + const unresolved = Boolean( + evidence?.unresolvedExternalIds.has(externalId) && validation.valid && acl.length === 0 + ) + const key = JSON.stringify([acl, requirements, unresolved]) const group = byAcl.get(key) if (group) group.externalIds.push(externalId) - else byAcl.set(key, { acl, requirements, externalIds: [externalId] }) + else byAcl.set(key, { acl, requirements, externalIds: [externalId], unresolved }) } let updated = 0 - for (const { acl, requirements, externalIds } of byAcl.values()) { + for (const { acl, requirements, externalIds, unresolved } of byAcl.values()) { for (const batch of chunkArray(externalIds, ACL_WRITE_BATCH_SIZE)) { const rows = await executor .update(document) .set({ acl, aclRequirements: requirements, - aclVerifiedAt: acl.length > 0 ? verifiedAt : null, + aclVerifiedAt: acl.length > 0 ? sql`statement_timestamp() AT TIME ZONE 'UTC'` : null, }) - .where(and(eq(document.connectorId, connectorId), inArray(document.externalId, batch))) + .where( + and( + eq(document.connectorId, connectorId), + inArray(document.externalId, batch), + unresolved && evidence + ? or( + isNull(document.aclVerifiedAt), + lt(document.aclVerifiedAt, evidence.generationStartedAt) + ) + : undefined + ) + ) .returning({ id: document.id }) updated += rows.length } diff --git a/apps/sim/lib/oauth/credential-service.test.ts b/apps/sim/lib/oauth/credential-service.test.ts index 35f56f59fe5..d28450cba11 100644 --- a/apps/sim/lib/oauth/credential-service.test.ts +++ b/apps/sim/lib/oauth/credential-service.test.ts @@ -1,12 +1,14 @@ /** * @vitest-environment node */ +import { generateKeyPairSync, verify } from 'node:crypto' import { account, credential } from '@sim/db/schema' import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ coalesceLocally: vi.fn(), + decryptSecret: vi.fn(), getFreshestSlackChain: vi.fn(), getRecentTerminalError: vi.fn(), logger: { @@ -34,6 +36,10 @@ vi.mock('@/lib/concurrency/leader-lock', () => ({ withLeaderLock: mocks.withLeaderLock, })) +vi.mock('@/lib/core/security/encryption', () => ({ + decryptSecret: mocks.decryptSecret, +})) + vi.mock('@/lib/oauth/instagram', () => ({ isInstagramProvider: vi.fn(() => false), shouldProactivelyRefreshInstagramToken: vi.fn(() => false), @@ -71,9 +77,11 @@ vi.mock('@/lib/oauth/terminal-errors', () => ({ import { getOAuthToken, + getServiceAccountToken, refreshTokenIfNeeded, resolveCredentialTokenBundle, } from '@/lib/oauth/credential-service' +import { GOOGLE_SERVICE_ACCOUNT_PROVIDER_ID } from '@/lib/oauth/types' const RAW_CREDENTIAL_ID = 'credential-raw-secret-id' const RAW_ACCOUNT_ID = 'account-raw-secret-id' @@ -329,3 +337,120 @@ describe('non-refreshable OAuth token expiry', () => { } ) }) + +describe('Google service-account token minting', () => { + const { privateKey, publicKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }) + const fetchMock = vi.fn() + const row = { + type: 'service_account', + providerId: GOOGLE_SERVICE_ACCOUNT_PROVIDER_ID, + revokedAt: null, + encryptedServiceAccountKey: 'encrypted-google-key', + } + const driveScope = 'https://www.googleapis.com/auth/drive.readonly' + const now = new Date('2026-09-09T18:00:00.000Z') + + beforeEach(() => { + resetDbChainMock() + vi.clearAllMocks() + vi.useFakeTimers() + vi.setSystemTime(now) + vi.stubGlobal('fetch', fetchMock) + fetchMock.mockImplementation(async () => Response.json({ access_token: 'google-access-token' })) + mocks.decryptSecret.mockResolvedValue({ + decrypted: JSON.stringify({ + client_email: 'crawler@qa-project.iam.gserviceaccount.com', + private_key: privateKey.export({ type: 'pkcs8', format: 'pem' }).toString(), + token_uri: 'https://oauth2.googleapis.com/token', + }), + }) + }) + + afterEach(() => { + vi.unstubAllGlobals() + vi.useRealTimers() + }) + + it.each([ + { label: 'missing', rows: [] }, + { label: 'OAuth', rows: [{ ...row, type: 'oauth' }] }, + { label: 'managed OAuth', rows: [{ ...row, type: 'managed_oauth' }] }, + { label: 'another provider', rows: [{ ...row, providerId: 'atlassian-service-account' }] }, + { label: 'Google OAuth provider', rows: [{ ...row, providerId: 'google' }] }, + { label: 'missing provider', rows: [{ ...row, providerId: null }] }, + { label: 'revoked', rows: [{ ...row, revokedAt: now }] }, + { label: 'missing key', rows: [{ ...row, encryptedServiceAccountKey: null }] }, + { label: 'empty key', rows: [{ ...row, encryptedServiceAccountKey: '' }] }, + ])('rejects a $label credential before decryption or token exchange', async ({ rows }) => { + queueTableRows(credential, rows) + + await expect( + getServiceAccountToken('credential-1', [driveScope], 'member@example.com') + ).rejects.toThrow('Google service account credential is unavailable') + + expect(mocks.decryptSecret).not.toHaveBeenCalled() + expect(fetchMock).not.toHaveBeenCalled() + expect(mocks.refreshOAuthToken).not.toHaveBeenCalled() + }) + + it.each([ + { label: 'delegated Drive', scope: driveScope, subject: 'member@example.com' }, + { + label: 'project-scoped Vertex', + scope: 'https://www.googleapis.com/auth/cloud-platform', + subject: undefined, + }, + ])('preserves $label JWT claims and signs with the validated key', async ({ scope, subject }) => { + queueTableRows(credential, [row]) + + await expect(getServiceAccountToken('credential-1', [scope], subject)).resolves.toBe( + 'google-access-token' + ) + + expect(mocks.decryptSecret).toHaveBeenCalledWith('encrypted-google-key') + expect(fetchMock).toHaveBeenCalledExactlyOnceWith('https://oauth2.googleapis.com/token', { + method: 'POST', + redirect: 'error', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: expect.any(URLSearchParams), + }) + const body = fetchMock.mock.calls[0][1]?.body + expect(body).toBeInstanceOf(URLSearchParams) + if (!(body instanceof URLSearchParams)) throw new Error('Expected a JWT token exchange') + expect(body.get('grant_type')).toBe('urn:ietf:params:oauth:grant-type:jwt-bearer') + const [header, payload, signature] = (body.get('assertion') ?? '').split('.') + expect(JSON.parse(Buffer.from(header, 'base64url').toString())).toEqual({ + alg: 'RS256', + typ: 'JWT', + }) + expect(JSON.parse(Buffer.from(payload, 'base64url').toString())).toEqual({ + iss: 'crawler@qa-project.iam.gserviceaccount.com', + scope, + aud: 'https://oauth2.googleapis.com/token', + iat: now.getTime() / 1000, + exp: now.getTime() / 1000 + 3600, + ...(subject ? { sub: subject } : {}), + }) + expect( + verify( + 'RSA-SHA256', + Buffer.from(`${header}.${payload}`), + publicKey, + Buffer.from(signature, 'base64url') + ) + ).toBe(true) + }) + + it('checks revocation again before every delegated token mint', async () => { + queueTableRows(credential, [row]) + await getServiceAccountToken('credential-1', [driveScope], 'first@example.com') + queueTableRows(credential, [{ ...row, revokedAt: now }]) + + await expect( + getServiceAccountToken('credential-1', [driveScope], 'second@example.com') + ).rejects.toThrow('Google service account credential is unavailable') + + expect(mocks.decryptSecret).toHaveBeenCalledTimes(1) + expect(fetchMock).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/lib/oauth/credential-service.ts b/apps/sim/lib/oauth/credential-service.ts index bd68fa4f4cb..33181593f2d 100644 --- a/apps/sim/lib/oauth/credential-service.ts +++ b/apps/sim/lib/oauth/credential-service.ts @@ -205,14 +205,22 @@ export async function getServiceAccountToken( ): Promise { const [credentialRow] = await db .select({ + type: credential.type, + providerId: credential.providerId, + revokedAt: credential.revokedAt, encryptedServiceAccountKey: credential.encryptedServiceAccountKey, }) .from(credential) .where(eq(credential.id, credentialId)) .limit(1) - if (!credentialRow?.encryptedServiceAccountKey) { - throw new Error('Service account key not found') + if ( + credentialRow?.type !== 'service_account' || + credentialRow.providerId !== GOOGLE_SERVICE_ACCOUNT_PROVIDER_ID || + credentialRow.revokedAt !== null || + !credentialRow.encryptedServiceAccountKey + ) { + throw new Error('Google service account credential is unavailable') } const { decrypted } = await decryptSecret(credentialRow.encryptedServiceAccountKey) diff --git a/apps/sim/lib/selectors/server/providers/confluence.test.ts b/apps/sim/lib/selectors/server/providers/confluence.test.ts index 1f63418b4ab..a4437bb0b90 100644 --- a/apps/sim/lib/selectors/server/providers/confluence.test.ts +++ b/apps/sim/lib/selectors/server/providers/confluence.test.ts @@ -61,6 +61,7 @@ function spaceIdDetailArgs(): ExecuteServerSelectorArgs { describe('Confluence server selector adapters', () => { beforeEach(() => { vi.clearAllMocks() + mockFetch.mockReset() vi.stubGlobal('fetch', mockFetch) mockResolveCredentialBundle.mockResolvedValue({ accessToken: 'server-only-token' }) mockResolveCloudId.mockResolvedValue('cloud-1') @@ -156,13 +157,19 @@ describe('Confluence server selector adapters', () => { mockFetch .mockResolvedValueOnce( new Response( - JSON.stringify({ results: [{ id: '12345', key: 'ENG', name: 'Engineering' }] }), + JSON.stringify({ + results: [{ id: '12345', key: 'ENG', name: 'Engineering' }], + _links: { next: '/wiki/api/v2/spaces?cursor=next-page' }, + }), { status: 200 } ) ) .mockResolvedValueOnce( new Response( - JSON.stringify({ results: [{ id: '12345', key: 'ENG', name: 'Engineering' }] }), + JSON.stringify({ + results: [{ id: '12345', key: 'ENG', name: 'Engineering' }], + _links: { next: '/wiki/api/v2/spaces?cursor=next-page' }, + }), { status: 200 } ) ) @@ -179,6 +186,127 @@ describe('Confluence server selector adapters', () => { ).resolves.toMatchObject({ items: [{ id: 'ENG', label: 'Engineering (ENG)' }] }) }) + it.each(['confluence.spaces', 'confluence.spacesById'] as const)( + '%s finishes a short list without advertising an empty archived page', + async (selectorKey) => { + mockFetch + .mockResolvedValueOnce( + Response.json({ + results: [{ id: '12345', key: 'ENG', name: 'Engineering' }], + }) + ) + .mockResolvedValueOnce(Response.json({ results: [] })) + const controller = new AbortController() + + await expect( + confluenceSelectorAttachments[selectorKey].execute({ + ...spaceDetailArgs(controller.signal), + selectorKey, + request: { kind: 'list' }, + }) + ).resolves.toEqual({ + kind: 'list', + items: [ + { id: selectorKey === 'confluence.spaces' ? 'ENG' : '12345', label: 'Engineering (ENG)' }, + ], + }) + expect(mockFetch).toHaveBeenCalledTimes(2) + expect( + mockFetch.mock.calls.map(([url]) => new URL(String(url)).searchParams.get('status')) + ).toEqual(['current', 'archived']) + for (const [url, init] of mockFetch.mock.calls) { + expect(new URL(String(url)).searchParams.get('limit')).toBe('250') + expect(init.signal.aborted).toBe(false) + } + controller.abort() + for (const [, init] of mockFetch.mock.calls) expect(init.signal.aborted).toBe(true) + } + ) + + it('continues current spaces before fetching and paginating archived spaces', async () => { + mockFetch + .mockResolvedValueOnce( + Response.json({ + results: [{ id: '1', key: 'ENG', name: 'Engineering' }], + _links: { next: '/wiki/api/v2/spaces?cursor=current-next' }, + }) + ) + .mockResolvedValueOnce( + Response.json({ results: [{ id: '2', key: 'OPS', name: 'Operations' }] }) + ) + .mockResolvedValueOnce( + Response.json({ + results: [{ id: '3', key: 'OLD', name: 'Old team' }], + _links: { next: '/wiki/api/v2/spaces?cursor=archived-next' }, + }) + ) + .mockResolvedValueOnce( + Response.json({ results: [{ id: '4', key: 'LEGACY', name: 'Legacy' }] }) + ) + const args = spaceDetailArgs() + const execute = confluenceSelectorAttachments['confluence.spaces'].execute + + await expect(execute({ ...args, request: { kind: 'list' } })).resolves.toMatchObject({ + nextCursor: 'current:current-next', + }) + expect(mockFetch).toHaveBeenCalledTimes(1) + await expect( + execute({ ...args, request: { kind: 'list', cursor: 'current:current-next' } }) + ).resolves.toEqual({ + kind: 'list', + items: [ + { id: 'OPS', label: 'Operations (OPS)' }, + { id: 'OLD', label: 'Old team (OLD) — archived' }, + ], + nextCursor: 'archived:archived-next', + }) + expect(mockFetch).toHaveBeenCalledTimes(3) + await expect( + execute({ ...args, request: { kind: 'list', cursor: 'archived:archived-next' } }) + ).resolves.toEqual({ + kind: 'list', + items: [{ id: 'LEGACY', label: 'Legacy (LEGACY) — archived' }], + }) + expect( + mockFetch.mock.calls.map(([url]) => { + const params = new URL(String(url)).searchParams + return [params.get('status'), params.get('cursor')] + }) + ).toEqual([ + ['current', null], + ['current', 'current-next'], + ['archived', null], + ['archived', 'archived-next'], + ]) + }) + + it('includes archived spaces when there are no current spaces', async () => { + mockFetch + .mockResolvedValueOnce(Response.json({ results: [] })) + .mockResolvedValueOnce( + Response.json({ results: [{ id: '1', key: 'OLD', name: 'Old team' }] }) + ) + await expect( + confluenceSelectorAttachments['confluence.spaces'].execute({ + ...spaceDetailArgs(), + request: { kind: 'list' }, + }) + ).resolves.toEqual({ kind: 'list', items: [{ id: 'OLD', label: 'Old team (OLD) — archived' }] }) + expect(mockFetch).toHaveBeenCalledTimes(2) + }) + + it('preserves an archived-page failure rather than silently claiming the list is complete', async () => { + mockFetch + .mockResolvedValueOnce(Response.json({ results: [] })) + .mockResolvedValueOnce(new Response(null, { status: 403 })) + await expect( + confluenceSelectorAttachments['confluence.spaces'].execute({ + ...spaceDetailArgs(), + request: { kind: 'list' }, + }) + ).rejects.toMatchObject({ name: 'SelectorConnectionUnavailableError', status: 403 }) + }) + it('preserves the first safe provider failure when both space detail requests fail', async () => { mockFetch .mockResolvedValueOnce(new Response(null, { status: 401 })) diff --git a/apps/sim/lib/selectors/server/providers/confluence.ts b/apps/sim/lib/selectors/server/providers/confluence.ts index 4d876a25c68..d2ae95ada2a 100644 --- a/apps/sim/lib/selectors/server/providers/confluence.ts +++ b/apps/sim/lib/selectors/server/providers/confluence.ts @@ -67,6 +67,16 @@ function parseSpaceCursor(raw: string | undefined): { status: SpaceStatus; inner return { status, ...(inner ? { inner } : {}) } } +function nextSpaceCursor(data: ConfluenceSpacesResponse, status: SpaceStatus) { + if (!data._links?.next) return undefined + try { + const cursor = new URL(data._links.next, 'https://api.atlassian.com').searchParams.get('cursor') + return cursor ? `${status}:${cursor}` : undefined + } catch { + return undefined + } +} + function spaceOption( space: ConfluenceSpace, fallbackStatus: SpaceStatus, @@ -182,25 +192,23 @@ async function executeSpaces(args: ExecuteServerSelectorArgs, identifier: 'key' if (inner) params.set('cursor', inner) const data = await requestSpaces({ ...auth, params, signal: args.signal }) - let nextInner: string | undefined - if (data._links?.next) { - try { - nextInner = - new URL(data._links.next, 'https://api.atlassian.com').searchParams.get('cursor') || - undefined - } catch { - nextInner = undefined - } + const items = (data.results ?? []).map((space) => spaceOption(space, status, identifier)) + const nextCursor = nextSpaceCursor(data, status) + if (!nextCursor && status === 'current') { + const archived = await requestSpaces({ + ...auth, + params: new URLSearchParams({ limit: String(SPACE_PAGE_LIMIT), status: 'archived' }), + signal: args.signal, + }) + return listSelectorResult( + [ + ...items, + ...(archived.results ?? []).map((space) => spaceOption(space, 'archived', identifier)), + ], + nextSpaceCursor(archived, 'archived') + ) } - const nextCursor = nextInner - ? `${status}:${nextInner}` - : status === 'current' - ? 'archived:' - : undefined - return listSelectorResult( - (data.results ?? []).map((space) => spaceOption(space, status, identifier)), - nextCursor - ) + return listSelectorResult(items, nextCursor) } async function executePages(args: ExecuteServerSelectorArgs) { diff --git a/apps/sim/lib/selectors/server/providers/google.test.ts b/apps/sim/lib/selectors/server/providers/google.test.ts index 137fffab406..318a68b1136 100644 --- a/apps/sim/lib/selectors/server/providers/google.test.ts +++ b/apps/sim/lib/selectors/server/providers/google.test.ts @@ -105,7 +105,7 @@ describe('Google server selector adapters', () => { } ) - it('searches shared-drive roots before continuing to matching folders', async () => { + it('includes matching folders when the shared-drive search is complete', async () => { mockFetch .mockResolvedValueOnce( new Response(JSON.stringify({ drives: [{ id: 'drive-1', name: "Team's notes" }] })) @@ -119,14 +119,12 @@ describe('Google server selector adapters', () => { await expect(googleSelectorAttachments['google.drive'].execute(args)).resolves.toEqual({ kind: 'list', - items: [{ id: 'drive-1', label: "Team's notes" }], - nextCursor: 'f:', - }) - args.request = { ...args.request, cursor: 'f:' } - await expect(googleSelectorAttachments['google.drive'].execute(args)).resolves.toEqual({ - kind: 'list', - items: [{ id: 'folder-1', label: "Team's notes folder" }], + items: [ + { id: 'drive-1', label: "Team's notes" }, + { id: 'folder-1', label: "Team's notes folder" }, + ], }) + expect(mockFetch).toHaveBeenCalledTimes(2) const [driveUrl, fileUrl] = mockFetch.mock.calls.map(([url]) => new URL(String(url))) expect(driveUrl.pathname).toBe('/drive/v3/drives') @@ -135,6 +133,75 @@ describe('Google server selector adapters', () => { expect(fileUrl.searchParams.get('q')).toContain("name contains 'Team\\'s notes'") }) + it('does not offer another page when only a shared-drive root matches', async () => { + mockFetch + .mockResolvedValueOnce( + new Response(JSON.stringify({ drives: [{ id: 'drive-1', name: 'Engineering' }] })) + ) + .mockResolvedValueOnce(new Response(JSON.stringify({ files: [] }))) + const args = listArgs('google.drive') + args.context.mimeType = 'application/vnd.google-apps.folder' + args.request = { kind: 'list', search: 'Engineering' } + + await expect(googleSelectorAttachments['google.drive'].execute(args)).resolves.toEqual({ + kind: 'list', + items: [{ id: 'drive-1', label: 'Engineering' }], + }) + expect(mockFetch).toHaveBeenCalledTimes(2) + }) + + it('continues real folder pages after the final shared-drive page', async () => { + mockFetch + .mockResolvedValueOnce( + new Response(JSON.stringify({ drives: [{ id: 'drive-1', name: 'Engineering' }] })) + ) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ files: [{ id: 'folder-1', name: 'Notes' }], nextPageToken: 'next' }) + ) + ) + .mockResolvedValueOnce( + new Response(JSON.stringify({ files: [{ id: 'folder-2', name: 'Plans' }] })) + ) + const args = listArgs('google.drive', 'd:previous') + args.context.mimeType = 'application/vnd.google-apps.folder' + + await expect(googleSelectorAttachments['google.drive'].execute(args)).resolves.toEqual({ + kind: 'list', + items: [ + { id: 'drive-1', label: 'Engineering' }, + { id: 'folder-1', label: 'Notes' }, + ], + nextCursor: 'f:next', + }) + expect(mockFetch).toHaveBeenCalledTimes(2) + args.request = { kind: 'list', cursor: 'f:next' } + await expect(googleSelectorAttachments['google.drive'].execute(args)).resolves.toEqual({ + kind: 'list', + items: [{ id: 'folder-2', label: 'Plans' }], + }) + expect(mockFetch).toHaveBeenCalledTimes(3) + const urls = mockFetch.mock.calls.map(([url]) => new URL(String(url))) + expect(urls[0].searchParams.get('pageToken')).toBe('previous') + expect(urls[1].searchParams.has('pageToken')).toBe(false) + expect(urls[2].searchParams.get('pageToken')).toBe('next') + }) + + it('surfaces a folder-list failure after the final shared-drive page', async () => { + mockFetch + .mockResolvedValueOnce( + new Response(JSON.stringify({ drives: [{ id: 'drive-1', name: 'Engineering' }] })) + ) + .mockResolvedValueOnce(new Response('private provider error', { status: 503 })) + const args = listArgs('google.drive') + args.context.mimeType = 'application/vnd.google-apps.folder' + + await expect(googleSelectorAttachments['google.drive'].execute(args)).rejects.toMatchObject({ + status: 502, + }) + expect(mockFetch).toHaveBeenCalledTimes(2) + }) + it('continues a shared-drive search on demand', async () => { mockFetch.mockResolvedValueOnce( new Response( @@ -153,6 +220,7 @@ describe('Google server selector adapters', () => { const url = new URL(String(mockFetch.mock.calls[0]?.[0])) expect(url.searchParams.get('pageToken')).toBe('previous') expect(url.searchParams.get('q')).toBe("name contains 'Engineering'") + expect(mockFetch).toHaveBeenCalledTimes(1) }) it('still lists personal folders when the account has no shared drives', async () => { diff --git a/apps/sim/lib/selectors/server/providers/google.ts b/apps/sim/lib/selectors/server/providers/google.ts index 00f5040eae9..8784433ffaa 100644 --- a/apps/sim/lib/selectors/server/providers/google.ts +++ b/apps/sim/lib/selectors/server/providers/google.ts @@ -253,8 +253,8 @@ function parseDriveCursor(cursor: string | undefined): DriveCursor | undefined { return { source: source === 'd:' ? 'drives' : 'files', pageToken } } -function driveCursor(source: DriveCursor['source'], pageToken?: string): string { - return `${source === 'drives' ? 'd' : 'f'}:${pageToken ?? ''}` +function driveCursor(source: DriveCursor['source'], pageToken: string): string { + return `${source === 'drives' ? 'd' : 'f'}:${pageToken}` } async function listDriveFiles( @@ -278,16 +278,16 @@ async function listDriveFiles( throw new SelectorContextUnavailableError() } + let sharedDrives: DriveFile[] = [] if (includeSharedDrives && (!cursor || cursor.source === 'drives')) { const drives = await fetchSharedDrivePage(accessToken, cursor?.pageToken, search, args.signal) - if (drives.items.length > 0 || drives.nextCursor) { + if (drives.nextCursor) { return { items: drives.items, - nextCursor: drives.nextCursor - ? driveCursor('drives', drives.nextCursor) - : driveCursor('files'), + nextCursor: driveCursor('drives', drives.nextCursor), } } + sharedDrives = drives.items } const pageToken = cursor?.source === 'files' ? cursor.pageToken : undefined @@ -306,7 +306,7 @@ async function listDriveFiles( const nextPageToken = data.nextPageToken?.trim() return { - items: data.files ?? [], + items: [...sharedDrives, ...(data.files ?? [])], ...(nextPageToken ? { nextCursor: driveCursor('files', nextPageToken) } : {}), } } diff --git a/apps/sim/lib/sim-search/source-identity.test.ts b/apps/sim/lib/sim-search/source-identity.test.ts index a7f439fb1be..67208256a0d 100644 --- a/apps/sim/lib/sim-search/source-identity.test.ts +++ b/apps/sim/lib/sim-search/source-identity.test.ts @@ -10,6 +10,7 @@ import { } from '@/lib/sim-search/source-identity' import { confluenceConnectorMeta } from '@/connectors/confluence/meta' import { gitlabConnectorMeta } from '@/connectors/gitlab/meta' +import { googleCalendarConnectorMeta } from '@/connectors/google-calendar/meta' import { googleDriveConnectorMeta } from '@/connectors/google-drive/meta' describe('Search source identity', () => { @@ -99,6 +100,45 @@ describe('Search source identity', () => { ) }) + it.each([ + ['primary', '1 calendar selected'], + ['team@group.calendar.google.com', '1 calendar selected'], + ['primary, team@group.calendar.google.com, primary', '2 calendars selected'], + [ + ['first@group.calendar.google.com', 'second@group.calendar.google.com'], + '2 calendars selected', + ], + ['', ''], + [[], ''], + ])('describes manual Calendar selections without displaying raw IDs: %j', (calendarId, title) => { + expect(describeSearchSource(googleCalendarConnectorMeta, { calendarId })).toBe(title) + }) + + it('uses saved calendar names without changing identity and drops them after a selection change', () => { + const config = { calendarId: ['primary', 'team@group.calendar.google.com'] } + const labeledConfig = { + ...config, + [SOURCE_LABELS_KEY]: createSourceLabelMetadata(googleCalendarConnectorMeta, config, { + calendarId: [ + { id: 'primary', label: 'My calendar' }, + { id: 'team@group.calendar.google.com', label: 'Engineering' }, + ], + }), + } + expect(describeSearchSource(googleCalendarConnectorMeta, labeledConfig)).toBe( + 'My calendar · Engineering' + ) + expect(searchSourceIdentity(googleCalendarConnectorMeta, labeledConfig)).toBe( + searchSourceIdentity(googleCalendarConnectorMeta, config) + ) + expect( + describeSearchSource(googleCalendarConnectorMeta, { + ...labeledConfig, + calendarId: ['other@group.calendar.google.com'], + }) + ).toBe('1 calendar selected') + }) + it('drops saved labels when selections or source settings change', () => { const config = { folderId: ['folder-a'], fileType: 'documents' } const labeledConfig = { diff --git a/apps/sim/lib/sim-search/source-identity.ts b/apps/sim/lib/sim-search/source-identity.ts index cfdd475f57b..5f6875948e8 100644 --- a/apps/sim/lib/sim-search/source-identity.ts +++ b/apps/sim/lib/sim-search/source-identity.ts @@ -36,6 +36,7 @@ const SOURCE_ADDRESS_FIELDS = new Set([ 'projectKey', 'spaceKey', 'folderId', + 'calendarId', 'channel', 'channelIds', 'teamId', @@ -65,6 +66,7 @@ interface SourceLabelMetadata { const OPAQUE_SOURCE_FIELDS = new Map([ ['folderId', 'folder'], + ['calendarId', 'calendar'], ['channel', 'channel'], ['channelIds', 'channel'], ['teamId', 'team'],