chore: import upstream snapshot with attribution

apps/sim/connectors/google-docs/google-docs.ts

@@ -0,0 +1,382 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
+import { googleDocsConnectorMeta } from '@/connectors/google-docs/meta'
+import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
+import {
+  buildDriveParentsClause,
+  joinTagArray,
+  parseMultiValue,
+  parseTagDate,
+} from '@/connectors/utils'
+
+const logger = createLogger('GoogleDocsConnector')
+
+/**
+ * Represents a Google Drive file entry returned by the Drive API.
+ */
+interface DriveFile {
+  id: string
+  name: string
+  mimeType: string
+  modifiedTime?: string
+  createdTime?: string
+  webViewLink?: string
+  owners?: { displayName?: string; emailAddress?: string }[]
+}
+
+/**
+ * Represents a structural element within a Google Docs document body.
+ */
+interface DocsStructuralElement {
+  paragraph?: {
+    paragraphStyle?: {
+      namedStyleType?: string
+    }
+    elements?: {
+      textRun?: {
+        content?: string
+      }
+    }[]
+  }
+}
+
+/**
+ * Represents the response from the Google Docs API for a single document.
+ */
+interface DocsDocument {
+  documentId: string
+  title: string
+  body?: {
+    content?: DocsStructuralElement[]
+  }
+}
+
+/**
+ * Maps a Google Docs heading style to a Markdown heading prefix.
+ */
+function headingPrefix(namedStyleType?: string): string {
+  switch (namedStyleType) {
+    case 'HEADING_1':
+      return '# '
+    case 'HEADING_2':
+      return '## '
+    case 'HEADING_3':
+      return '### '
+    case 'HEADING_4':
+      return '#### '
+    case 'HEADING_5':
+      return '##### '
+    case 'HEADING_6':
+      return '###### '
+    default:
+      return ''
+  }
+}
+
+/**
+ * Extracts plain text from a Google Docs API structured document response.
+ * Headings are prefixed with Markdown-style `#` markers.
+ */
+function extractTextFromDocsBody(doc: DocsDocument): string {
+  const elements = doc.body?.content
+  if (!elements) return ''
+
+  const parts: string[] = []
+
+  for (const element of elements) {
+    const paragraph = element.paragraph
+    if (!paragraph?.elements) continue
+
+    const prefix = headingPrefix(paragraph.paragraphStyle?.namedStyleType)
+    /**
+     * Each paragraph's final `textRun.content` already ends with `\n`. Strip
+     * it before joining with `\n` so a heading followed by a body paragraph
+     * is separated by a single newline, not two.
+     */
+    const text = paragraph.elements
+      .map((el) => el.textRun?.content ?? '')
+      .join('')
+      .replace(/\n+$/, '')
+
+    if (text.trim()) {
+      parts.push(`${prefix}${text}`)
+    }
+  }
+
+  return parts.join('\n').trim()
+}
+
+/**
+ * Fetches the structured content of a Google Doc via the Docs API and
+ * extracts it as plain text.
+ */
+async function fetchDocContent(accessToken: string, documentId: string): Promise<string> {
+  const url = `https://docs.googleapis.com/v1/documents/${documentId}?fields=body.content`
+
+  const response = await fetchWithRetry(url, {
+    method: 'GET',
+    headers: {
+      Authorization: `Bearer ${accessToken}`,
+      Accept: 'application/json',
+    },
+  })
+
+  if (!response.ok) {
+    throw new Error(`Failed to fetch Google Doc content ${documentId}: ${response.status}`)
+  }
+
+  const doc = (await response.json()) as DocsDocument
+  return extractTextFromDocsBody(doc)
+}
+
+/**
+ * Creates a lightweight stub from a Drive file entry. Content is deferred
+ * and only fetched via getDocument for new or changed documents.
+ */
+function fileToStub(file: DriveFile): ExternalDocument {
+  return {
+    externalId: file.id,
+    title: file.name || 'Untitled',
+    content: '',
+    contentDeferred: true,
+    mimeType: 'text/plain',
+    sourceUrl: file.webViewLink || `https://docs.google.com/document/d/${file.id}/edit`,
+    contentHash: `gdocs:${file.id}:${file.modifiedTime ?? ''}`,
+    metadata: {
+      modifiedTime: file.modifiedTime,
+      createdTime: file.createdTime,
+      owners: file.owners?.map((o) => o.displayName || o.emailAddress).filter(Boolean),
+    },
+  }
+}
+
+/**
+ * Builds the Drive API query string for listing Google Docs.
+ */
+function buildQuery(sourceConfig: Record<string, unknown>): string {
+  const parts: string[] = ['trashed = false', "mimeType = 'application/vnd.google-apps.document'"]
+
+  const parentsClause = buildDriveParentsClause(parseMultiValue(sourceConfig.folderId))
+  if (parentsClause) parts.push(parentsClause)
+
+  return parts.join(' and ')
+}
+
+export const googleDocsConnector: ConnectorConfig = {
+  ...googleDocsConnectorMeta,
+
+  listDocuments: async (
+    accessToken: string,
+    sourceConfig: Record<string, unknown>,
+    cursor?: string,
+    syncContext?: Record<string, unknown>
+  ): Promise<ExternalDocumentList> => {
+    const query = buildQuery(sourceConfig)
+    const pageSize = 100
+
+    const queryParams = new URLSearchParams({
+      q: query,
+      pageSize: String(pageSize),
+      fields: 'nextPageToken,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners)',
+      supportsAllDrives: 'true',
+      includeItemsFromAllDrives: 'true',
+    })
+
+    if (cursor) {
+      queryParams.set('pageToken', cursor)
+    }
+
+    const url = `https://www.googleapis.com/drive/v3/files?${queryParams.toString()}`
+
+    logger.info('Listing Google Docs', { query, cursor: cursor ?? 'initial' })
+
+    const response = await fetchWithRetry(url, {
+      method: 'GET',
+      headers: {
+        Authorization: `Bearer ${accessToken}`,
+        Accept: 'application/json',
+      },
+    })
+
+    if (!response.ok) {
+      const errorText = await response.text()
+      logger.error('Failed to list Google Docs', {
+        status: response.status,
+        error: errorText,
+      })
+      throw new Error(`Failed to list Google Docs: ${response.status}`)
+    }
+
+    const data = await response.json()
+    const files = (data.files || []) as DriveFile[]
+
+    /**
+     * Drive sets `incompleteSearch` when it could not search every corpus (it
+     * arises with the `allDrives` scope enabled by `includeItemsFromAllDrives`).
+     * A partial listing drops still-existing docs, so reconciliation must be
+     * suppressed to avoid hard-deleting valid documents.
+     */
+    const incompleteSearch = data.incompleteSearch === true
+
+    const maxDocs = sourceConfig.maxDocs ? Number(sourceConfig.maxDocs) : 0
+    const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
+
+    let documents = files.map(fileToStub)
+    let slicedSome = false
+    if (maxDocs > 0) {
+      const remaining = maxDocs - previouslyFetched
+      if (documents.length > remaining) {
+        slicedSome = true
+        documents = documents.slice(0, remaining)
+      }
+    }
+
+    const totalFetched = previouslyFetched + documents.length
+    if (syncContext) syncContext.totalDocsFetched = totalFetched
+    const hitLimit = maxDocs > 0 && totalFetched >= maxDocs
+
+    const nextPageToken = data.nextPageToken as string | undefined
+
+    /**
+     * Mark the listing as incomplete so the sync engine skips deletion
+     * reconciliation when this page does not represent the full source set:
+     * - `slicedSome`: the page held more docs than the `maxDocs` cap allowed.
+     * - `hitLimit` with a next page: the cap was reached while more pages remain.
+     * - `incompleteSearch`: Drive could not search every corpus, so the page is
+     *   partial and may omit still-existing docs.
+     * Reconciliation against any of these would hard-delete valid documents.
+     */
+    if (syncContext && (slicedSome || (hitLimit && Boolean(nextPageToken)) || incompleteSearch)) {
+      syncContext.listingCapped = true
+    }
+
+    return {
+      documents,
+      nextCursor: hitLimit ? undefined : nextPageToken,
+      hasMore: hitLimit ? false : Boolean(nextPageToken),
+    }
+  },
+
+  getDocument: async (
+    accessToken: string,
+    _sourceConfig: Record<string, unknown>,
+    externalId: string
+  ): Promise<ExternalDocument | null> => {
+    const fields = 'id,name,mimeType,modifiedTime,createdTime,webViewLink,owners,trashed'
+    const url = `https://www.googleapis.com/drive/v3/files/${externalId}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true`
+
+    const response = await fetchWithRetry(url, {
+      method: 'GET',
+      headers: {
+        Authorization: `Bearer ${accessToken}`,
+        Accept: 'application/json',
+      },
+    })
+
+    if (!response.ok) {
+      if (response.status === 404) return null
+      throw new Error(`Failed to get Google Doc metadata: ${response.status}`)
+    }
+
+    const file = (await response.json()) as DriveFile & { trashed?: boolean }
+
+    if (file.trashed) return null
+    if (file.mimeType !== 'application/vnd.google-apps.document') return null
+
+    try {
+      const content = await fetchDocContent(accessToken, file.id)
+      if (!content.trim()) return null
+
+      return { ...fileToStub(file), content, contentDeferred: false }
+    } catch (error) {
+      logger.warn(`Failed to extract content from document: ${file.name} (${file.id})`, {
+        error: toError(error).message,
+      })
+      return null
+    }
+  },
+
+  validateConfig: async (
+    accessToken: string,
+    sourceConfig: Record<string, unknown>
+  ): Promise<{ valid: boolean; error?: string }> => {
+    const folderIds = parseMultiValue(sourceConfig.folderId)
+    const maxDocs = sourceConfig.maxDocs as string | undefined
+
+    if (maxDocs && (Number.isNaN(Number(maxDocs)) || Number(maxDocs) <= 0)) {
+      return { valid: false, error: 'Max documents must be a positive number' }
+    }
+
+    try {
+      if (folderIds.length > 0) {
+        for (const folderId of folderIds) {
+          const url = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(folderId)}?fields=id,name,mimeType&supportsAllDrives=true`
+          const response = await fetchWithRetry(
+            url,
+            {
+              method: 'GET',
+              headers: {
+                Authorization: `Bearer ${accessToken}`,
+                Accept: 'application/json',
+              },
+            },
+            VALIDATE_RETRY_OPTIONS
+          )
+
+          if (!response.ok) {
+            if (response.status === 404) {
+              return {
+                valid: false,
+                error: `Folder "${folderId}" not found. Check the folder ID and permissions.`,
+              }
+            }
+            return {
+              valid: false,
+              error: `Failed to access folder "${folderId}": ${response.status}`,
+            }
+          }
+
+          const folder = await response.json()
+          if (folder.mimeType !== 'application/vnd.google-apps.folder') {
+            return { valid: false, error: `"${folderId}" is not a folder` }
+          }
+        }
+      } else {
+        const url =
+          "https://www.googleapis.com/drive/v3/files?pageSize=1&q=mimeType%3D'application%2Fvnd.google-apps.document'&fields=files(id)"
+        const response = await fetchWithRetry(
+          url,
+          {
+            method: 'GET',
+            headers: {
+              Authorization: `Bearer ${accessToken}`,
+              Accept: 'application/json',
+            },
+          },
+          VALIDATE_RETRY_OPTIONS
+        )
+
+        if (!response.ok) {
+          return { valid: false, error: `Failed to access Google Docs: ${response.status}` }
+        }
+      }
+
+      return { valid: true }
+    } catch (error) {
+      return { valid: false, error: toError(error).message || 'Failed to validate configuration' }
+    }
+  },
+
+  mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
+    const result: Record<string, unknown> = {}
+
+    const owners = joinTagArray(metadata.owners)
+    if (owners) result.owners = owners
+
+    const lastModified = parseTagDate(metadata.modifiedTime)
+    if (lastModified) result.lastModified = lastModified
+
+    return result
+  },
+}

apps/sim/connectors/google-docs/index.ts

@@ -0,0 +1 @@
+export { googleDocsConnector } from '@/connectors/google-docs/google-docs'

apps/sim/connectors/google-docs/meta.ts

@@ -0,0 +1,53 @@
+import { GoogleDocsIcon } from '@/components/icons'
+import type { ConnectorMeta } from '@/connectors/types'
+
+export const googleDocsConnectorMeta: ConnectorMeta = {
+  id: 'google_docs',
+  name: 'Google Docs',
+  description: 'Sync Google Docs documents',
+  version: '1.0.0',
+  icon: GoogleDocsIcon,
+
+  auth: {
+    mode: 'oauth',
+    provider: 'google-docs',
+    requiredScopes: ['https://www.googleapis.com/auth/drive'],
+  },
+
+  configFields: [
+    {
+      id: 'folderSelector',
+      title: 'Folders',
+      type: 'selector',
+      selectorKey: 'google.drive',
+      mimeType: 'application/vnd.google-apps.folder',
+      canonicalParamId: 'folderId',
+      mode: 'basic',
+      multi: true,
+      placeholder: 'Select one or more folders (optional)',
+      required: false,
+    },
+    {
+      id: 'folderId',
+      title: 'Folder IDs',
+      type: 'short-input',
+      canonicalParamId: 'folderId',
+      mode: 'advanced',
+      multi: true,
+      placeholder: 'e.g. 1aBcDeFg…, 2cDeFgHi… (comma-separated for multiple)',
+      required: false,
+    },
+    {
+      id: 'maxDocs',
+      title: 'Max Documents',
+      type: 'short-input',
+      required: false,
+      placeholder: 'e.g. 500 (default: unlimited)',
+    },
+  ],
+
+  tagDefinitions: [
+    { id: 'owners', displayName: 'Owner', fieldType: 'text' },
+    { id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
+  ],
+}