import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { googleDriveConnectorMeta } from '@/connectors/google-drive/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { buildDriveParentsClause, CONNECTOR_MAX_FILE_BYTES, ConnectorFileTooLargeError, htmlToPlainText, joinTagArray, markSkipped, parseMultiValue, parseTagDate, readBodyWithLimit, sizeLimitSkipReason, } from '@/connectors/utils' const logger = createLogger('GoogleDriveConnector') const GOOGLE_WORKSPACE_MIME_TYPES: Record = { 'application/vnd.google-apps.document': 'text/plain', 'application/vnd.google-apps.spreadsheet': 'text/csv', 'application/vnd.google-apps.presentation': 'text/plain', } const SUPPORTED_TEXT_MIME_TYPES = [ 'text/plain', 'text/csv', 'text/html', 'text/markdown', 'application/json', 'application/xml', ] // Google Drive's `files.export` API rejects exports over 10 MB (exportSizeLimitExceeded), // so this is a hard external limit for Google Workspace docs — not the connector cap. const MAX_EXPORT_SIZE = 10 * 1024 * 1024 function isGoogleWorkspaceFile(mimeType: string): boolean { return mimeType in GOOGLE_WORKSPACE_MIME_TYPES } function isSupportedTextFile(mimeType: string): boolean { return SUPPORTED_TEXT_MIME_TYPES.some((t) => mimeType.startsWith(t)) } async function exportGoogleWorkspaceFile( accessToken: string, fileId: string, sourceMimeType: string ): Promise { const exportMimeType = GOOGLE_WORKSPACE_MIME_TYPES[sourceMimeType] if (!exportMimeType) { throw new Error(`Unsupported Google Workspace MIME type: ${sourceMimeType}`) } const url = `https://www.googleapis.com/drive/v3/files/${fileId}/export?mimeType=${encodeURIComponent(exportMimeType)}` const response = await fetchWithRetry(url, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}` }, }) if (!response.ok) { // Google rejects exports over its 10 MB limit with a 403 exportSizeLimitExceeded // before streaming any bytes — surface that as an oversize skip, not a hard error. if (response.status === 403) { const body = await response.text().catch(() => '') if (body.includes('exportSizeLimitExceeded')) { throw new ConnectorFileTooLargeError(MAX_EXPORT_SIZE) } } throw new Error(`Failed to export file ${fileId}: ${response.status}`) } const buffer = await readBodyWithLimit(response, MAX_EXPORT_SIZE) if (!buffer) { throw new ConnectorFileTooLargeError(MAX_EXPORT_SIZE) } return buffer.toString('utf8') } async function downloadTextFile(accessToken: string, fileId: string): Promise { const url = `https://www.googleapis.com/drive/v3/files/${fileId}?alt=media` const response = await fetchWithRetry(url, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}` }, }) if (!response.ok) { throw new Error(`Failed to download file ${fileId}: ${response.status}`) } // Stream with a hard byte cap so a file with missing/under-reported listing // size metadata is never fully buffered into memory. Oversized files raise // DriveFileTooLargeError so getDocument can surface them as skipped (failed) rows. const buffer = await readBodyWithLimit(response, CONNECTOR_MAX_FILE_BYTES) if (!buffer) { throw new ConnectorFileTooLargeError(CONNECTOR_MAX_FILE_BYTES) } return buffer.toString('utf8') } async function fetchFileContent( accessToken: string, fileId: string, mimeType: string ): Promise { if (isGoogleWorkspaceFile(mimeType)) { return exportGoogleWorkspaceFile(accessToken, fileId, mimeType) } if (mimeType === 'text/html') { const html = await downloadTextFile(accessToken, fileId) return htmlToPlainText(html) } if (isSupportedTextFile(mimeType)) { return downloadTextFile(accessToken, fileId) } throw new Error(`Unsupported MIME type for content extraction: ${mimeType}`) } interface DriveFile { id: string name: string mimeType: string modifiedTime?: string createdTime?: string webViewLink?: string parents?: string[] owners?: { displayName?: string; emailAddress?: string }[] size?: string starred?: boolean trashed?: boolean } function buildQuery(sourceConfig: Record): string { const parts: string[] = ['trashed = false'] const parentsClause = buildDriveParentsClause(parseMultiValue(sourceConfig.folderId)) if (parentsClause) parts.push(parentsClause) const fileType = (sourceConfig.fileType as string) || 'all' switch (fileType) { case 'documents': parts.push("mimeType = 'application/vnd.google-apps.document'") break case 'spreadsheets': parts.push("mimeType = 'application/vnd.google-apps.spreadsheet'") break case 'presentations': parts.push("mimeType = 'application/vnd.google-apps.presentation'") break case 'text': parts.push(`(${SUPPORTED_TEXT_MIME_TYPES.map((t) => `mimeType = '${t}'`).join(' or ')})`) break default: { // Include Google Workspace files + plain text files, exclude folders const allMimeTypes = [ ...Object.keys(GOOGLE_WORKSPACE_MIME_TYPES), ...SUPPORTED_TEXT_MIME_TYPES, ] parts.push(`(${allMimeTypes.map((t) => `mimeType = '${t}'`).join(' or ')})`) break } } return parts.join(' and ') } function fileToStub(file: DriveFile): ExternalDocument { return { externalId: file.id, title: file.name || 'Untitled', content: '', contentDeferred: true, mimeType: 'text/plain', sourceUrl: file.webViewLink || `https://drive.google.com/file/d/${file.id}/view`, contentHash: `gdrive:${file.id}:${file.modifiedTime ?? ''}`, metadata: { originalMimeType: file.mimeType, modifiedTime: file.modifiedTime, createdTime: file.createdTime, owners: file.owners?.map((o) => o.displayName || o.emailAddress).filter(Boolean), starred: file.starred, fileSize: file.size ? Number(file.size) : undefined, }, } } export const googleDriveConnector: ConnectorConfig = { ...googleDriveConnectorMeta, listDocuments: async ( accessToken: string, sourceConfig: Record, cursor?: string, syncContext?: Record ): Promise => { const query = buildQuery(sourceConfig) const pageSize = 100 const maxFiles = sourceConfig.maxFiles ? Number(sourceConfig.maxFiles) : 0 const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0 if (maxFiles > 0 && previouslyFetched >= maxFiles) { return { documents: [], hasMore: false } } const remaining = maxFiles > 0 ? maxFiles - previouslyFetched : 0 const effectivePageSize = maxFiles > 0 ? Math.min(pageSize, remaining) : pageSize const queryParams = new URLSearchParams({ q: query, pageSize: String(effectivePageSize), orderBy: 'modifiedTime desc', fields: 'nextPageToken,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,parents,owners,size,starred)', supportsAllDrives: 'true', includeItemsFromAllDrives: 'true', }) if (cursor) { queryParams.set('pageToken', cursor) } const url = `https://www.googleapis.com/drive/v3/files?${queryParams.toString()}` logger.info('Listing Google Drive files', { query, cursor: cursor ?? 'initial' }) const response = await fetchWithRetry(url, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json', }, }) if (!response.ok) { const errorText = await response.text() logger.error('Failed to list Google Drive files', { status: response.status, error: errorText, }) throw new Error(`Failed to list Google Drive files: ${response.status}`) } const data = await response.json() const files = (data.files || []) as DriveFile[] /** * Drive sets `incompleteSearch` when it could not search every corpus (it * arises with the `allDrives` scope enabled by `includeItemsFromAllDrives`). * A partial listing drops still-existing files, so reconciliation must be * suppressed to avoid hard-deleting valid documents. */ const incompleteSearch = data.incompleteSearch === true const documents = files .filter((f) => isGoogleWorkspaceFile(f.mimeType) || isSupportedTextFile(f.mimeType)) .map(fileToStub) const totalFetched = previouslyFetched + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched const hitLimit = maxFiles > 0 && totalFetched >= maxFiles if (syncContext && (hitLimit || incompleteSearch)) syncContext.listingCapped = true const nextPageToken = data.nextPageToken as string | undefined return { documents, nextCursor: hitLimit ? undefined : nextPageToken, hasMore: hitLimit ? false : Boolean(nextPageToken), } }, getDocument: async ( accessToken: string, sourceConfig: Record, externalId: string ): Promise => { const fields = 'id,name,mimeType,modifiedTime,createdTime,webViewLink,parents,owners,size,starred,trashed' const url = `https://www.googleapis.com/drive/v3/files/${externalId}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true` const response = await fetchWithRetry(url, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json', }, }) if (!response.ok) { if (response.status === 404) return null throw new Error(`Failed to get Google Drive file: ${response.status}`) } const file = (await response.json()) as DriveFile if (file.trashed) return null try { const content = await fetchFileContent(accessToken, file.id, file.mimeType) if (!content.trim()) return null const stub = fileToStub(file) return { ...stub, content, contentDeferred: false } } catch (error) { if (error instanceof ConnectorFileTooLargeError) { logger.info('Skipping oversized Google Drive file', { fileId: file.id, name: file.name }) return markSkipped(fileToStub(file), sizeLimitSkipReason(error.limitBytes)) } logger.warn(`Failed to fetch content for file: ${file.name} (${file.id})`, { error: toError(error).message, }) return null } }, validateConfig: async ( accessToken: string, sourceConfig: Record ): Promise<{ valid: boolean; error?: string }> => { const folderIds = parseMultiValue(sourceConfig.folderId) const maxFiles = sourceConfig.maxFiles as string | undefined if (maxFiles && (Number.isNaN(Number(maxFiles)) || Number(maxFiles) <= 0)) { return { valid: false, error: 'Max files must be a positive number' } } // Verify access to Drive API try { if (folderIds.length > 0) { // Verify each folder exists, is accessible, and is actually a folder for (const folderId of folderIds) { const url = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(folderId)}?fields=id,name,mimeType&supportsAllDrives=true` const response = await fetchWithRetry( url, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json', }, }, VALIDATE_RETRY_OPTIONS ) if (!response.ok) { if (response.status === 404) { return { valid: false, error: `Folder "${folderId}" not found. Check the folder ID and permissions.`, } } return { valid: false, error: `Failed to access folder "${folderId}": ${response.status}`, } } const folder = await response.json() if (folder.mimeType !== 'application/vnd.google-apps.folder') { return { valid: false, error: `"${folderId}" is not a folder` } } } } else { // Verify basic Drive access by listing one file const url = 'https://www.googleapis.com/drive/v3/files?pageSize=1&fields=files(id)' const response = await fetchWithRetry( url, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json', }, }, VALIDATE_RETRY_OPTIONS ) if (!response.ok) { return { valid: false, error: `Failed to access Google Drive: ${response.status}` } } } return { valid: true } } catch (error) { const message = getErrorMessage(error, 'Failed to validate configuration') return { valid: false, error: message } } }, mapTags: (metadata: Record): Record => { const result: Record = {} const owners = joinTagArray(metadata.owners) if (owners) result.owners = owners if (typeof metadata.originalMimeType === 'string') { const mimeType = metadata.originalMimeType if (mimeType.includes('document')) result.fileType = 'Google Doc' else if (mimeType.includes('spreadsheet')) result.fileType = 'Google Sheet' else if (mimeType.includes('presentation')) result.fileType = 'Google Slides' else if (mimeType.startsWith('text/')) result.fileType = 'Text File' else result.fileType = mimeType } const lastModified = parseTagDate(metadata.modifiedTime) if (lastModified) result.lastModified = lastModified if (typeof metadata.starred === 'boolean') { result.starred = metadata.starred } return result }, }