chore: import upstream snapshot with attribution

apps/sim/connectors/dropbox/dropbox.ts

@@ -0,0 +1,327 @@
+import { createLogger } from '@sim/logger'
+import { getErrorMessage, toError } from '@sim/utils/errors'
+import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
+import { dropboxConnectorMeta } from '@/connectors/dropbox/meta'
+import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
+import {
+  CONNECTOR_MAX_FILE_BYTES,
+  ConnectorFileTooLargeError,
+  htmlToPlainText,
+  isSkippedDocument,
+  markSkipped,
+  parseTagDate,
+  readBodyWithLimit,
+  sizeLimitSkipReason,
+  stubOrSkipBySize,
+  takeIndexableWithinCap,
+} from '@/connectors/utils'
+
+const logger = createLogger('DropboxConnector')
+
+const SUPPORTED_EXTENSIONS = new Set([
+  '.txt',
+  '.md',
+  '.markdown',
+  '.html',
+  '.htm',
+  '.csv',
+  '.json',
+  '.xml',
+  '.yaml',
+  '.yml',
+  '.log',
+  '.rst',
+  '.tsv',
+])
+
+const MAX_FILE_SIZE = CONNECTOR_MAX_FILE_BYTES
+
+interface DropboxFileEntry {
+  '.tag': 'file' | 'folder' | 'deleted'
+  id: string
+  name: string
+  path_lower: string
+  path_display: string
+  client_modified?: string
+  server_modified?: string
+  size?: number
+  content_hash?: string
+  is_downloadable?: boolean
+}
+
+interface DropboxListFolderResponse {
+  entries: DropboxFileEntry[]
+  cursor: string
+  has_more: boolean
+}
+
+function hasSupportedExtension(name: string): boolean {
+  const lower = name.toLowerCase()
+  const dotIndex = lower.lastIndexOf('.')
+  if (dotIndex === -1) return false
+  return SUPPORTED_EXTENSIONS.has(lower.slice(dotIndex))
+}
+
+/** A downloadable file with a supported extension, regardless of size. */
+function isDownloadableFile(entry: DropboxFileEntry): boolean {
+  return (
+    entry['.tag'] === 'file' && entry.is_downloadable !== false && hasSupportedExtension(entry.name)
+  )
+}
+
+async function downloadFileContent(accessToken: string, filePath: string): Promise<string> {
+  const response = await fetchWithRetry('https://content.dropboxapi.com/2/files/download', {
+    method: 'POST',
+    headers: {
+      Authorization: `Bearer ${accessToken}`,
+      'Dropbox-API-Arg': JSON.stringify({ path: filePath }),
+    },
+  })
+
+  if (!response.ok) {
+    throw new Error(`Failed to download file ${filePath}: ${response.status}`)
+  }
+
+  // Stream with a hard byte cap so a file whose listing metadata under-reported
+  // (or omitted) its size can never be fully buffered into memory. Oversize raises
+  // so getDocument can surface it as a skipped (failed) row rather than dropping it.
+  const buffer = await readBodyWithLimit(response, MAX_FILE_SIZE)
+  if (!buffer) {
+    throw new ConnectorFileTooLargeError(MAX_FILE_SIZE)
+  }
+
+  const text = buffer.toString('utf8')
+
+  if (filePath.endsWith('.html') || filePath.endsWith('.htm')) {
+    return htmlToPlainText(text)
+  }
+
+  return text
+}
+
+function fileToStub(entry: DropboxFileEntry): ExternalDocument {
+  return {
+    externalId: entry.id,
+    title: entry.name,
+    content: '',
+    contentDeferred: true,
+    mimeType: 'text/plain',
+    sourceUrl: `https://www.dropbox.com/home${entry.path_display}`,
+    contentHash: `dropbox:${entry.id}:${entry.content_hash ?? entry.server_modified ?? ''}`,
+    metadata: {
+      path: entry.path_display,
+      lastModified: entry.server_modified || entry.client_modified,
+      fileSize: entry.size,
+    },
+  }
+}
+
+export const dropboxConnector: ConnectorConfig = {
+  ...dropboxConnectorMeta,
+
+  listDocuments: async (
+    accessToken: string,
+    sourceConfig: Record<string, unknown>,
+    cursor?: string,
+    syncContext?: Record<string, unknown>
+  ): Promise<ExternalDocumentList> => {
+    let data: DropboxListFolderResponse
+
+    if (cursor) {
+      const response = await fetchWithRetry(
+        'https://api.dropboxapi.com/2/files/list_folder/continue',
+        {
+          method: 'POST',
+          headers: {
+            Authorization: `Bearer ${accessToken}`,
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify({ cursor }),
+        }
+      )
+
+      if (!response.ok) {
+        const errorText = await response.text()
+        logger.error('Failed to continue listing Dropbox folder', {
+          status: response.status,
+          error: errorText,
+        })
+        throw new Error(`Failed to continue listing Dropbox folder: ${response.status}`)
+      }
+
+      data = await response.json()
+    } else {
+      const folderPath = (sourceConfig.folderPath as string)?.trim() || ''
+      const path = folderPath.startsWith('/') ? folderPath : folderPath ? `/${folderPath}` : ''
+
+      logger.info('Listing Dropbox folder', { path: path || '(root)' })
+
+      const response = await fetchWithRetry('https://api.dropboxapi.com/2/files/list_folder', {
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${accessToken}`,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          path,
+          recursive: true,
+          include_deleted: false,
+          include_non_downloadable_files: false,
+          limit: 2000,
+        }),
+      })
+
+      if (!response.ok) {
+        const errorText = await response.text()
+        logger.error('Failed to list Dropbox folder', {
+          status: response.status,
+          error: errorText,
+        })
+        throw new Error(`Failed to list Dropbox folder: ${response.status}`)
+      }
+
+      data = await response.json()
+    }
+
+    // Keep oversized files and surface them as skipped (failed) documents instead
+    // of dropping them silently at listing time.
+    const candidateFiles = data.entries.filter(isDownloadableFile)
+
+    const maxFiles = sourceConfig.maxFiles ? Number(sourceConfig.maxFiles) : 0
+    const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
+
+    const stubs = candidateFiles.map((entry) =>
+      stubOrSkipBySize(fileToStub(entry), entry.size, MAX_FILE_SIZE)
+    )
+
+    const { documents, indexableCount, capReached } = takeIndexableWithinCap(
+      stubs,
+      isSkippedDocument,
+      maxFiles,
+      previouslyFetched
+    )
+
+    const totalFetched = previouslyFetched + indexableCount
+    if (syncContext) syncContext.totalDocsFetched = totalFetched
+    const hitLimit = capReached
+    if (hitLimit && syncContext) syncContext.listingCapped = true
+
+    return {
+      documents,
+      nextCursor: hitLimit ? undefined : data.has_more ? data.cursor : undefined,
+      hasMore: hitLimit ? false : data.has_more,
+    }
+  },
+
+  getDocument: async (
+    accessToken: string,
+    _sourceConfig: Record<string, unknown>,
+    externalId: string
+  ): Promise<ExternalDocument | null> => {
+    try {
+      const response = await fetchWithRetry('https://api.dropboxapi.com/2/files/get_metadata', {
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${accessToken}`,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({ path: externalId }),
+      })
+
+      if (!response.ok) {
+        if (response.status === 409) return null
+        throw new Error(`Failed to get metadata: ${response.status}`)
+      }
+
+      const entry = (await response.json()) as DropboxFileEntry
+
+      if (!isDownloadableFile(entry)) return null
+
+      const stub = fileToStub(entry)
+      if (entry.size && entry.size > MAX_FILE_SIZE) {
+        return markSkipped(stub, sizeLimitSkipReason(MAX_FILE_SIZE))
+      }
+
+      let content: string
+      try {
+        content = await downloadFileContent(accessToken, entry.path_lower)
+      } catch (error) {
+        if (error instanceof ConnectorFileTooLargeError) {
+          return markSkipped(stub, sizeLimitSkipReason(error.limitBytes))
+        }
+        throw error
+      }
+      if (!content.trim()) return null
+
+      return { ...stub, content, contentDeferred: false }
+    } catch (error) {
+      logger.warn(`Failed to fetch document ${externalId}`, {
+        error: toError(error).message,
+      })
+      return null
+    }
+  },
+
+  validateConfig: async (
+    accessToken: string,
+    sourceConfig: Record<string, unknown>
+  ): Promise<{ valid: boolean; error?: string }> => {
+    const maxFiles = sourceConfig.maxFiles as string | undefined
+    if (maxFiles && (Number.isNaN(Number(maxFiles)) || Number(maxFiles) <= 0)) {
+      return { valid: false, error: 'Max files must be a positive number' }
+    }
+
+    try {
+      const folderPath = (sourceConfig.folderPath as string)?.trim() || ''
+      const path = folderPath.startsWith('/') ? folderPath : folderPath ? `/${folderPath}` : ''
+
+      const response = await fetchWithRetry(
+        'https://api.dropboxapi.com/2/files/list_folder',
+        {
+          method: 'POST',
+          headers: {
+            Authorization: `Bearer ${accessToken}`,
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify({
+            path,
+            limit: 1,
+            recursive: false,
+          }),
+        },
+        VALIDATE_RETRY_OPTIONS
+      )
+
+      if (!response.ok) {
+        const errorText = await response.text()
+        if (errorText.includes('not_found')) {
+          return { valid: false, error: 'Folder not found. Check the path and try again.' }
+        }
+        return { valid: false, error: `Failed to access Dropbox: ${response.status}` }
+      }
+
+      return { valid: true }
+    } catch (error) {
+      const message = getErrorMessage(error, 'Failed to validate configuration')
+      return { valid: false, error: message }
+    }
+  },
+
+  mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
+    const result: Record<string, unknown> = {}
+
+    if (typeof metadata.path === 'string') {
+      result.path = metadata.path
+    }
+
+    const lastModified = parseTagDate(metadata.lastModified)
+    if (lastModified) result.lastModified = lastModified
+
+    if (metadata.fileSize != null) {
+      const num = Number(metadata.fileSize)
+      if (!Number.isNaN(num)) result.fileSize = num
+    }
+
+    return result
+  },
+}

apps/sim/connectors/dropbox/index.ts

@@ -0,0 +1 @@
+export { dropboxConnector } from '@/connectors/dropbox/dropbox'

apps/sim/connectors/dropbox/meta.ts

@@ -0,0 +1,40 @@
+import { DropboxIcon } from '@/components/icons'
+import type { ConnectorMeta } from '@/connectors/types'
+
+export const dropboxConnectorMeta: ConnectorMeta = {
+  id: 'dropbox',
+  name: 'Dropbox',
+  description: 'Sync text files from Dropbox',
+  version: '1.0.0',
+  icon: DropboxIcon,
+
+  auth: {
+    mode: 'oauth',
+    provider: 'dropbox',
+    requiredScopes: ['files.metadata.read', 'files.content.read'],
+  },
+
+  configFields: [
+    {
+      id: 'folderPath',
+      title: 'Folder Path',
+      type: 'short-input',
+      placeholder: 'e.g. /Documents (default: entire Dropbox)',
+      required: false,
+      description: 'Leave empty to sync all supported files',
+    },
+    {
+      id: 'maxFiles',
+      title: 'Max Files',
+      type: 'short-input',
+      required: false,
+      placeholder: 'e.g. 500 (default: unlimited)',
+    },
+  ],
+
+  tagDefinitions: [
+    { id: 'path', displayName: 'File Path', fieldType: 'text' },
+    { id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
+    { id: 'fileSize', displayName: 'File Size (bytes)', fieldType: 'number' },
+  ],
+}