chore: import upstream snapshot with attribution

apps/sim/connectors/github/github.ts

@@ -0,0 +1,448 @@
+import { createLogger } from '@sim/logger'
+import { getErrorMessage, toError } from '@sim/utils/errors'
+import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
+import { githubConnectorMeta } from '@/connectors/github/meta'
+import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
+import {
+  CONNECTOR_MAX_FILE_BYTES,
+  markSkipped,
+  parseTagDate,
+  sizeLimitSkipReason,
+  stubOrSkipBySize,
+  takeIndexableWithinCap,
+} from '@/connectors/utils'
+
+const logger = createLogger('GitHubConnector')
+
+const GITHUB_API_URL = 'https://api.github.com'
+const BATCH_SIZE = 30
+const GIT_SHA_PREFIX = 'git-sha:'
+const MAX_FILE_SIZE = CONNECTOR_MAX_FILE_BYTES
+const BINARY_SNIFF_BYTES = 8000
+
+/**
+ * Heuristic binary detection: Git treats files containing a NUL byte in the
+ * first 8000 bytes as binary. Matches `git diff` / `git grep` semantics.
+ */
+function isBinaryBuffer(buf: Buffer): boolean {
+  const len = Math.min(buf.length, BINARY_SNIFF_BYTES)
+  for (let i = 0; i < len; i++) {
+    if (buf[i] === 0) return true
+  }
+  return false
+}
+
+/**
+ * Parses the repository string into owner and repo.
+ */
+function parseRepo(repository: string): { owner: string; repo: string } {
+  const cleaned = repository.replace(/^https?:\/\/github\.com\//, '').replace(/\.git$/, '')
+  const parts = cleaned.split('/')
+  if (parts.length < 2 || !parts[0] || !parts[1]) {
+    throw new Error(`Invalid repository format: "${repository}". Use "owner/repo".`)
+  }
+  return { owner: parts[0], repo: parts[1] }
+}
+
+/**
+ * File extension filter set from user config. Returns null if no filter (accept all).
+ */
+function parseExtensions(extensions: string): Set<string> | null {
+  const trimmed = extensions.trim()
+  if (!trimmed) return null
+  const exts = trimmed
+    .split(',')
+    .map((e) => e.trim().toLowerCase())
+    .filter(Boolean)
+    .map((e) => (e.startsWith('.') ? e : `.${e}`))
+  return exts.length > 0 ? new Set(exts) : null
+}
+
+/**
+ * Checks whether a file path matches the extension filter.
+ */
+function matchesExtension(filePath: string, extSet: Set<string> | null): boolean {
+  if (!extSet) return true
+  const lastDot = filePath.lastIndexOf('.')
+  if (lastDot === -1) return false
+  return extSet.has(filePath.slice(lastDot).toLowerCase())
+}
+
+interface TreeItem {
+  path: string
+  mode: string
+  type: string
+  sha: string
+  size?: number
+}
+
+/**
+ * Fetches the full recursive tree for a branch.
+ */
+async function fetchTree(
+  accessToken: string,
+  owner: string,
+  repo: string,
+  branch: string
+): Promise<TreeItem[]> {
+  const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/git/trees/${encodeURIComponent(branch)}?recursive=1`
+
+  const response = await fetchWithRetry(url, {
+    method: 'GET',
+    headers: {
+      Accept: 'application/vnd.github+json',
+      Authorization: `Bearer ${accessToken}`,
+      'X-GitHub-Api-Version': '2022-11-28',
+    },
+  })
+
+  if (!response.ok) {
+    const errorText = await response.text()
+    logger.error('Failed to fetch GitHub tree', { status: response.status, error: errorText })
+    throw new Error(`Failed to fetch repository tree: ${response.status}`)
+  }
+
+  const data = await response.json()
+
+  if (data.truncated) {
+    logger.warn('GitHub tree was truncated — some files may be missing', { owner, repo, branch })
+  }
+
+  return (data.tree || []).filter((item: TreeItem) => item.type === 'blob')
+}
+
+/**
+ * Fetches blob content via the Git Blobs API. Used as a fallback when the
+ * `/contents/` endpoint cannot return the file body (files larger than 1 MB
+ * return `content: ""` and `encoding: "none"`). Supports blobs up to 100 MB.
+ */
+async function fetchBlobContent(
+  accessToken: string,
+  owner: string,
+  repo: string,
+  sha: string
+): Promise<string | null> {
+  const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/git/blobs/${encodeURIComponent(sha)}`
+  const response = await fetchWithRetry(url, {
+    method: 'GET',
+    headers: {
+      Accept: 'application/vnd.github+json',
+      Authorization: `Bearer ${accessToken}`,
+      'X-GitHub-Api-Version': '2022-11-28',
+    },
+  })
+
+  if (!response.ok) {
+    throw new Error(`Failed to fetch git blob ${sha}: ${response.status}`)
+  }
+
+  const data = await response.json()
+  const content = (data.content as string) || ''
+  const encoding = data.encoding as string | undefined
+
+  if (encoding === 'base64') {
+    const buf = Buffer.from(content, 'base64')
+    if (isBinaryBuffer(buf)) return null
+    return buf.toString('utf8')
+  }
+  /**
+   * Per https://docs.github.com/en/rest/git/blobs the Blobs API only ever
+   * returns base64. Refuse to silently persist empty content for an
+   * unexpected encoding so a sync surfaces the error instead.
+   */
+  throw new Error(`Unexpected git blob encoding for ${sha}: ${encoding ?? 'undefined'}`)
+}
+
+/**
+ * Creates a lightweight stub ExternalDocument from a tree item.
+ * Uses the Git blob SHA as contentHash for change detection, avoiding
+ * the need to fetch blob content for every file during listing.
+ * Content is deferred and only fetched for new/changed documents.
+ */
+function treeItemToStub(
+  owner: string,
+  repo: string,
+  branch: string,
+  item: TreeItem
+): ExternalDocument {
+  return {
+    externalId: item.path,
+    title: item.path.split('/').pop() || item.path,
+    content: '',
+    contentDeferred: true,
+    mimeType: 'text/plain',
+    sourceUrl: `https://github.com/${owner}/${repo}/blob/${branch.split('/').map(encodeURIComponent).join('/')}/${item.path.split('/').map(encodeURIComponent).join('/')}`,
+    contentHash: `${GIT_SHA_PREFIX}${item.sha}`,
+    metadata: {
+      path: item.path,
+      sha: item.sha,
+      size: item.size,
+      branch,
+      repository: `${owner}/${repo}`,
+    },
+  }
+}
+
+export const githubConnector: ConnectorConfig = {
+  ...githubConnectorMeta,
+
+  listDocuments: async (
+    accessToken: string,
+    sourceConfig: Record<string, unknown>,
+    cursor?: string,
+    syncContext?: Record<string, unknown>
+  ): Promise<ExternalDocumentList> => {
+    const { owner, repo } = parseRepo(sourceConfig.repository as string)
+    const branch = ((sourceConfig.branch as string) || 'main').trim()
+    const pathPrefix = ((sourceConfig.pathPrefix as string) || '').trim()
+    const extSet = parseExtensions((sourceConfig.extensions as string) || '')
+    const maxFiles = sourceConfig.maxFiles ? Number(sourceConfig.maxFiles) : 0
+
+    let capped: TreeItem[]
+    if (syncContext?.filteredTree) {
+      capped = syncContext.filteredTree as TreeItem[]
+    } else {
+      const tree = await fetchTree(accessToken, owner, repo, branch)
+
+      // Filter by path prefix and extensions. Oversized files are kept here and
+      // surfaced as skipped (failed) documents at stub time so they stay visible.
+      const filtered = tree.filter((item) => {
+        if (pathPrefix && !item.path.startsWith(pathPrefix)) return false
+        if (!matchesExtension(item.path, extSet)) return false
+        return true
+      })
+
+      // Apply the max-files limit to indexable files only; oversized files within
+      // the capped window are kept (and surfaced as skipped) but never consume the cap.
+      capped =
+        maxFiles > 0
+          ? takeIndexableWithinCap(
+              filtered,
+              (item) => Boolean(item.size && item.size > MAX_FILE_SIZE),
+              maxFiles,
+              0
+            ).documents
+          : filtered
+      if (syncContext) syncContext.filteredTree = capped
+    }
+
+    // Paginate using offset cursor
+    const offset = cursor ? Number(cursor) : 0
+    const batch = capped.slice(offset, offset + BATCH_SIZE)
+
+    logger.info('Listing GitHub files', {
+      owner,
+      repo,
+      branch,
+      totalFiltered: capped.length,
+      offset,
+      batchSize: batch.length,
+    })
+
+    const documents = batch.map((item) =>
+      stubOrSkipBySize(treeItemToStub(owner, repo, branch, item), item.size, MAX_FILE_SIZE)
+    )
+
+    const nextOffset = offset + BATCH_SIZE
+    const hasMore = nextOffset < capped.length
+
+    return {
+      documents,
+      nextCursor: hasMore ? String(nextOffset) : undefined,
+      hasMore,
+    }
+  },
+
+  getDocument: async (
+    accessToken: string,
+    sourceConfig: Record<string, unknown>,
+    externalId: string,
+    _syncContext?: Record<string, unknown>
+  ): Promise<ExternalDocument | null> => {
+    const { owner, repo } = parseRepo(sourceConfig.repository as string)
+    const branch = ((sourceConfig.branch as string) || 'main').trim()
+
+    // externalId is the file path
+    const path = externalId
+
+    try {
+      const encodedPath = path.split('/').map(encodeURIComponent).join('/')
+      const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/contents/${encodedPath}?ref=${encodeURIComponent(branch)}`
+      const response = await fetchWithRetry(url, {
+        method: 'GET',
+        headers: {
+          Accept: 'application/vnd.github+json',
+          Authorization: `Bearer ${accessToken}`,
+          'X-GitHub-Api-Version': '2022-11-28',
+        },
+      })
+
+      if (!response.ok) {
+        if (response.status === 404) return null
+        if (response.status === 403) {
+          logger.info('Skipping GitHub file rejected by Contents API', {
+            path,
+            status: response.status,
+          })
+          return null
+        }
+        throw new Error(`Failed to fetch file ${path}: ${response.status}`)
+      }
+
+      const lastModifiedHeader = response.headers.get('last-modified') || undefined
+      const data = await response.json()
+
+      const size = typeof data.size === 'number' ? data.size : 0
+      if (size > MAX_FILE_SIZE) {
+        logger.info('Skipping GitHub file exceeding size limit', {
+          path,
+          size,
+          limit: MAX_FILE_SIZE,
+        })
+        return markSkipped(
+          {
+            externalId,
+            title: path.split('/').pop() || path,
+            content: '',
+            mimeType: 'text/plain',
+            sourceUrl: `https://github.com/${owner}/${repo}/blob/${branch.split('/').map(encodeURIComponent).join('/')}/${path.split('/').map(encodeURIComponent).join('/')}`,
+            contentHash: `${GIT_SHA_PREFIX}${data.sha as string}`,
+            metadata: {
+              path,
+              sha: data.sha as string,
+              size,
+              branch,
+              repository: `${owner}/${repo}`,
+            },
+          },
+          sizeLimitSkipReason(MAX_FILE_SIZE)
+        )
+      }
+
+      const rawContent = (data.content as string) || ''
+      const encoding = data.encoding as string | undefined
+      let content: string
+      if (encoding === 'base64' && rawContent.length > 0) {
+        const buf = Buffer.from(rawContent, 'base64')
+        if (isBinaryBuffer(buf)) {
+          logger.info('Skipping binary GitHub file', { path, size })
+          return null
+        }
+        content = buf.toString('utf8')
+      } else if (encoding === 'none' && data.sha && size > 0) {
+        const blobContent = await fetchBlobContent(accessToken, owner, repo, data.sha as string)
+        if (blobContent === null) {
+          logger.info('Skipping binary GitHub file', { path, size })
+          return null
+        }
+        content = blobContent
+      } else {
+        content = ''
+      }
+
+      return {
+        externalId,
+        title: path.split('/').pop() || path,
+        content,
+        contentDeferred: false,
+        mimeType: 'text/plain',
+        sourceUrl: `https://github.com/${owner}/${repo}/blob/${branch.split('/').map(encodeURIComponent).join('/')}/${path.split('/').map(encodeURIComponent).join('/')}`,
+        contentHash: `${GIT_SHA_PREFIX}${data.sha as string}`,
+        metadata: {
+          path,
+          sha: data.sha as string,
+          size: data.size as number,
+          branch,
+          repository: `${owner}/${repo}`,
+          lastModified: lastModifiedHeader,
+        },
+      }
+    } catch (error) {
+      logger.warn(`Failed to fetch GitHub document ${externalId}`, {
+        error: toError(error).message,
+      })
+      return null
+    }
+  },
+
+  validateConfig: async (
+    accessToken: string,
+    sourceConfig: Record<string, unknown>
+  ): Promise<{ valid: boolean; error?: string }> => {
+    const repository = (sourceConfig.repository as string)?.trim()
+    if (!repository) {
+      return { valid: false, error: 'Repository is required' }
+    }
+
+    let owner: string
+    let repo: string
+    try {
+      const parsed = parseRepo(repository)
+      owner = parsed.owner
+      repo = parsed.repo
+    } catch (error) {
+      return {
+        valid: false,
+        error: getErrorMessage(error, 'Invalid repository format'),
+      }
+    }
+
+    const maxFiles = sourceConfig.maxFiles as string | undefined
+    if (maxFiles && (Number.isNaN(Number(maxFiles)) || Number(maxFiles) <= 0)) {
+      return { valid: false, error: 'Max files must be a positive number' }
+    }
+
+    const branch = ((sourceConfig.branch as string) || 'main').trim()
+
+    try {
+      // Verify repo and branch are accessible
+      const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/branches/${encodeURIComponent(branch)}`
+      const response = await fetchWithRetry(
+        url,
+        {
+          method: 'GET',
+          headers: {
+            Accept: 'application/vnd.github+json',
+            Authorization: `Bearer ${accessToken}`,
+            'X-GitHub-Api-Version': '2022-11-28',
+          },
+        },
+        VALIDATE_RETRY_OPTIONS
+      )
+
+      if (response.status === 404) {
+        return {
+          valid: false,
+          error: `Repository "${owner}/${repo}" or branch "${branch}" not found`,
+        }
+      }
+
+      if (!response.ok) {
+        return { valid: false, error: `Cannot access repository: ${response.status}` }
+      }
+
+      return { valid: true }
+    } catch (error) {
+      const message = getErrorMessage(error, 'Failed to validate configuration')
+      return { valid: false, error: message }
+    }
+  },
+
+  mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
+    const result: Record<string, unknown> = {}
+
+    if (typeof metadata.path === 'string') result.path = metadata.path
+    if (typeof metadata.repository === 'string') result.repository = metadata.repository
+    if (typeof metadata.branch === 'string') result.branch = metadata.branch
+
+    if (metadata.size != null) {
+      const num = Number(metadata.size)
+      if (!Number.isNaN(num)) result.size = num
+    }
+
+    const lastModified = parseTagDate(metadata.lastModified)
+    if (lastModified) result.lastModified = lastModified
+
+    return result
+  },
+}

apps/sim/connectors/github/index.ts

@@ -0,0 +1 @@
+export { githubConnector } from '@/connectors/github/github'

apps/sim/connectors/github/meta.ts

@@ -0,0 +1,62 @@
+import { GithubIcon } from '@/components/icons'
+import type { ConnectorMeta } from '@/connectors/types'
+
+export const githubConnectorMeta: ConnectorMeta = {
+  id: 'github',
+  name: 'GitHub',
+  description: 'Sync files from a GitHub repository',
+  version: '1.0.0',
+  icon: GithubIcon,
+
+  auth: {
+    mode: 'apiKey',
+    label: 'Personal Access Token',
+    placeholder: 'ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx',
+  },
+
+  configFields: [
+    {
+      id: 'repository',
+      title: 'Repository',
+      type: 'short-input',
+      placeholder: 'owner/repo',
+      required: true,
+    },
+    {
+      id: 'branch',
+      title: 'Branch',
+      type: 'short-input',
+      placeholder: 'main (default)',
+      required: false,
+    },
+    {
+      id: 'pathPrefix',
+      title: 'Path Filter',
+      type: 'short-input',
+      placeholder: 'e.g. docs/, src/components/',
+      required: false,
+    },
+    {
+      id: 'extensions',
+      title: 'File Extensions',
+      type: 'short-input',
+      placeholder: 'e.g. .md, .txt, .mdx',
+      required: false,
+    },
+    {
+      id: 'maxFiles',
+      title: 'Max Files',
+      type: 'short-input',
+      required: false,
+      placeholder: 'e.g. 500 (default: unlimited)',
+    },
+  ],
+
+  tagDefinitions: [
+    { id: 'path', displayName: 'File Path', fieldType: 'text' },
+    { id: 'repository', displayName: 'Repository', fieldType: 'text' },
+    { id: 'branch', displayName: 'Branch', fieldType: 'text' },
+    { id: 'size', displayName: 'File Size', fieldType: 'number' },
+    { id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
+  ],
+}