simstudioai--sim/apps/sim/connectors/sharepoint/sharepoint.ts

import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { sharepointConnectorMeta } from '@/connectors/sharepoint/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import {
  CONNECTOR_MAX_FILE_BYTES,
  ConnectorFileTooLargeError,
  htmlToPlainText,
  isSkippedDocument,
  markSkipped,
  parseTagDate,
  readBodyWithLimit,
  sizeLimitSkipReason,
  stubOrSkipBySize,
  takeIndexableWithinCap,
} from '@/connectors/utils'

const logger = createLogger('SharePointConnector')

const GRAPH_BASE = 'https://graph.microsoft.com/v1.0'

const SUPPORTED_TEXT_EXTENSIONS = new Set([
  '.txt',
  '.md',
  '.html',
  '.htm',
  '.csv',
  '.json',
  '.xml',
  '.yaml',
  '.yml',
  '.log',
  '.rst',
  '.tsv',
])

const MAX_DOWNLOAD_SIZE = CONNECTOR_MAX_FILE_BYTES

/** Microsoft Graph drive item shape (subset of fields we use). */
interface DriveItem {
  id: string
  name: string
  webUrl?: string
  size?: number
  file?: { mimeType?: string }
  folder?: { childCount?: number }
  lastModifiedDateTime?: string
  createdDateTime?: string
  createdBy?: { user?: { displayName?: string } }
  parentReference?: { path?: string; siteId?: string }
}

interface DriveItemListResponse {
  value: DriveItem[]
  '@odata.nextLink'?: string
}

/**
 * Returns true when the file extension is in the supported text set.
 */
function isSupportedTextFile(name: string): boolean {
  const dotIndex = name.lastIndexOf('.')
  if (dotIndex === -1) return false
  return SUPPORTED_TEXT_EXTENSIONS.has(name.slice(dotIndex).toLowerCase())
}

/**
 * Resolves a SharePoint site URL like "contoso.sharepoint.com/sites/mysite"
 * into a Microsoft Graph siteId.
 */
async function resolveSiteId(
  accessToken: string,
  siteUrl: string,
  retryOptions?: Parameters<typeof fetchWithRetry>[2]
): Promise<{ id: string; displayName: string }> {
  // Normalise: strip protocol, trailing slashes
  const cleaned = siteUrl.replace(/^https?:\/\//, '').replace(/\/+$/, '')

  // Split into hostname and server-relative path
  const firstSlash = cleaned.indexOf('/')
  let hostname: string
  let serverRelativePath: string

  if (firstSlash === -1) {
    hostname = cleaned
    serverRelativePath = ''
  } else {
    hostname = cleaned.slice(0, firstSlash)
    serverRelativePath = cleaned.slice(firstSlash)
  }

  // Graph endpoint: GET /sites/{hostname}:/{path}
  const url = serverRelativePath
    ? `${GRAPH_BASE}/sites/${hostname}:${serverRelativePath}`
    : `${GRAPH_BASE}/sites/${hostname}`

  const response = await fetchWithRetry(
    url,
    {
      method: 'GET',
      headers: {
        Authorization: `Bearer ${accessToken}`,
        Accept: 'application/json',
      },
    },
    retryOptions
  )

  if (!response.ok) {
    const errorText = await response.text()
    throw new Error(
      `Failed to resolve SharePoint site "${siteUrl}": ${response.status} – ${errorText}`
    )
  }

  const site = (await response.json()) as { id: string; displayName?: string }
  logger.info('Resolved SharePoint site', {
    siteUrl,
    siteId: site.id,
    displayName: site.displayName,
  })
  return { id: site.id, displayName: site.displayName ?? '' }
}

/**
 * Downloads the text content of a drive item.
 */
async function downloadFileContent(
  accessToken: string,
  siteId: string,
  itemId: string,
  fileName: string
): Promise<string> {
  const url = `${GRAPH_BASE}/sites/${siteId}/drive/items/${itemId}/content`

  const response = await fetchWithRetry(url, {
    method: 'GET',
    headers: { Authorization: `Bearer ${accessToken}` },
    redirect: 'follow',
  })

  if (!response.ok) {
    throw new Error(`Failed to download file "${fileName}" (${itemId}): ${response.status}`)
  }

  // Stream with a hard byte cap so a file with missing/under-reported listing
  // size metadata is never fully buffered into memory. Oversized files are
  // skipped (returned empty) rather than indexed as truncated partial content.
  const buffer = await readBodyWithLimit(response, MAX_DOWNLOAD_SIZE)
  if (!buffer) {
    throw new ConnectorFileTooLargeError(MAX_DOWNLOAD_SIZE)
  }
  return buffer.toString('utf8')
}

/**
 * Fetches file content, applying HTML-to-text conversion for .html files.
 */
async function fetchFileContent(
  accessToken: string,
  siteId: string,
  itemId: string,
  fileName: string
): Promise<string> {
  const raw = await downloadFileContent(accessToken, siteId, itemId, fileName)
  if (fileName.toLowerCase().endsWith('.html') || fileName.toLowerCase().endsWith('.htm')) {
    return htmlToPlainText(raw)
  }
  return raw
}

/**
 * Converts a DriveItem to a lightweight metadata stub (no content download).
 */
function itemToStub(item: DriveItem, siteName: string): ExternalDocument {
  return {
    externalId: item.id,
    title: item.name || 'Untitled',
    content: '',
    contentDeferred: true,
    mimeType: 'text/plain',
    sourceUrl: item.webUrl,
    contentHash: `sharepoint:${item.id}:${item.lastModifiedDateTime ?? ''}`,
    metadata: {
      lastModifiedDateTime: item.lastModifiedDateTime,
      createdDateTime: item.createdDateTime,
      createdBy: item.createdBy?.user?.displayName,
      fileSize: item.size,
      path: item.parentReference?.path,
      siteName,
    },
  }
}

/**
 * Lists items in a folder. When folderId is omitted the root of the default
 * document library is listed.
 */
async function listFolderItems(
  accessToken: string,
  siteId: string,
  folderId?: string,
  nextLink?: string
): Promise<DriveItemListResponse> {
  const url =
    nextLink ??
    (folderId
      ? `${GRAPH_BASE}/sites/${siteId}/drive/items/${folderId}/children?$top=200`
      : `${GRAPH_BASE}/sites/${siteId}/drive/root/children?$top=200`)

  const response = await fetchWithRetry(url, {
    method: 'GET',
    headers: {
      Authorization: `Bearer ${accessToken}`,
      Accept: 'application/json',
    },
  })

  if (!response.ok) {
    const errorText = await response.text()
    throw new Error(`Failed to list folder items: ${response.status} – ${errorText}`)
  }

  return response.json() as Promise<DriveItemListResponse>
}

/**
 * Resolves a slash-separated folder path (e.g. "Documents/Reports") to a
 * DriveItem ID by walking the path segments from root.
 */
async function resolveFolderPath(
  accessToken: string,
  siteId: string,
  folderPath: string
): Promise<string> {
  const cleaned = folderPath.replace(/^\/+|\/+$/g, '')
  if (!cleaned) {
    throw new Error('Folder path is empty after normalisation')
  }

  const encoded = cleaned.split('/').map(encodeURIComponent).join('/')
  const url = `${GRAPH_BASE}/sites/${siteId}/drive/root:/${encoded}`

  const response = await fetchWithRetry(url, {
    method: 'GET',
    headers: {
      Authorization: `Bearer ${accessToken}`,
      Accept: 'application/json',
    },
  })

  if (!response.ok) {
    if (response.status === 404) {
      throw new Error(`Folder not found: "${folderPath}"`)
    }
    throw new Error(`Failed to resolve folder path "${folderPath}": ${response.status}`)
  }

  const item = (await response.json()) as DriveItem
  if (!item.folder) {
    throw new Error(`Path "${folderPath}" is not a folder`)
  }

  return item.id
}

/**
 * Pagination state encoded as the cursor string.
 * We track a stack of folder IDs to traverse plus an optional @odata.nextLink.
 */
interface PaginationState {
  /** Folders still to be listed (depth-first) */
  folderStack: string[]
  /** Current folder being listed (undefined = root) */
  currentFolder?: string
  /** @odata.nextLink for the current folder page */
  nextLink?: string
}

function encodeCursor(state: PaginationState): string {
  return Buffer.from(JSON.stringify(state)).toString('base64')
}

function decodeCursor(cursor: string): PaginationState {
  return JSON.parse(Buffer.from(cursor, 'base64').toString('utf-8')) as PaginationState
}

export const sharepointConnector: ConnectorConfig = {
  ...sharepointConnectorMeta,

  listDocuments: async (
    accessToken: string,
    sourceConfig: Record<string, unknown>,
    cursor?: string,
    syncContext?: Record<string, unknown>
  ): Promise<ExternalDocumentList> => {
    const siteUrl = sourceConfig.siteUrl as string

    // Resolve and cache siteId in syncContext
    let siteId: string
    let siteName: string
    if (syncContext?.siteId) {
      siteId = syncContext.siteId as string
      siteName = (syncContext.siteName as string) ?? ''
    } else {
      const site = await resolveSiteId(accessToken, siteUrl)
      siteId = site.id
      siteName = site.displayName || siteUrl

      if (syncContext) {
        syncContext.siteId = siteId
        syncContext.siteName = siteName
      }
    }

    // Resolve starting folder if configured (cache in syncContext)
    let rootFolderId: string | undefined
    const folderPath = (sourceConfig.folderPath as string)?.trim()
    if (folderPath) {
      if (syncContext?.rootFolderId) {
        rootFolderId = syncContext.rootFolderId as string
      } else {
        rootFolderId = await resolveFolderPath(accessToken, siteId, folderPath)
        if (syncContext) syncContext.rootFolderId = rootFolderId
      }
    }

    // Decode or initialise pagination state
    let state: PaginationState
    if (cursor) {
      state = decodeCursor(cursor)
    } else {
      state = {
        folderStack: [],
        currentFolder: rootFolderId,
      }
    }

    const documents: ExternalDocument[] = []
    const maxFiles = sourceConfig.maxFiles ? Number(sourceConfig.maxFiles) : 0
    let totalFetched = (syncContext?.totalDocsFetched as number) ?? 0

    // Process one page of items from the current folder
    const data = await listFolderItems(accessToken, siteId, state.currentFolder, state.nextLink)

    // Separate files and subfolders
    const subfolders: string[] = []
    const files: DriveItem[] = []

    for (const item of data.value) {
      if (item.folder) {
        subfolders.push(item.id)
      } else if (item.file && isSupportedTextFile(item.name)) {
        // Keep oversized files; they are surfaced as skipped (failed) docs below.
        files.push(item)
      }
    }

    // Push subfolders onto the stack for depth-first traversal
    state.folderStack.push(...subfolders)

    // Convert files to lightweight stubs (no content download). Oversized files are
    // kept as skipped stubs but do not consume the max-files cap.
    const previouslyFetched = totalFetched
    const stubs = files.map((file) =>
      stubOrSkipBySize(itemToStub(file, siteName), file.size, MAX_DOWNLOAD_SIZE)
    )
    const {
      documents: pageDocuments,
      indexableCount,
      capReached,
    } = takeIndexableWithinCap(stubs, isSkippedDocument, maxFiles, previouslyFetched)
    documents.push(...pageDocuments)

    totalFetched += indexableCount

    if (syncContext) syncContext.totalDocsFetched = totalFetched
    const hitLimit = capReached
    if (hitLimit && syncContext) syncContext.listingCapped = true

    if (hitLimit) {
      return { documents, hasMore: false }
    }

    if (data['@odata.nextLink']) {
      // More pages in the current folder
      state.nextLink = data['@odata.nextLink']
      return {
        documents,
        nextCursor: encodeCursor(state),
        hasMore: true,
      }
    }

    // Current folder exhausted — move to next folder on the stack
    if (state.folderStack.length > 0) {
      const nextFolder = state.folderStack.pop()!
      state.currentFolder = nextFolder
      state.nextLink = undefined
      return {
        documents,
        nextCursor: encodeCursor(state),
        hasMore: true,
      }
    }

    // Nothing left
    return { documents, hasMore: false }
  },

  getDocument: async (
    accessToken: string,
    sourceConfig: Record<string, unknown>,
    externalId: string,
    syncContext?: Record<string, unknown>
  ): Promise<ExternalDocument | null> => {
    const siteUrl = sourceConfig.siteUrl as string

    let siteId = syncContext?.siteId as string | undefined
    let siteName = syncContext?.siteName as string | undefined
    if (!siteId) {
      const site = await resolveSiteId(accessToken, siteUrl)
      siteId = site.id
      siteName = site.displayName ?? siteUrl
      if (syncContext) {
        syncContext.siteId = siteId
        syncContext.siteName = siteName
      }
    }

    const url = `${GRAPH_BASE}/sites/${siteId}/drive/items/${externalId}`
    const response = await fetchWithRetry(url, {
      method: 'GET',
      headers: {
        Authorization: `Bearer ${accessToken}`,
        Accept: 'application/json',
      },
    })

    if (!response.ok) {
      if (response.status === 404) return null
      throw new Error(`Failed to get SharePoint file: ${response.status}`)
    }

    const item = (await response.json()) as DriveItem

    if (!item.file || !isSupportedTextFile(item.name)) {
      return null
    }

    try {
      const content = await fetchFileContent(accessToken, siteId, item.id, item.name)
      if (!content.trim()) return null

      const stub = itemToStub(item, siteName ?? siteUrl)
      return { ...stub, content, contentDeferred: false }
    } catch (error) {
      if (error instanceof ConnectorFileTooLargeError) {
        logger.info('Skipping oversized SharePoint file', { fileId: item.id, name: item.name })
        return markSkipped(
          itemToStub(item, siteName ?? siteUrl),
          sizeLimitSkipReason(error.limitBytes)
        )
      }
      logger.warn(`Failed to fetch content for file: ${item.name} (${item.id})`, {
        error: toError(error).message,
      })
      return null
    }
  },

  validateConfig: async (
    accessToken: string,
    sourceConfig: Record<string, unknown>
  ): Promise<{ valid: boolean; error?: string }> => {
    const siteUrl = (sourceConfig.siteUrl as string)?.trim()
    if (!siteUrl) {
      return { valid: false, error: 'Site URL is required' }
    }

    const maxFiles = sourceConfig.maxFiles as string | undefined
    if (maxFiles && (Number.isNaN(Number(maxFiles)) || Number(maxFiles) <= 0)) {
      return { valid: false, error: 'Max files must be a positive number' }
    }

    try {
      const site = await resolveSiteId(accessToken, siteUrl, VALIDATE_RETRY_OPTIONS)
      const siteId = site.id

      // If a folder path is configured, verify it exists
      const folderPath = (sourceConfig.folderPath as string)?.trim()
      if (folderPath) {
        const encodedPath = folderPath
          .replace(/^\/+|\/+$/g, '')
          .split('/')
          .map(encodeURIComponent)
          .join('/')
        const folderUrl = `${GRAPH_BASE}/sites/${siteId}/drive/root:/${encodedPath}`
        const response = await fetchWithRetry(
          folderUrl,
          {
            method: 'GET',
            headers: {
              Authorization: `Bearer ${accessToken}`,
              Accept: 'application/json',
            },
          },
          VALIDATE_RETRY_OPTIONS
        )

        if (!response.ok) {
          if (response.status === 404) {
            return { valid: false, error: `Folder not found: "${folderPath}"` }
          }
          return { valid: false, error: `Failed to access folder: ${response.status}` }
        }

        const item = (await response.json()) as DriveItem
        if (!item.folder) {
          return { valid: false, error: `Path "${folderPath}" is not a folder` }
        }
      }

      return { valid: true }
    } catch (error) {
      const message = getErrorMessage(error, 'Failed to validate configuration')
      return { valid: false, error: message }
    }
  },

  mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
    const result: Record<string, unknown> = {}

    if (typeof metadata.path === 'string') {
      result.path = metadata.path
    }

    const lastModified = parseTagDate(metadata.lastModifiedDateTime)
    if (lastModified) result.lastModified = lastModified

    if (typeof metadata.fileSize === 'number') {
      result.fileSize = metadata.fileSize
    }

    if (typeof metadata.createdBy === 'string') {
      result.createdBy = metadata.createdBy
    }

    if (typeof metadata.siteName === 'string') {
      result.siteName = metadata.siteName
    }

    return result
  },
}