chore: import upstream snapshot with attribution
Publish CLI Package / publish-npm (push) Waiting to run
Publish Python SDK / publish-pypi (push) Waiting to run
Publish TypeScript SDK / publish-npm (push) Waiting to run
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:20:55 +08:00
commit d25d482dc2
13754 changed files with 4996608 additions and 0 deletions
+339
View File
@@ -0,0 +1,339 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { airtableConnectorMeta } from '@/connectors/airtable/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { computeContentHash, parseTagDate } from '@/connectors/utils'
const logger = createLogger('AirtableConnector')
const AIRTABLE_API = 'https://api.airtable.com/v0'
const PAGE_SIZE = 100
/**
* Flattens a record's fields into a plain-text representation.
* Each field is rendered as "Field Name: value" on its own line.
*/
function recordToPlainText(
fields: Record<string, unknown>,
fieldNames?: Map<string, string>
): string {
const lines: string[] = []
for (const [key, value] of Object.entries(fields)) {
if (value == null) continue
const displayName = fieldNames?.get(key) ?? key
if (Array.isArray(value)) {
// Attachments or linked records
const items = value.map((v) => {
if (typeof v === 'object' && v !== null) {
const obj = v as Record<string, unknown>
return (obj.url as string) || (obj.name as string) || JSON.stringify(v)
}
return String(v)
})
lines.push(`${displayName}: ${items.join(', ')}`)
} else if (typeof value === 'object') {
lines.push(`${displayName}: ${JSON.stringify(value)}`)
} else {
lines.push(`${displayName}: ${String(value)}`)
}
}
return lines.join('\n')
}
/**
* Extracts a human-readable title from a record's fields.
* Prefers the configured title field, then falls back to common field names.
*/
function extractTitle(fields: Record<string, unknown>, titleField?: string): string {
if (titleField && fields[titleField] != null) {
return String(fields[titleField])
}
const candidates = ['Name', 'Title', 'name', 'title', 'Summary', 'summary']
for (const candidate of candidates) {
if (fields[candidate] != null) {
return String(fields[candidate])
}
}
for (const value of Object.values(fields)) {
if (typeof value === 'string' && value.trim()) {
return value.length > 80 ? `${value.slice(0, 80)}` : value
}
}
return 'Untitled'
}
/**
* Parses the cursor format: "offset:<airtable_offset>"
*/
function parseCursor(cursor?: string): string | undefined {
if (!cursor) return undefined
if (cursor.startsWith('offset:')) return cursor.slice(7)
return cursor
}
export const airtableConnector: ConnectorConfig = {
...airtableConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const baseId = sourceConfig.baseId as string
const tableIdOrName = sourceConfig.tableIdOrName as string
const viewId = sourceConfig.viewId as string | undefined
const titleField = sourceConfig.titleField as string | undefined
const maxRecords = sourceConfig.maxRecords ? Number(sourceConfig.maxRecords) : 0
const fieldNames = await fetchFieldNames(accessToken, baseId, tableIdOrName, syncContext)
const params = new URLSearchParams()
params.append('pageSize', String(PAGE_SIZE))
if (viewId) params.append('view', viewId)
if (maxRecords > 0) params.append('maxRecords', String(maxRecords))
const offset = parseCursor(cursor)
if (offset) params.append('offset', offset)
const encodedTable = encodeURIComponent(tableIdOrName)
const url = `${AIRTABLE_API}/${baseId}/${encodedTable}?${params.toString()}`
logger.info(`Listing records from ${baseId}/${tableIdOrName}`, {
offset: offset ?? 'none',
view: viewId ?? 'default',
})
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
},
})
if (!response.ok) {
const errorText = await response.text()
logger.error('Failed to list Airtable records', {
status: response.status,
error: errorText,
})
throw new Error(`Failed to list Airtable records: ${response.status}`)
}
const data = (await response.json()) as {
records: AirtableRecord[]
offset?: string
}
const records = data.records || []
const documents: ExternalDocument[] = await Promise.all(
records.map((record) =>
recordToDocument(record, baseId, tableIdOrName, titleField, fieldNames)
)
)
const nextOffset = data.offset
return {
documents,
nextCursor: nextOffset ? `offset:${nextOffset}` : undefined,
hasMore: Boolean(nextOffset),
}
},
getDocument: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
const baseId = sourceConfig.baseId as string
const tableIdOrName = sourceConfig.tableIdOrName as string
const titleField = sourceConfig.titleField as string | undefined
const fieldNames = await fetchFieldNames(accessToken, baseId, tableIdOrName)
const encodedTable = encodeURIComponent(tableIdOrName)
const url = `${AIRTABLE_API}/${baseId}/${encodedTable}/${externalId}`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
},
})
if (!response.ok) {
if (response.status === 404 || response.status === 422) return null
throw new Error(`Failed to get Airtable record: ${response.status}`)
}
const record = (await response.json()) as AirtableRecord
return recordToDocument(record, baseId, tableIdOrName, titleField, fieldNames)
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const baseId = sourceConfig.baseId as string
const tableIdOrName = sourceConfig.tableIdOrName as string
if (!baseId || !tableIdOrName) {
return { valid: false, error: 'Base ID and table name are required' }
}
if (baseId && !baseId.startsWith('app')) {
return { valid: false, error: 'Base ID should start with "app"' }
}
const maxRecords = sourceConfig.maxRecords as string | undefined
if (maxRecords && (Number.isNaN(Number(maxRecords)) || Number(maxRecords) <= 0)) {
return { valid: false, error: 'Max records must be a positive number' }
}
try {
const encodedTable = encodeURIComponent(tableIdOrName)
const url = `${AIRTABLE_API}/${baseId}/${encodedTable}?pageSize=1`
const response = await fetchWithRetry(
url,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
},
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
const errorText = await response.text()
if (response.status === 404 || response.status === 422) {
return { valid: false, error: `Table "${tableIdOrName}" not found in base "${baseId}"` }
}
if (response.status === 403) {
return { valid: false, error: 'Access denied. Check your Airtable permissions.' }
}
return { valid: false, error: `Airtable API error: ${response.status} - ${errorText}` }
}
const viewId = sourceConfig.viewId as string | undefined
if (viewId) {
const viewUrl = `${AIRTABLE_API}/${baseId}/${encodedTable}?pageSize=1&view=${encodeURIComponent(viewId)}`
const viewResponse = await fetchWithRetry(
viewUrl,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
},
},
VALIDATE_RETRY_OPTIONS
)
if (!viewResponse.ok) {
return { valid: false, error: `View "${viewId}" not found in table "${tableIdOrName}"` }
}
}
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
const createdTime = parseTagDate(metadata.createdTime)
if (createdTime) result.createdTime = createdTime
return result
},
}
interface AirtableRecord {
id: string
fields: Record<string, unknown>
createdTime: string
}
/**
* Converts an Airtable record to an ExternalDocument.
*/
async function recordToDocument(
record: AirtableRecord,
baseId: string,
tableIdOrName: string,
titleField: string | undefined,
fieldNames: Map<string, string>
): Promise<ExternalDocument> {
const plainText = recordToPlainText(record.fields, fieldNames)
const contentHash = await computeContentHash(plainText)
const title = extractTitle(record.fields, titleField)
const encodedTable = encodeURIComponent(tableIdOrName)
const sourceUrl = `https://airtable.com/${baseId}/${encodedTable}/${record.id}`
return {
externalId: record.id,
title,
content: plainText,
mimeType: 'text/plain',
sourceUrl,
contentHash,
metadata: {
createdTime: record.createdTime,
},
}
}
/**
* Fetches the table schema to build a field ID → field name mapping.
*/
async function fetchFieldNames(
accessToken: string,
baseId: string,
tableIdOrName: string,
syncContext?: Record<string, unknown>
): Promise<Map<string, string>> {
const cacheKey = `fieldNames:${baseId}/${tableIdOrName}`
if (syncContext?.[cacheKey]) return syncContext[cacheKey] as Map<string, string>
const fieldNames = new Map<string, string>()
try {
const url = `${AIRTABLE_API}/meta/bases/${baseId}/tables`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
},
})
if (!response.ok) {
logger.warn('Failed to fetch Airtable schema, using raw field keys', {
status: response.status,
})
return fieldNames
}
const data = (await response.json()) as {
tables: { id: string; name: string; fields: { id: string; name: string; type: string }[] }[]
}
const table = data.tables.find((t) => t.id === tableIdOrName || t.name === tableIdOrName)
if (table) {
for (const field of table.fields) {
fieldNames.set(field.id, field.name)
fieldNames.set(field.name, field.name)
}
}
} catch (error) {
logger.warn('Error fetching Airtable schema', {
error: toError(error).message,
})
}
if (syncContext) syncContext[cacheKey] = fieldNames
return fieldNames
}
+1
View File
@@ -0,0 +1 @@
export { airtableConnector } from '@/connectors/airtable/airtable'
+81
View File
@@ -0,0 +1,81 @@
import { AirtableIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const airtableConnectorMeta: ConnectorMeta = {
id: 'airtable',
name: 'Airtable',
description: 'Sync records from an Airtable table',
version: '1.0.0',
icon: AirtableIcon,
auth: {
mode: 'oauth',
provider: 'airtable',
requiredScopes: ['data.records:read', 'schema.bases:read'],
},
configFields: [
{
id: 'baseSelector',
title: 'Base',
type: 'selector',
selectorKey: 'airtable.bases',
canonicalParamId: 'baseId',
mode: 'basic',
placeholder: 'Select a base',
required: true,
},
{
id: 'baseId',
title: 'Base ID',
type: 'short-input',
canonicalParamId: 'baseId',
mode: 'advanced',
placeholder: 'e.g. appXXXXXXXXXXXXXX',
required: true,
},
{
id: 'tableSelector',
title: 'Table',
type: 'selector',
selectorKey: 'airtable.tables',
canonicalParamId: 'tableIdOrName',
mode: 'basic',
dependsOn: ['baseSelector'],
placeholder: 'Select a table',
required: true,
},
{
id: 'tableIdOrName',
title: 'Table Name or ID',
type: 'short-input',
canonicalParamId: 'tableIdOrName',
mode: 'advanced',
placeholder: 'e.g. Tasks or tblXXXXXXXXXXXXXX',
required: true,
},
{
id: 'viewId',
title: 'View',
type: 'short-input',
placeholder: 'e.g. Grid view or viwXXXXXXXXXXXXXX',
required: false,
},
{
id: 'titleField',
title: 'Title Field',
type: 'short-input',
placeholder: 'e.g. Name',
required: false,
},
{
id: 'maxRecords',
title: 'Max Records',
type: 'short-input',
placeholder: 'e.g. 1000 (default: unlimited)',
required: false,
},
],
tagDefinitions: [{ id: 'createdTime', displayName: 'Created Time', fieldType: 'date' }],
}
+343
View File
@@ -0,0 +1,343 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { asanaConnectorMeta } from '@/connectors/asana/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { joinTagArray, parseTagDate } from '@/connectors/utils'
const logger = createLogger('AsanaConnector')
const ASANA_API = 'https://app.asana.com/api/1.0'
const TASK_OPT_FIELDS =
'name,notes,completed,completed_at,modified_at,assignee.name,tags.name,permalink_url'
/**
* Asana API response shape for paginated endpoints.
*/
interface AsanaPageResponse {
data: AsanaTask[]
next_page: { offset: string; uri: string } | null
}
/**
* Minimal Asana task shape used by this connector.
*/
interface AsanaTask {
gid: string
name: string
notes?: string
completed: boolean
completed_at?: string
modified_at?: string
assignee?: { name: string }
tags?: { name: string }[]
permalink_url?: string
}
/**
* Asana workspace shape.
*/
interface AsanaWorkspace {
gid: string
name: string
}
/**
* Asana project shape.
*/
interface AsanaProject {
gid: string
name: string
}
/**
* Makes a GET request to the Asana REST API.
*/
async function asanaGet<T>(
accessToken: string,
path: string,
retryOptions?: Parameters<typeof fetchWithRetry>[2]
): Promise<T> {
const response = await fetchWithRetry(
`${ASANA_API}${path}`,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
},
retryOptions
)
if (!response.ok) {
const errorText = await response.text()
logger.error('Asana API request failed', { status: response.status, path, error: errorText })
throw new Error(`Asana API error: ${response.status}`)
}
return (await response.json()) as T
}
/**
* Builds a formatted text document from an Asana task.
*/
function buildTaskContent(task: AsanaTask): string {
const parts: string[] = []
parts.push(task.name || 'Untitled')
if (task.assignee?.name) parts.push(`Assignee: ${task.assignee.name}`)
parts.push(`Completed: ${task.completed ? 'Yes' : 'No'}`)
const tagNames = task.tags?.map((t) => t.name).filter(Boolean)
if (tagNames && tagNames.length > 0) {
parts.push(`Labels: ${tagNames.join(', ')}`)
}
if (task.notes) {
parts.push('')
parts.push(task.notes)
}
return parts.join('\n')
}
/**
* Fetches all project GIDs in a workspace, used when no specific project is configured.
*/
async function listWorkspaceProjects(
accessToken: string,
workspaceGid: string
): Promise<AsanaProject[]> {
const projects: AsanaProject[] = []
let offset: string | undefined
// eslint-disable-next-line no-constant-condition
while (true) {
const offsetParam = offset ? `&offset=${offset}` : ''
const result = await asanaGet<{ data: AsanaProject[]; next_page: { offset: string } | null }>(
accessToken,
`/projects?workspace=${workspaceGid}&limit=100${offsetParam}`
)
projects.push(...result.data)
if (!result.next_page) break
offset = result.next_page.offset
}
return projects
}
export const asanaConnector: ConnectorConfig = {
...asanaConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const workspaceGid = sourceConfig.workspace as string
const projectGid = (sourceConfig.project as string) || ''
const maxTasks = sourceConfig.maxTasks ? Number(sourceConfig.maxTasks) : 0
const pageSize = maxTasks > 0 ? Math.min(maxTasks, 100) : 100
/**
* Cursor format:
* - For a single project: the offset string directly, or undefined
* - For all projects: JSON-encoded { projectIndex, offset }
*/
let projectGids: string[]
let projectIndex = 0
let offset: string | undefined
if (projectGid) {
projectGids = [projectGid]
} else {
if (!syncContext?.projectGids) {
logger.info('Fetching all projects in workspace', { workspaceGid })
const projects = await listWorkspaceProjects(accessToken, workspaceGid)
if (syncContext) syncContext.projectGids = projects.map((p) => p.gid)
projectGids = projects.map((p) => p.gid)
} else {
projectGids = syncContext.projectGids as string[]
}
}
if (cursor) {
try {
const parsed = JSON.parse(cursor) as { projectIndex: number; offset?: string }
projectIndex = parsed.projectIndex
offset = parsed.offset
} catch {
offset = cursor
}
}
logger.info('Listing Asana tasks', {
workspaceGid,
projectCount: projectGids.length,
projectIndex,
offset,
pageSize,
})
const documents: ExternalDocument[] = []
let nextCursor: string | undefined
let hasMore = false
while (projectIndex < projectGids.length) {
const currentProjectGid = projectGids[projectIndex]
const offsetParam = offset ? `&offset=${offset}` : ''
const result = await asanaGet<AsanaPageResponse>(
accessToken,
`/tasks?project=${currentProjectGid}&opt_fields=${TASK_OPT_FIELDS}&limit=${pageSize}${offsetParam}`
)
for (const task of result.data) {
const content = buildTaskContent(task)
const tagNames = task.tags?.map((t) => t.name).filter(Boolean) || []
documents.push({
externalId: task.gid,
title: task.name || 'Untitled',
content,
mimeType: 'text/plain',
sourceUrl: task.permalink_url || undefined,
contentHash: `asana:${task.gid}:${task.modified_at ?? ''}`,
metadata: {
project: currentProjectGid,
assignee: task.assignee?.name,
completed: task.completed,
lastModified: task.modified_at,
labels: tagNames,
},
})
}
if (result.next_page) {
nextCursor = JSON.stringify({ projectIndex, offset: result.next_page.offset })
hasMore = true
break
}
projectIndex++
offset = undefined
if (projectIndex < projectGids.length) {
nextCursor = JSON.stringify({ projectIndex, offset: undefined })
hasMore = true
break
}
}
const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
if (maxTasks > 0) {
const remaining = maxTasks - previouslyFetched
if (documents.length > remaining) {
documents.splice(remaining)
}
}
const totalFetched = previouslyFetched + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = maxTasks > 0 && totalFetched >= maxTasks
if (hitLimit) {
hasMore = false
nextCursor = undefined
}
return {
documents,
nextCursor: hasMore ? nextCursor : undefined,
hasMore,
}
},
getDocument: async (
accessToken: string,
_sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
try {
const result = await asanaGet<{ data: AsanaTask }>(
accessToken,
`/tasks/${externalId}?opt_fields=${TASK_OPT_FIELDS}`
)
const task = result.data
if (!task) return null
const content = buildTaskContent(task)
const tagNames = task.tags?.map((t) => t.name).filter(Boolean) || []
return {
externalId: task.gid,
title: task.name || 'Untitled',
content,
mimeType: 'text/plain',
sourceUrl: task.permalink_url || undefined,
contentHash: `asana:${task.gid}:${task.modified_at ?? ''}`,
metadata: {
assignee: task.assignee?.name,
completed: task.completed,
lastModified: task.modified_at,
labels: tagNames,
},
}
} catch (error) {
logger.error('Failed to get Asana task', {
externalId,
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const workspaceGid = sourceConfig.workspace as string | undefined
if (!workspaceGid) {
return { valid: false, error: 'Workspace GID is required' }
}
const maxTasks = sourceConfig.maxTasks as string | undefined
if (maxTasks && (Number.isNaN(Number(maxTasks)) || Number(maxTasks) <= 0)) {
return { valid: false, error: 'Max tasks must be a positive number' }
}
try {
await asanaGet<{ data: AsanaWorkspace }>(
accessToken,
`/workspaces/${workspaceGid}`,
VALIDATE_RETRY_OPTIONS
)
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.project === 'string') result.project = metadata.project
if (typeof metadata.assignee === 'string') result.assignee = metadata.assignee
if (typeof metadata.completed === 'boolean') result.completed = metadata.completed
const lastModified = parseTagDate(metadata.lastModified)
if (lastModified) result.lastModified = lastModified
const labels = joinTagArray(metadata.labels)
if (labels) result.labels = labels
return result
},
}
+1
View File
@@ -0,0 +1 @@
export { asanaConnector } from '@/connectors/asana/asana'
+56
View File
@@ -0,0 +1,56 @@
import { AsanaIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const asanaConnectorMeta: ConnectorMeta = {
id: 'asana',
name: 'Asana',
description: 'Sync tasks from Asana',
version: '1.0.0',
icon: AsanaIcon,
auth: { mode: 'oauth', provider: 'asana', requiredScopes: ['default'] },
configFields: [
{
id: 'workspaceSelector',
title: 'Workspace',
type: 'selector',
selectorKey: 'asana.workspaces',
canonicalParamId: 'workspace',
mode: 'basic',
placeholder: 'Select a workspace',
required: true,
},
{
id: 'workspace',
title: 'Workspace GID',
type: 'short-input',
canonicalParamId: 'workspace',
mode: 'advanced',
placeholder: 'e.g. 1234567890',
required: true,
},
{
id: 'project',
title: 'Project GID',
type: 'short-input',
placeholder: 'e.g. 9876543210 (leave empty for all projects)',
required: false,
},
{
id: 'maxTasks',
title: 'Max Tasks',
type: 'short-input',
placeholder: 'e.g. 500 (default: unlimited)',
required: false,
},
],
tagDefinitions: [
{ id: 'project', displayName: 'Project', fieldType: 'text' },
{ id: 'assignee', displayName: 'Assignee', fieldType: 'text' },
{ id: 'completed', displayName: 'Completed', fieldType: 'boolean' },
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
{ id: 'labels', displayName: 'Labels', fieldType: 'text' },
],
}
+601
View File
@@ -0,0 +1,601 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { ashbyConnectorMeta } from '@/connectors/ashby/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { parseTagDate } from '@/connectors/utils'
const logger = createLogger('AshbyConnector')
const ASHBY_API_BASE = 'https://api.ashbyhq.com'
const CANDIDATES_PER_PAGE = 100
const NOTES_PER_PAGE = 100
const FEEDBACK_PER_PAGE = 100
/**
* Hard cap on the number of applications whose interview feedback is fetched for a
* single candidate document. Candidates with many applications are rare, but this
* bounds the number of feedback API calls per `getDocument` invocation.
*/
const MAX_APPLICATIONS_FOR_FEEDBACK = 10
type UnknownRecord = Record<string, unknown>
/**
* Builds the standard Ashby Authorization header. Ashby uses HTTP Basic auth with
* the API key as the username and an empty password, i.e. `Basic base64(apiKey + ':')`.
*/
function ashbyHeaders(accessToken: string): Record<string, string> {
return {
'Content-Type': 'application/json',
Accept: 'application/json; version=1',
Authorization: `Basic ${Buffer.from(`${accessToken}:`).toString('base64')}`,
}
}
interface AshbyEnvelope {
success: boolean
results?: unknown
moreDataAvailable?: boolean
nextCursor?: string | null
errors?: unknown
errorInfo?: { message?: string }
}
/**
* Extracts a human-readable error message from an Ashby error envelope. Ashby returns
* errors as either `errorInfo.message` or an `errors` string array.
*/
function ashbyErrorMessage(data: AshbyEnvelope, fallback: string): string {
if (data.errorInfo?.message) return data.errorInfo.message
if (Array.isArray(data.errors) && data.errors.length > 0) {
return data.errors.map((e) => String(e)).join('; ')
}
return fallback
}
/**
* Executes an Ashby RPC-style POST request and returns the parsed envelope.
* Ashby exposes a flat set of POST endpoints under `https://api.ashbyhq.com`.
*/
async function ashbyPost(
accessToken: string,
endpoint: string,
body: UnknownRecord,
retryOptions?: Parameters<typeof fetchWithRetry>[2]
): Promise<AshbyEnvelope> {
const response = await fetchWithRetry(
`${ASHBY_API_BASE}/${endpoint}`,
{
method: 'POST',
headers: ashbyHeaders(accessToken),
body: JSON.stringify(body),
},
retryOptions
)
if (!response.ok) {
const errorText = await response.text().catch(() => '')
throw new Error(
`Ashby ${endpoint} HTTP error: ${response.status}${errorText ? `${errorText.slice(0, 300)}` : ''}`
)
}
const data = (await response.json()) as AshbyEnvelope
if (!data.success) {
throw new Error(ashbyErrorMessage(data, `Ashby ${endpoint} request failed`))
}
return data
}
interface AshbyCandidateSummary {
id: string
name: string
position: string | null
company: string | null
school: string | null
location: string | null
source: string | null
emailDomain: string | null
profileUrl: string | null
applicationIds: string[]
createdAt: string | null
updatedAt: string | null
}
/**
* Extracts a human-readable location string from an Ashby candidate's `location`
* object. Prefers the API-provided `locationSummary`. Falls back to joining the
* `name` values of the `locationComponents` array (each entry is `{ type, name }`
* ordered city → region → country, per the candidate entity returned by
* `candidate.list`/`candidate.info`). As a final fallback, supports the flat
* `{ city, region, country }` shape used by candidate write inputs.
*/
function extractLocation(raw: UnknownRecord): string | null {
const location = raw.location as UnknownRecord | undefined
if (!location) return null
const summary = location.locationSummary as string | undefined
if (summary?.trim()) return summary.trim()
if (Array.isArray(location.locationComponents)) {
const parts = (location.locationComponents as UnknownRecord[])
.map((c) => c?.name)
.filter((n): n is string => typeof n === 'string' && n.trim().length > 0)
.map((n) => n.trim())
if (parts.length > 0) return parts.join(', ')
}
const parts = [location.city, location.region, location.country]
.filter((p): p is string => typeof p === 'string' && p.trim().length > 0)
.map((p) => p.trim())
return parts.length > 0 ? parts.join(', ') : null
}
/**
* Extracts the source title from an Ashby candidate's `source` object, which
* references the organization's sources list (e.g. "LinkedIn", "Referral").
*/
function extractSource(raw: UnknownRecord): string | null {
const source = raw.source as UnknownRecord | undefined
const title = source?.title as string | undefined
return title?.trim() || null
}
/**
* Extracts the lowercased domain from an Ashby candidate's primary email address
* (`primaryEmailAddress.value`), enabling filtering candidates by email domain.
*/
function extractEmailDomain(raw: UnknownRecord): string | null {
const email = raw.primaryEmailAddress as UnknownRecord | undefined
const value = email?.value as string | undefined
const at = value?.lastIndexOf('@') ?? -1
if (!value || at < 0 || at === value.length - 1) return null
return (
value
.slice(at + 1)
.trim()
.toLowerCase() || null
)
}
/**
* Normalizes a raw Ashby candidate record into the fields this connector cares about.
* Field names mirror the Ashby candidate object returned by `candidate.list` and
* `candidate.info` (`position`, `company`, `school`, `location`, `source`,
* `primaryEmailAddress`, `profileUrl`, `applicationIds`, `createdAt`, `updatedAt`).
* Stage and status live on applications rather than candidates, so they are
* intentionally not surfaced here.
*/
function mapCandidate(raw: unknown): AshbyCandidateSummary {
const c = (raw ?? {}) as UnknownRecord
return {
id: (c.id as string) ?? '',
name: (c.name as string) ?? '',
position: (c.position as string) ?? null,
company: (c.company as string) ?? null,
school: (c.school as string) ?? null,
location: extractLocation(c),
source: extractSource(c),
emailDomain: extractEmailDomain(c),
profileUrl: (c.profileUrl as string) ?? null,
applicationIds: Array.isArray(c.applicationIds) ? (c.applicationIds as string[]) : [],
createdAt: (c.createdAt as string) ?? null,
updatedAt: (c.updatedAt as string) ?? null,
}
}
interface AshbyNote {
content: string | null
authorName: string | null
createdAt: string | null
}
/**
* Maps a raw Ashby candidate note into a plain-text-friendly shape, combining the
* author's first and last name into a single display name.
*/
function mapNote(raw: unknown): AshbyNote {
const n = (raw ?? {}) as UnknownRecord
const author = n.author as UnknownRecord | undefined
const first = (author?.firstName as string) ?? ''
const last = (author?.lastName as string) ?? ''
const authorName = `${first} ${last}`.trim() || (author?.email as string) || null
return {
content: (n.content as string) ?? null,
authorName,
createdAt: (n.createdAt as string) ?? null,
}
}
interface AshbyFeedbackSummary {
submittedByName: string | null
submittedAt: string | null
lines: string[]
}
/**
* Collects `{ field.path -> field.title }` entries from a feedback form definition.
* Ashby's `formDefinition` exposes fields either flat under `fields[]` or grouped
* under `sections[].fields[]`, and individual entries are sometimes wrapped in a
* `{ field }` envelope — all variants are handled.
*/
function collectFieldTitles(formDefinition: UnknownRecord | undefined): Map<string, string> {
const titleByPath = new Map<string, string>()
if (!formDefinition) return titleByPath
const addField = (entry: UnknownRecord): void => {
const field = (entry?.field ?? entry) as UnknownRecord
const path = field?.path as string | undefined
const title = (field?.title as string) || (field?.humanReadablePath as string)
if (path && title) titleByPath.set(path, title)
}
if (Array.isArray(formDefinition.fields)) {
for (const entry of formDefinition.fields as UnknownRecord[]) addField(entry)
}
if (Array.isArray(formDefinition.sections)) {
for (const section of formDefinition.sections as UnknownRecord[]) {
const fields = Array.isArray(section?.fields) ? (section.fields as UnknownRecord[]) : []
for (const entry of fields) addField(entry)
}
}
return titleByPath
}
/**
* Maps a raw Ashby application feedback submission into a flat list of
* `Title: value` lines, resolving each `submittedValues` key (the field's `path`)
* to its human-readable title via the form definition. Falls back to the raw path
* when no title is found.
*/
function mapFeedback(raw: unknown): AshbyFeedbackSummary {
const f = (raw ?? {}) as UnknownRecord
const submittedBy = f.submittedByUser as UnknownRecord | undefined
const first = (submittedBy?.firstName as string) ?? ''
const last = (submittedBy?.lastName as string) ?? ''
const submittedByName = `${first} ${last}`.trim() || (submittedBy?.email as string) || null
const titleByPath = collectFieldTitles(f.formDefinition as UnknownRecord | undefined)
const submittedValues = (f.submittedValues as UnknownRecord | undefined) ?? {}
const lines: string[] = []
for (const [path, value] of Object.entries(submittedValues)) {
if (value == null) continue
const label = titleByPath.get(path) ?? path
const rendered = renderFeedbackValue(value)
if (rendered) lines.push(`${label}: ${rendered}`)
}
const submittedAt =
(f.submittedAt as string) ?? (f.completedAt as string) ?? (f.createdAt as string) ?? null
return { submittedByName, submittedAt, lines }
}
/**
* Renders an arbitrary submitted feedback value (string, number, boolean, or a
* rich-text / structured object) into a single-line plain-text string.
*/
function renderFeedbackValue(value: unknown): string {
if (typeof value === 'string') return value.trim()
if (typeof value === 'number' || typeof value === 'boolean') return String(value)
if (Array.isArray(value)) {
return value
.map((v) => renderFeedbackValue(v))
.filter(Boolean)
.join(', ')
}
if (value && typeof value === 'object') {
const obj = value as UnknownRecord
const label = obj.label ?? obj.value ?? obj.text ?? obj.content
if (typeof label === 'string') return label.trim()
}
return ''
}
/**
* Stable, metadata-based content hash for a candidate document. Identical between the
* listing stub and the fully-fetched document so unchanged candidates are skipped,
* which keeps the `getDocument` re-hydration (notes + feedback fetches) cheap: the
* sync engine only re-hydrates a deferred stub when this hash differs from the stored
* document's hash (see `lib/knowledge/connectors/sync-engine.ts`).
*
* Known limitation — notes/feedback freshness depends on `candidate.updatedAt`.
* Candidate notes (`candidate.listNotes`) and interview feedback
* (`applicationFeedback.list`) are separate Ashby objects, not candidate fields. This
* hash is derived solely from the candidate's own `updatedAt`, so a new note or newly
* submitted feedback is only re-synced if Ashby advances `candidate.updatedAt` as a
* side effect of that write.
*
* As of this writing Ashby's public API docs do not specify what counts as a
* "modification" for `candidate.updatedAt` or for `candidate.list` syncToken
* incremental sync, and no third-party ATS-integration vendor (Merge, Nango, Knit)
* documents it either — so this behavior is unverified. If Ashby does NOT touch
* `candidate.updatedAt` on note/feedback writes, those additions will not be picked up
* until some other candidate field changes; a forced full sync re-hydrates everything
* regardless. No cheaper listing-time signal exists to fold into this hash: the
* `candidate.list` object exposes no note/feedback count, and syncToken carries the
* same unspecified change semantics as `updatedAt`.
*
* Refs:
* - https://developers.ashbyhq.com/reference/candidatelist
* - https://developers.ashbyhq.com/reference/candidatecreatenote
* - https://developers.ashbyhq.com/docs/pagination-and-incremental-sync
*/
function buildContentHash(id: string, updatedAt: string | null): string {
return `ashby:${id}:${updatedAt ?? ''}`
}
/**
* Creates a lightweight document stub from a candidate listing entry. Content is
* deferred and only fetched (via `getDocument`) for new or changed candidates.
*/
function candidateToStub(candidate: AshbyCandidateSummary): ExternalDocument {
return {
externalId: candidate.id,
title: candidate.name || 'Unnamed Candidate',
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: candidate.profileUrl ?? undefined,
contentHash: buildContentHash(candidate.id, candidate.updatedAt),
metadata: candidateMetadata(candidate),
}
}
/**
* Builds the tag-carrying metadata block shared by the listing stub and the
* fully-fetched document, keeping the keys aligned with `mapTags`/`tagDefinitions`.
*/
function candidateMetadata(candidate: AshbyCandidateSummary): Record<string, unknown> {
return {
candidateName: candidate.name,
company: candidate.company,
school: candidate.school,
location: candidate.location,
source: candidate.source,
emailDomain: candidate.emailDomain,
createdAt: candidate.createdAt,
updatedAt: candidate.updatedAt,
}
}
/**
* Fetches all notes for a candidate, following cursor pagination.
*/
async function fetchAllNotes(accessToken: string, candidateId: string): Promise<AshbyNote[]> {
const notes: AshbyNote[] = []
let cursor: string | undefined
let hasMore = true
while (hasMore) {
const body: UnknownRecord = { candidateId, limit: NOTES_PER_PAGE }
if (cursor) body.cursor = cursor
const data = await ashbyPost(accessToken, 'candidate.listNotes', body)
const results = Array.isArray(data.results) ? data.results : []
for (const raw of results) notes.push(mapNote(raw))
cursor = data.nextCursor ?? undefined
hasMore = Boolean(data.moreDataAvailable) && Boolean(cursor)
}
return notes
}
/**
* Fetches all interview feedback submissions for a single application, following
* cursor pagination.
*/
async function fetchFeedbackForApplication(
accessToken: string,
applicationId: string
): Promise<AshbyFeedbackSummary[]> {
const feedback: AshbyFeedbackSummary[] = []
let cursor: string | undefined
let hasMore = true
while (hasMore) {
const body: UnknownRecord = { applicationId, limit: FEEDBACK_PER_PAGE }
if (cursor) body.cursor = cursor
const data = await ashbyPost(accessToken, 'applicationFeedback.list', body)
const results = Array.isArray(data.results) ? data.results : []
for (const raw of results) feedback.push(mapFeedback(raw))
cursor = data.nextCursor ?? undefined
hasMore = Boolean(data.moreDataAvailable) && Boolean(cursor)
}
return feedback
}
/**
* Assembles a candidate's profile, notes, and interview feedback into a single
* plain-text document body for indexing.
*/
function formatCandidateContent(
candidate: AshbyCandidateSummary,
notes: AshbyNote[],
feedback: AshbyFeedbackSummary[]
): string {
const parts: string[] = []
parts.push(`Candidate: ${candidate.name || 'Unnamed Candidate'}`)
if (candidate.position) parts.push(`Current Role: ${candidate.position}`)
if (candidate.company) parts.push(`Current Company: ${candidate.company}`)
if (candidate.school) parts.push(`School: ${candidate.school}`)
if (candidate.location) parts.push(`Location: ${candidate.location}`)
if (candidate.source) parts.push(`Source: ${candidate.source}`)
if (candidate.createdAt) parts.push(`Created: ${candidate.createdAt}`)
if (candidate.updatedAt) parts.push(`Last Updated: ${candidate.updatedAt}`)
const nonEmptyNotes = notes.filter((n) => n.content?.trim())
if (nonEmptyNotes.length > 0) {
parts.push('')
parts.push('--- Notes ---')
for (const note of nonEmptyNotes) {
const header = [note.authorName, note.createdAt].filter(Boolean).join(' — ')
if (header) parts.push(`[${header}]`)
parts.push((note.content ?? '').trim())
parts.push('')
}
}
const nonEmptyFeedback = feedback.filter((f) => f.lines.length > 0)
if (nonEmptyFeedback.length > 0) {
parts.push('--- Interview Feedback ---')
for (const f of nonEmptyFeedback) {
const header = [f.submittedByName, f.submittedAt].filter(Boolean).join(' — ')
if (header) parts.push(`[${header}]`)
for (const line of f.lines) parts.push(line)
parts.push('')
}
}
return parts.join('\n').trim()
}
export const ashbyConnector: ConnectorConfig = {
...ashbyConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const maxCandidates = sourceConfig.maxCandidates ? Number(sourceConfig.maxCandidates) : 0
const createdAfterMs = (() => {
const raw = sourceConfig.createdAfter
if (typeof raw !== 'string' || !raw.trim()) return undefined
const ms = new Date(raw.trim()).getTime()
return Number.isNaN(ms) ? undefined : ms
})()
const prevFetched = (syncContext?.totalCandidatesFetched as number) ?? 0
if (maxCandidates > 0 && prevFetched >= maxCandidates) {
if (syncContext) syncContext.listingCapped = true
return { documents: [], hasMore: false }
}
const body: UnknownRecord = { limit: CANDIDATES_PER_PAGE }
if (cursor) body.cursor = cursor
if (createdAfterMs !== undefined) body.createdAfter = createdAfterMs
logger.info('Listing Ashby candidates', {
cursor: cursor ?? 'initial',
maxCandidates: maxCandidates || 'unlimited',
})
const data = await ashbyPost(accessToken, 'candidate.list', body)
const results = Array.isArray(data.results) ? data.results : []
const candidates = results.map(mapCandidate).filter((c) => c.id)
let documents = candidates.map(candidateToStub)
if (maxCandidates > 0) {
const remaining = Math.max(0, maxCandidates - prevFetched)
if (documents.length > remaining) documents = documents.slice(0, remaining)
}
const totalFetched = prevFetched + documents.length
if (syncContext) syncContext.totalCandidatesFetched = totalFetched
const hitLimit = maxCandidates > 0 && totalFetched >= maxCandidates
if (hitLimit && syncContext) syncContext.listingCapped = true
const nextCursor = data.nextCursor ?? undefined
const hasMore = !hitLimit && Boolean(data.moreDataAvailable) && Boolean(nextCursor)
return {
documents,
nextCursor: hasMore ? nextCursor : undefined,
hasMore,
}
},
getDocument: async (
accessToken: string,
_sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
try {
if (!externalId) return null
const infoData = await ashbyPost(accessToken, 'candidate.info', { id: externalId })
if (!infoData.results) return null
const candidate = mapCandidate(infoData.results)
if (!candidate.id) return null
const notes = await fetchAllNotes(accessToken, candidate.id)
const feedback: AshbyFeedbackSummary[] = []
const applicationIds = candidate.applicationIds.slice(0, MAX_APPLICATIONS_FOR_FEEDBACK)
for (const applicationId of applicationIds) {
try {
const applicationFeedback = await fetchFeedbackForApplication(accessToken, applicationId)
feedback.push(...applicationFeedback)
} catch (error) {
logger.warn('Failed to fetch Ashby feedback for application', {
applicationId,
error: toError(error).message,
})
}
}
const content = formatCandidateContent(candidate, notes, feedback)
if (!content.trim()) return null
return {
externalId: candidate.id,
title: candidate.name || 'Unnamed Candidate',
content,
contentDeferred: false,
mimeType: 'text/plain',
sourceUrl: candidate.profileUrl ?? undefined,
contentHash: buildContentHash(candidate.id, candidate.updatedAt),
metadata: candidateMetadata(candidate),
}
} catch (error) {
logger.warn('Failed to get Ashby candidate', {
externalId,
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const maxCandidates = sourceConfig.maxCandidates as string | undefined
if (maxCandidates && (Number.isNaN(Number(maxCandidates)) || Number(maxCandidates) < 0)) {
return { valid: false, error: 'Max candidates must be a non-negative number' }
}
try {
await ashbyPost(accessToken, 'candidate.list', { limit: 1 }, VALIDATE_RETRY_OPTIONS)
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
const textTags = ['candidateName', 'company', 'school', 'location', 'source', 'emailDomain']
for (const key of textTags) {
const value = metadata[key]
if (typeof value === 'string' && value.trim()) result[key] = value.trim()
}
const createdAt = parseTagDate(metadata.createdAt)
if (createdAt) result.createdAt = createdAt
const updatedAt = parseTagDate(metadata.updatedAt)
if (updatedAt) result.updatedAt = updatedAt
return result
},
}
+1
View File
@@ -0,0 +1 @@
export { ashbyConnector } from '@/connectors/ashby/ashby'
+49
View File
@@ -0,0 +1,49 @@
import { AshbyIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const ashbyConnectorMeta: ConnectorMeta = {
id: 'ashby',
name: 'Ashby',
description: 'Sync candidate notes and interview feedback from Ashby',
version: '1.0.0',
icon: AshbyIcon,
auth: {
mode: 'apiKey',
label: 'API Key',
placeholder: 'Enter your Ashby API key',
},
configFields: [
{
id: 'maxCandidates',
title: 'Max Candidates',
type: 'short-input',
required: false,
placeholder: 'e.g. 500 (default: unlimited)',
description:
'Cap the number of candidates synced. Leave empty to sync ALL candidates in the organization.',
},
{
id: 'createdAfter',
title: 'Created After',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. 2025-01-01 or 2025-01-01T00:00:00Z',
description:
'Only sync candidates created on or after this date (ISO 8601). Leave blank to sync candidates regardless of creation date.',
},
],
tagDefinitions: [
{ id: 'candidateName', displayName: 'Candidate Name', fieldType: 'text' },
{ id: 'company', displayName: 'Current Company', fieldType: 'text' },
{ id: 'school', displayName: 'School', fieldType: 'text' },
{ id: 'location', displayName: 'Location', fieldType: 'text' },
{ id: 'source', displayName: 'Source', fieldType: 'text' },
{ id: 'emailDomain', displayName: 'Email Domain', fieldType: 'text' },
{ id: 'createdAt', displayName: 'Created', fieldType: 'date' },
{ id: 'updatedAt', displayName: 'Last Updated', fieldType: 'date' },
],
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1 @@
export { azureDevopsConnector } from '@/connectors/azure-devops/azure-devops'
+177
View File
@@ -0,0 +1,177 @@
import { AzureIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const azureDevopsConnectorMeta: ConnectorMeta = {
id: 'azure_devops',
name: 'Azure DevOps',
description:
'Sync wiki pages, work items, and repository files from an Azure DevOps project into your knowledge base',
version: '1.1.0',
icon: AzureIcon,
auth: {
mode: 'apiKey',
label: 'Personal Access Token',
placeholder: 'Enter your Azure DevOps PAT (scopes: Wiki Read, Work Items Read, Code Read)',
},
/**
* Incremental sync applies to work items only, via a `System.ChangedDate`
* WIQL filter derived from lastSyncAt. Wiki pages have no change timestamp on
* listing, so they are always re-listed and reconciled by ETag content hash.
* Repository files are likewise always re-listed in full and reconciled by the
* git blob objectId hash — a commit-diff incremental path is intentionally
* avoided to match the github/gitlab full-listing approach, keeping change
* detection correct without tracking per-branch commit state. Unchanged
* documents are skipped without a content fetch in every case.
*/
supportsIncrementalSync: true,
configFields: [
{
id: 'organization',
title: 'Organization',
type: 'short-input',
placeholder: 'e.g. my-org',
required: true,
},
{
id: 'project',
title: 'Project',
type: 'short-input',
placeholder: 'e.g. my-project',
required: true,
},
{
id: 'contentType',
title: 'Content',
type: 'dropdown',
required: false,
options: [
{ label: 'Wiki pages only', id: 'wiki' },
{ label: 'Work items only', id: 'workitems' },
{ label: 'Repository files only', id: 'files' },
{ label: 'Wiki pages and work items', id: 'both' },
{ label: 'Wiki, work items, and files', id: 'all' },
],
description: 'Which content to index from the project.',
},
{
id: 'wikiName',
title: 'Wiki',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'Wiki name or ID (all wikis if blank)',
description:
'Restrict syncing to a single wiki by name or ID. Applies only when syncing wiki pages.',
},
{
id: 'workItemType',
title: 'Work Item Type',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. Bug, Task, User Story',
description: 'Only sync work items of this type. Applies only when syncing work items.',
},
{
id: 'state',
title: 'State',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. Active, Closed',
description: 'Only sync work items in this state. Applies only when syncing work items.',
},
{
id: 'areaPath',
title: 'Area Path',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. MyProject\\Team A',
description:
'Only sync work items under this area path (and its children). Applies only when syncing work items.',
},
{
id: 'workItemTags',
title: 'Tags',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. customer, urgent (comma-separated)',
description:
'Only sync work items containing all of these tags (comma-separated). Applies only when syncing work items.',
},
{
id: 'customWiql',
title: 'Custom WIQL Query',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'SELECT [System.Id] FROM workitems WHERE ...',
description:
'Advanced: a full WIQL query selecting [System.Id]. Overrides the type, state, area path, and tag filters when set. Custom queries always run as full listings on every sync (the incremental changed-date filter is not applied).',
},
{
id: 'repositoryName',
title: 'Repository',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'Repository name or ID (all repos if blank)',
description:
'Restrict syncing to a single repository by name or ID. Applies only when syncing repository files.',
},
{
id: 'branch',
title: 'Branch',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: "Each repo's default branch",
description:
'Branch to sync repository files from. Defaults to each repositorys default branch. Applies only when syncing repository files.',
},
{
id: 'pathPrefix',
title: 'Path Filter',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. docs/, src/',
description:
'Only sync repository files under this path prefix. Applies only when syncing repository files.',
},
{
id: 'fileExtensions',
title: 'File Extensions',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. .md, .txt, .ts',
description:
'Only sync repository files with these extensions (comma-separated). Leave blank for all text files. Applies only when syncing repository files.',
},
{
id: 'maxItems',
title: 'Max Items',
type: 'short-input',
required: false,
placeholder: 'e.g. 500 (default: unlimited)',
},
],
tagDefinitions: [
{ id: 'kind', displayName: 'Type', fieldType: 'text' },
{ id: 'wikiName', displayName: 'Wiki', fieldType: 'text' },
{ id: 'workItemType', displayName: 'Work Item Type', fieldType: 'text' },
{ id: 'state', displayName: 'State', fieldType: 'text' },
{ id: 'areaPath', displayName: 'Area Path', fieldType: 'text' },
{ id: 'tags', displayName: 'Tags', fieldType: 'text' },
{ id: 'repository', displayName: 'Repository', fieldType: 'text' },
{ id: 'path', displayName: 'File Path', fieldType: 'text' },
{ id: 'changedDate', displayName: 'Changed Date', fieldType: 'date' },
],
}
@@ -0,0 +1,31 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import { escapeCql } from '@/connectors/confluence/confluence'
describe('escapeCql', () => {
it.concurrent('returns plain strings unchanged', () => {
expect(escapeCql('Engineering')).toBe('Engineering')
})
it.concurrent('escapes double quotes', () => {
expect(escapeCql('say "hello"')).toBe('say \\"hello\\"')
})
it.concurrent('escapes backslashes', () => {
expect(escapeCql('path\\to\\file')).toBe('path\\\\to\\\\file')
})
it.concurrent('escapes backslashes before quotes', () => {
expect(escapeCql('a\\"b')).toBe('a\\\\\\"b')
})
it.concurrent('handles empty string', () => {
expect(escapeCql('')).toBe('')
})
it.concurrent('leaves other special chars unchanged', () => {
expect(escapeCql("it's a test & <tag>")).toBe("it's a test & <tag>")
})
})
@@ -0,0 +1,629 @@
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { confluenceConnectorMeta } from '@/connectors/confluence/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { htmlToPlainText, joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils'
import { getConfluenceCloudId, normalizeConfluenceDomainHost } from '@/tools/confluence/utils'
const logger = createLogger('ConfluenceConnector')
/**
* Escapes a value for use inside CQL double-quoted strings.
*/
export function escapeCql(value: string): string {
return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"')
}
/**
* Builds a CQL clause restricting content to the given space keys.
* Single key uses `space = "X"`; multiple keys use `space in ("X","Y")`.
*/
function buildSpaceClause(spaceKeys: string[]): string {
if (spaceKeys.length === 1) {
return `space="${escapeCql(spaceKeys[0])}"`
}
const list = spaceKeys.map((k) => `"${escapeCql(k)}"`).join(',')
return `space in (${list})`
}
/**
* Fetches labels for a batch of page IDs using the v2 labels endpoint.
*/
const LABEL_FETCH_CONCURRENCY = 5
async function fetchLabelsForPages(
cloudId: string,
accessToken: string,
pageIds: string[]
): Promise<Map<string, string[]>> {
const labelsByPageId = new Map<string, string[]>()
for (let i = 0; i < pageIds.length; i += LABEL_FETCH_CONCURRENCY) {
const batch = pageIds.slice(i, i + LABEL_FETCH_CONCURRENCY)
const results = await Promise.all(
batch.map(async (pageId) => {
try {
let data: Record<string, unknown> | null = null
for (const contentType of ['pages', 'blogposts']) {
const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/${contentType}/${pageId}/labels`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/json',
Authorization: `Bearer ${accessToken}`,
},
})
if (response.ok) {
data = await response.json()
break
}
if (response.status !== 404) {
logger.warn(`Failed to fetch labels for ${contentType} ${pageId}`, {
status: response.status,
})
}
}
if (!data) {
return { pageId, labels: [] as string[] }
}
const labels = ((data.results as Record<string, unknown>[]) || []).map(
(label) => label.name as string
)
return { pageId, labels }
} catch (error) {
logger.warn(`Error fetching labels for page ${pageId}`, {
error: toError(error).message,
})
return { pageId, labels: [] as string[] }
}
})
)
for (const { pageId, labels } of results) {
labelsByPageId.set(pageId, labels)
}
}
return labelsByPageId
}
/**
* Produces a canonical metadata stub with a deterministic contentHash that
* does not depend on which API surface (v1 CQL or v2) returned the page.
*/
function pageToStub(
page: Record<string, unknown>,
options: {
spaceId?: unknown
labels?: string[]
sourceUrl?: string
} = {}
): ExternalDocument {
const version = page.version as Record<string, unknown> | undefined
const versionNumber = version?.number as number | undefined
const lastModified = (version?.createdAt ?? version?.when ?? '') as string
const versionKey = versionNumber ?? lastModified
return {
externalId: String(page.id),
title: (page.title as string) || 'Untitled',
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: options.sourceUrl,
contentHash: `confluence:${page.id}:${versionKey}`,
metadata: {
spaceId: options.spaceId,
status: page.status,
version: versionNumber,
labels: options.labels ?? [],
lastModified,
},
}
}
/**
* Converts a v1 CQL search result item to a lightweight metadata stub.
*/
function cqlResultToStub(item: Record<string, unknown>, domain: string): ExternalDocument {
const links = item._links as Record<string, string> | undefined
const metadata = item.metadata as Record<string, unknown> | undefined
const labelsWrapper = metadata?.labels as Record<string, unknown> | undefined
const labelResults = (labelsWrapper?.results || []) as Record<string, unknown>[]
const labels = labelResults.map((l) => l.name as string)
return pageToStub(item, {
spaceId: (item.space as Record<string, unknown>)?.key,
labels,
sourceUrl: links?.webui ? `https://${domain}/wiki${links.webui}` : undefined,
})
}
export const confluenceConnector: ConnectorConfig = {
...confluenceConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const domain = normalizeConfluenceDomainHost(sourceConfig.domain as string)
const spaceKeys = parseMultiValue(sourceConfig.spaceKey)
const contentType = (sourceConfig.contentType as string) || 'page'
const labelFilter = (sourceConfig.labelFilter as string) || ''
const maxPages = sourceConfig.maxPages ? Number(sourceConfig.maxPages) : 0
if (spaceKeys.length === 0) {
throw new Error('At least one space key is required')
}
let cloudId = syncContext?.cloudId as string | undefined
if (!cloudId) {
cloudId = await getConfluenceCloudId(domain, accessToken)
if (syncContext) syncContext.cloudId = cloudId
}
/**
* Route through CQL when a label filter is set or when multiple spaces are
* selected — the v2 `/spaces/{spaceId}/pages` endpoint is single-space only,
* but CQL natively supports `space in (...)`.
*/
if (labelFilter.trim() || spaceKeys.length > 1) {
return listDocumentsViaCql(
cloudId,
accessToken,
domain,
spaceKeys,
contentType,
labelFilter,
maxPages,
cursor,
syncContext
)
}
const spaceKey = spaceKeys[0]
let spaceId = syncContext?.spaceId as string | undefined
if (!spaceId) {
spaceId = await resolveSpaceId(cloudId, accessToken, spaceKey)
if (syncContext) syncContext.spaceId = spaceId
}
if (contentType === 'all') {
return listAllContentTypes(
cloudId,
accessToken,
domain,
spaceId,
spaceKey,
maxPages,
cursor,
syncContext
)
}
return listDocumentsV2(
cloudId,
accessToken,
domain,
spaceId,
spaceKey,
contentType,
maxPages,
cursor,
syncContext
)
},
getDocument: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
externalId: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocument | null> => {
const domain = normalizeConfluenceDomainHost(sourceConfig.domain as string)
let cloudId = syncContext?.cloudId as string | undefined
if (!cloudId) {
cloudId = await getConfluenceCloudId(domain, accessToken)
if (syncContext) syncContext.cloudId = cloudId
}
// Try pages first, fall back to blogposts if not found
let page: Record<string, unknown> | null = null
for (const endpoint of ['pages', 'blogposts']) {
const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/${endpoint}/${externalId}?body-format=storage`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/json',
Authorization: `Bearer ${accessToken}`,
},
})
if (response.ok) {
page = await response.json()
break
}
if (response.status !== 404) {
throw new Error(`Failed to get Confluence content: ${response.status}`)
}
}
if (!page) return null
const body = page.body as Record<string, unknown> | undefined
const storage = body?.storage as Record<string, unknown> | undefined
const rawContent = (storage?.value as string) || ''
const plainText = htmlToPlainText(rawContent)
const labelMap = await fetchLabelsForPages(cloudId, accessToken, [String(page.id)])
const labels = labelMap.get(String(page.id)) ?? []
const links = page._links as Record<string, unknown> | undefined
const stub = pageToStub(page, {
spaceId: page.spaceId,
labels,
sourceUrl: links?.webui ? `https://${domain}/wiki${links.webui}` : undefined,
})
return {
...stub,
content: plainText,
contentDeferred: false,
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const domain = sourceConfig.domain as string
const spaceKeys = parseMultiValue(sourceConfig.spaceKey)
if (!domain || spaceKeys.length === 0) {
return { valid: false, error: 'Domain and at least one space key are required' }
}
const maxPages = sourceConfig.maxPages as string | undefined
if (maxPages && (Number.isNaN(Number(maxPages)) || Number(maxPages) <= 0)) {
return { valid: false, error: 'Max pages must be a positive number' }
}
try {
const cloudId = await getConfluenceCloudId(domain, accessToken, VALIDATE_RETRY_OPTIONS)
const params = new URLSearchParams()
for (const key of spaceKeys) params.append('keys', key)
params.append('limit', String(Math.max(spaceKeys.length, 1)))
const spaceUrl = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces?${params.toString()}`
const response = await fetchWithRetry(
spaceUrl,
{
method: 'GET',
headers: {
Accept: 'application/json',
Authorization: `Bearer ${accessToken}`,
},
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
return { valid: false, error: `Failed to validate spaces: ${response.status}` }
}
const data = await response.json()
const results = (data.results as Array<Record<string, unknown>> | undefined) ?? []
const foundKeys = new Set(results.map((r) => String(r.key)))
const missing = spaceKeys.filter((k) => !foundKeys.has(k))
if (missing.length > 0) {
return {
valid: false,
error: `Space${missing.length > 1 ? 's' : ''} not found: ${missing.join(', ')}`,
}
}
return { valid: true }
} catch (error) {
return { valid: false, error: toError(error).message || 'Failed to validate configuration' }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
const joined = joinTagArray(metadata.labels)
if (joined) result.labels = joined
if (metadata.version != null) {
const num = Number(metadata.version)
if (!Number.isNaN(num)) result.version = num
}
const lastModified = parseTagDate(metadata.lastModified)
if (lastModified) result.lastModified = lastModified
return result
},
}
/**
* Lists documents using the v2 API for a single content type (pages or blogposts).
*/
async function listDocumentsV2(
cloudId: string,
accessToken: string,
domain: string,
spaceId: string,
spaceKey: string,
contentType: string,
maxPages: number,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> {
const queryParams = new URLSearchParams()
queryParams.append('limit', '250')
if (cursor) {
queryParams.append('cursor', cursor)
}
const endpoint = contentType === 'blogpost' ? 'blogposts' : 'pages'
const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces/${spaceId}/${endpoint}?${queryParams.toString()}`
logger.info(`Listing ${endpoint} in space ${spaceKey} (ID: ${spaceId})`)
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/json',
Authorization: `Bearer ${accessToken}`,
},
})
if (!response.ok) {
const errorText = await response.text()
logger.error(`Failed to list Confluence ${endpoint}`, {
status: response.status,
error: errorText,
})
throw new Error(`Failed to list Confluence ${endpoint}: ${response.status}`)
}
const data = await response.json()
const results = data.results || []
const documents: ExternalDocument[] = results.map((page: Record<string, unknown>) => {
const links = page._links as Record<string, string> | undefined
return pageToStub(page, {
spaceId: page.spaceId,
sourceUrl: links?.webui ? `https://${domain}/wiki${links.webui}` : undefined,
})
})
let nextCursor: string | undefined
const nextLink = (data._links as Record<string, string>)?.next
if (nextLink) {
try {
nextCursor = new URL(nextLink, 'https://placeholder').searchParams.get('cursor') || undefined
} catch {
// Ignore malformed URLs
}
}
const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = maxPages > 0 && totalFetched >= maxPages
if (hitLimit && syncContext) syncContext.listingCapped = true
return {
documents,
nextCursor: hitLimit ? undefined : nextCursor,
hasMore: hitLimit ? false : Boolean(nextCursor),
}
}
/**
* Lists both pages and blogposts using a compound cursor that tracks
* pagination state for each content type independently.
*/
async function listAllContentTypes(
cloudId: string,
accessToken: string,
domain: string,
spaceId: string,
spaceKey: string,
maxPages: number,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> {
let pageCursor: string | undefined
let blogCursor: string | undefined
let pagesDone = false
let blogsDone = false
if (cursor) {
try {
const parsed = JSON.parse(cursor)
pageCursor = parsed.page
blogCursor = parsed.blog
pagesDone = parsed.pagesDone === true
blogsDone = parsed.blogsDone === true
} catch {
/**
* Older bare-string cursors are no longer emitted; fall through and
* restart instead of silently re-listing blogposts from page 0.
*/
logger.warn('Ignoring unparseable Confluence cursor; restarting listing')
}
}
const results: ExternalDocumentList = { documents: [], hasMore: false }
if (!pagesDone) {
const pagesResult = await listDocumentsV2(
cloudId,
accessToken,
domain,
spaceId,
spaceKey,
'page',
maxPages,
pageCursor,
syncContext
)
results.documents.push(...pagesResult.documents)
pageCursor = pagesResult.nextCursor
pagesDone = !pagesResult.hasMore
}
if (!blogsDone) {
const blogResult = await listDocumentsV2(
cloudId,
accessToken,
domain,
spaceId,
spaceKey,
'blogpost',
maxPages,
blogCursor,
syncContext
)
results.documents.push(...blogResult.documents)
blogCursor = blogResult.nextCursor
blogsDone = !blogResult.hasMore
}
results.hasMore = !pagesDone || !blogsDone
if (results.hasMore) {
results.nextCursor = JSON.stringify({
page: pageCursor,
blog: blogCursor,
pagesDone,
blogsDone,
})
}
return results
}
/**
* Lists documents using CQL search via the v1 API (used when label filtering is enabled).
*/
async function listDocumentsViaCql(
cloudId: string,
accessToken: string,
domain: string,
spaceKeys: string[],
contentType: string,
labelFilter: string,
maxPages: number,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> {
const labels = labelFilter
.split(',')
.map((l) => l.trim())
.filter(Boolean)
// Build CQL query
let cql = buildSpaceClause(spaceKeys)
if (contentType === 'blogpost') {
cql += ' AND type="blogpost"'
} else if (contentType === 'page' || !contentType) {
cql += ' AND type="page"'
}
// contentType === 'all' — no type filter
if (labels.length === 1) {
cql += ` AND label="${escapeCql(labels[0])}"`
} else if (labels.length > 1) {
const labelList = labels.map((l) => `"${escapeCql(l)}"`).join(',')
cql += ` AND label in (${labelList})`
}
const limit = maxPages > 0 ? Math.min(maxPages, 50) : 50
const start = cursor ? Number(cursor) : 0
const queryParams = new URLSearchParams()
queryParams.append('cql', cql)
queryParams.append('limit', String(limit))
queryParams.append('start', String(start))
queryParams.append('expand', 'version,metadata.labels')
const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/rest/api/content/search?${queryParams.toString()}`
logger.info(`Searching Confluence via CQL: ${cql}`, { start, limit })
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/json',
Authorization: `Bearer ${accessToken}`,
},
})
if (!response.ok) {
const errorText = await response.text()
logger.error('Failed to search Confluence via CQL', {
status: response.status,
error: errorText,
})
throw new Error(`Failed to search Confluence via CQL: ${response.status}`)
}
const data = await response.json()
const results = data.results || []
const documents: ExternalDocument[] = results.map((item: Record<string, unknown>) =>
cqlResultToStub(item, domain)
)
const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = maxPages > 0 && totalFetched >= maxPages
if (hitLimit && syncContext) syncContext.listingCapped = true
const totalSize = (data.totalSize as number) ?? 0
const nextStart = start + results.length
const hasMore = !hitLimit && nextStart < totalSize
return {
documents,
nextCursor: hasMore ? String(nextStart) : undefined,
hasMore,
}
}
/**
* Resolves a Confluence space key to its numeric space ID.
*/
async function resolveSpaceId(
cloudId: string,
accessToken: string,
spaceKey: string
): Promise<string> {
const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces?keys=${encodeURIComponent(spaceKey)}&limit=1`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/json',
Authorization: `Bearer ${accessToken}`,
},
})
if (!response.ok) {
throw new Error(`Failed to resolve space key "${spaceKey}": ${response.status}`)
}
const data = await response.json()
const results = data.results || []
if (results.length === 0) {
throw new Error(`Space "${spaceKey}" not found`)
}
return String(results[0].id)
}
+1
View File
@@ -0,0 +1 @@
export { confluenceConnector } from '@/connectors/confluence/confluence'
+87
View File
@@ -0,0 +1,87 @@
import { ConfluenceIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const confluenceConnectorMeta: ConnectorMeta = {
id: 'confluence',
name: 'Confluence',
description: 'Sync pages from a Confluence space',
version: '1.1.0',
icon: ConfluenceIcon,
auth: {
mode: 'oauth',
provider: 'confluence',
requiredScopes: [
'read:confluence-content.all',
'read:page:confluence',
'read:blogpost:confluence',
'read:space:confluence',
'read:label:confluence',
'search:confluence',
'offline_access',
],
},
configFields: [
{
id: 'domain',
title: 'Confluence Domain',
type: 'short-input',
placeholder: 'yoursite.atlassian.net',
required: true,
},
{
id: 'spaceSelector',
title: 'Spaces',
type: 'selector',
selectorKey: 'confluence.spaces',
canonicalParamId: 'spaceKey',
mode: 'basic',
multi: true,
dependsOn: ['domain'],
placeholder: 'Select one or more spaces',
required: true,
},
{
id: 'spaceKey',
title: 'Space Keys',
type: 'short-input',
canonicalParamId: 'spaceKey',
mode: 'advanced',
multi: true,
placeholder: 'e.g. ENG, PRODUCT (comma-separated for multiple)',
required: true,
},
{
id: 'contentType',
title: 'Content Type',
type: 'dropdown',
required: false,
options: [
{ label: 'Pages only', id: 'page' },
{ label: 'Blog posts only', id: 'blogpost' },
{ label: 'All content', id: 'all' },
],
},
{
id: 'labelFilter',
title: 'Filter by Label',
type: 'short-input',
required: false,
placeholder: 'e.g. published, engineering',
},
{
id: 'maxPages',
title: 'Max Pages',
type: 'short-input',
required: false,
placeholder: 'e.g. 500 (default: unlimited)',
},
],
tagDefinitions: [
{ id: 'labels', displayName: 'Labels', fieldType: 'text' },
{ id: 'version', displayName: 'Version', fieldType: 'number' },
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
],
}
+302
View File
@@ -0,0 +1,302 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { DEFAULT_MAX_MESSAGES, discordConnectorMeta } from '@/connectors/discord/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { computeContentHash, parseTagDate } from '@/connectors/utils'
const logger = createLogger('DiscordConnector')
const DISCORD_API_BASE = 'https://discord.com/api/v10'
const MESSAGES_PER_PAGE = 100
interface DiscordMessage {
id: string
channel_id: string
author: {
id: string
username: string
discriminator?: string
bot?: boolean
}
content: string
timestamp: string
edited_timestamp?: string | null
type: number
}
interface DiscordChannel {
id: string
name?: string
topic?: string | null
guild_id?: string
type: number
}
/**
* Calls the Discord REST API with Bot token auth.
* Unlike Slack, Discord returns proper HTTP status codes for errors.
*/
async function discordApiGet(
path: string,
botToken: string,
params?: Record<string, string>,
retryOptions?: Parameters<typeof fetchWithRetry>[2]
): Promise<unknown> {
const queryParams = params ? `?${new URLSearchParams(params).toString()}` : ''
const url = `${DISCORD_API_BASE}${path}${queryParams}`
const response = await fetchWithRetry(
url,
{
method: 'GET',
headers: {
Authorization: `Bot ${botToken}`,
Accept: 'application/json',
},
},
retryOptions
)
if (!response.ok) {
const body = await response.text().catch(() => '')
throw new Error(`Discord API error ${response.status}: ${body}`)
}
return response.json()
}
/**
* Fetches all messages from a channel, up to a maximum count, using `before`-based pagination.
* Discord returns messages newest-first; we collect them all then reverse for chronological order.
*/
async function fetchChannelMessages(
botToken: string,
channelId: string,
maxMessages: number
): Promise<{ messages: DiscordMessage[]; lastActivityTs?: string }> {
const allMessages: DiscordMessage[] = []
let beforeId: string | undefined
let lastActivityTs: string | undefined
while (allMessages.length < maxMessages) {
const limit = Math.min(MESSAGES_PER_PAGE, maxMessages - allMessages.length)
const params: Record<string, string> = { limit: String(limit) }
if (beforeId) {
params.before = beforeId
}
const messages = (await discordApiGet(
`/channels/${channelId}/messages`,
botToken,
params
)) as DiscordMessage[]
if (!messages || messages.length === 0) break
if (!lastActivityTs && messages.length > 0) {
lastActivityTs = messages[0].timestamp
}
allMessages.push(...messages)
// The last message in the batch is the oldest; use its ID for the next page
beforeId = messages[messages.length - 1].id
// If we got fewer than requested, there are no more messages
if (messages.length < limit) break
}
return { messages: allMessages.slice(0, maxMessages), lastActivityTs }
}
/**
* Converts fetched messages into a single document content string.
* Each line: "[ISO timestamp] username: message content"
* Messages are returned chronologically (oldest first).
*/
function formatMessages(messages: DiscordMessage[]): string {
const lines: string[] = []
// Discord returns newest first; reverse for chronological order
const chronological = [...messages].reverse()
for (const msg of chronological) {
// Skip system messages (type 0 = DEFAULT, type 19 = REPLY are user messages)
if (msg.type !== 0 && msg.type !== 19) continue
if (!msg.content) continue
const userName = msg.author.username
lines.push(`[${msg.timestamp}] ${userName}: ${msg.content}`)
}
return lines.join('\n')
}
export const discordConnector: ConnectorConfig = {
...discordConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
_cursor?: string,
_syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const channelId = sourceConfig.channelId as string
if (!channelId?.trim()) {
throw new Error('Channel ID is required')
}
const maxMessages = sourceConfig.maxMessages
? Number(sourceConfig.maxMessages)
: DEFAULT_MAX_MESSAGES
logger.info('Syncing Discord channel', { channelId, maxMessages })
const channel = (await discordApiGet(
`/channels/${channelId.trim()}`,
accessToken
)) as DiscordChannel
const { messages, lastActivityTs } = await fetchChannelMessages(
accessToken,
channel.id,
maxMessages
)
const content = formatMessages(messages)
if (!content.trim()) {
logger.info('No messages found in Discord channel', { channelId: channel.id })
return { documents: [], hasMore: false }
}
const contentHash = await computeContentHash(content)
const channelName = channel.name || channel.id
const sourceUrl = `https://discord.com/channels/${channel.guild_id || '@me'}/${channel.id}`
const document: ExternalDocument = {
externalId: channel.id,
title: `#${channelName}`,
content,
mimeType: 'text/plain',
sourceUrl,
contentHash,
metadata: {
channelName,
messageCount: messages.length,
lastActivity: lastActivityTs,
topic: channel.topic ?? undefined,
},
}
return {
documents: [document],
hasMore: false,
}
},
getDocument: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
const maxMessages = sourceConfig.maxMessages
? Number(sourceConfig.maxMessages)
: DEFAULT_MAX_MESSAGES
try {
const channel = (await discordApiGet(
`/channels/${externalId}`,
accessToken
)) as DiscordChannel
const { messages, lastActivityTs } = await fetchChannelMessages(
accessToken,
externalId,
maxMessages
)
const content = formatMessages(messages)
if (!content.trim()) return null
const contentHash = await computeContentHash(content)
const channelName = channel.name || channel.id
const sourceUrl = `https://discord.com/channels/${channel.guild_id || '@me'}/${channel.id}`
return {
externalId: channel.id,
title: `#${channelName}`,
content,
mimeType: 'text/plain',
sourceUrl,
contentHash,
metadata: {
channelName,
messageCount: messages.length,
lastActivity: lastActivityTs,
topic: channel.topic ?? undefined,
},
}
} catch (error) {
logger.warn('Failed to get Discord channel document', {
externalId,
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const channelId = sourceConfig.channelId as string | undefined
const maxMessages = sourceConfig.maxMessages as string | undefined
if (!channelId?.trim()) {
return { valid: false, error: 'Channel ID is required' }
}
if (maxMessages && (Number.isNaN(Number(maxMessages)) || Number(maxMessages) <= 0)) {
return { valid: false, error: 'Max messages must be a positive number' }
}
try {
await discordApiGet(
`/channels/${channelId.trim()}`,
accessToken,
undefined,
VALIDATE_RETRY_OPTIONS
)
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
if (message.includes('401') || message.includes('403')) {
return { valid: false, error: 'Invalid bot token or missing permissions for this channel' }
}
if (message.includes('404')) {
return { valid: false, error: `Channel not found: ${channelId}` }
}
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.channelName === 'string') {
result.channelName = metadata.channelName
}
if (typeof metadata.messageCount === 'number') {
result.messageCount = metadata.messageCount
}
const lastActivity = parseTagDate(metadata.lastActivity)
if (lastActivity) {
result.lastActivity = lastActivity
}
return result
},
}
+1
View File
@@ -0,0 +1 @@
export { discordConnector } from '@/connectors/discord/discord'
+42
View File
@@ -0,0 +1,42 @@
import { DiscordIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const DEFAULT_MAX_MESSAGES = 1000
export const discordConnectorMeta: ConnectorMeta = {
id: 'discord',
name: 'Discord',
description: 'Sync channel messages from Discord',
version: '1.0.0',
icon: DiscordIcon,
auth: {
mode: 'apiKey',
label: 'Bot Token',
placeholder: 'Enter your Discord bot token',
},
configFields: [
{
id: 'channelId',
title: 'Channel ID',
type: 'short-input',
placeholder: 'e.g. 123456789012345678',
required: true,
description: 'The Discord channel ID to sync messages from',
},
{
id: 'maxMessages',
title: 'Max Messages',
type: 'short-input',
required: false,
placeholder: `e.g. 500 (default: ${DEFAULT_MAX_MESSAGES})`,
},
],
tagDefinitions: [
{ id: 'channelName', displayName: 'Channel Name', fieldType: 'text' },
{ id: 'messageCount', displayName: 'Message Count', fieldType: 'number' },
{ id: 'lastActivity', displayName: 'Last Activity', fieldType: 'date' },
],
}
+620
View File
@@ -0,0 +1,620 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { docusignConnectorMeta } from '@/connectors/docusign/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { parseTagDate } from '@/connectors/utils'
const logger = createLogger('DocuSignConnector')
/**
* DocuSign OAuth userinfo endpoint. Sim's DocuSign OAuth integration is wired to the
* demo/sandbox authorization server (`account-d.docusign.com`, see lib/oauth/oauth.ts token
* endpoint), so the connector resolves account info from the matching demo userinfo host.
* The production host is `https://account.docusign.com/oauth/userinfo`.
*/
const DOCUSIGN_USERINFO_URL = 'https://account-d.docusign.com/oauth/userinfo'
/**
* DocuSign web-app base for envelope deep links. MUST match the same environment as
* {@link DOCUSIGN_USERINFO_URL}: demo/sandbox envelopes only exist in the demo web app
* (`appdemo.docusign.com`), not production (`app.docusign.com`). Keep these in lockstep
* if the OAuth environment ever changes.
*/
const DOCUSIGN_WEB_BASE = 'https://appdemo.docusign.com'
const DEFAULT_LOOKBACK_DAYS = 90
const MAX_PAGE_SIZE = 100
const DEFAULT_MAX_ENVELOPES = 0
/**
* Days of overlap added to the incremental sync window. DocuSign status changes are
* indexed by `statusChangedDateTime`, but webhook/processing lag and clock skew can let
* a change land slightly before the recorded sync time. A small overlap re-scans recent
* envelopes so late-recorded changes are not missed.
*/
const INCREMENTAL_OVERLAP_DAYS = 2
const MS_PER_DAY = 24 * 60 * 60 * 1000
interface DocuSignAccount {
account_id?: string
base_uri?: string
is_default?: boolean
}
interface DocuSignUserInfo {
accounts?: DocuSignAccount[]
}
interface ResolvedAccount {
accountId: string
baseUri: string
}
interface DocuSignSigner {
recipientId?: string
name?: string
email?: string
status?: string
}
interface DocuSignRecipients {
signers?: DocuSignSigner[]
carbonCopies?: DocuSignSigner[]
agents?: DocuSignSigner[]
editors?: DocuSignSigner[]
certifiedDeliveries?: DocuSignSigner[]
}
interface DocuSignCustomField {
name?: string
value?: string
}
interface DocuSignCustomFields {
textCustomFields?: DocuSignCustomField[]
listCustomFields?: DocuSignCustomField[]
}
interface DocuSignDocument {
documentId?: string
name?: string
}
interface DocuSignEnvelope {
envelopeId?: string
status?: string
emailSubject?: string
emailBlurb?: string
sentDateTime?: string
completedDateTime?: string
createdDateTime?: string
statusChangedDateTime?: string
lastModifiedDateTime?: string
sender?: { userName?: string; email?: string }
recipients?: DocuSignRecipients
customFields?: DocuSignCustomFields
envelopeDocuments?: DocuSignDocument[]
}
interface DocuSignEnvelopesListResponse {
envelopes?: DocuSignEnvelope[]
resultSetSize?: string
totalSetSize?: string
endPosition?: string
nextUri?: string
}
interface DocuSignFormValue {
name?: string
value?: string
}
interface DocuSignRecipientFormData {
formData?: DocuSignFormValue[]
}
/**
* Response shape of the envelope `form_data` endpoint. The envelope-level entered tab
* values are returned as a top-level `formData` array of `{ name, value }` pairs (there
* is no nested `formValues` property). Per-recipient values live under `recipientFormData`.
*/
interface DocuSignFormData {
formData?: DocuSignFormValue[]
recipientFormData?: DocuSignRecipientFormData[]
}
/**
* Formats a Date as a UTC ISO 8601 string with explicit time zone offset, the format
* DocuSign recommends for the `from_date` filter on listStatusChanges.
*/
function formatFromDate(date: Date): string {
return date.toISOString()
}
/**
* Computes the effective lookback window in days, narrowing to the time since the last
* successful sync (plus an overlap to catch late-recorded status changes) when incremental
* sync is active.
*/
function computeLookbackDays(
sourceConfig: Record<string, unknown>,
lastSyncAt: Date | undefined
): number {
const raw = sourceConfig.lookback as string | undefined
const configured = Number(raw)
const baseline =
Number.isFinite(configured) && configured > 0 ? Math.floor(configured) : DEFAULT_LOOKBACK_DAYS
if (!lastSyncAt) return baseline
const sinceLastSync = Math.ceil((Date.now() - lastSyncAt.getTime()) / MS_PER_DAY)
const incremental = Math.max(sinceLastSync + INCREMENTAL_OVERLAP_DAYS, INCREMENTAL_OVERLAP_DAYS)
return Math.min(incremental, baseline)
}
/**
* Resolves and caches the user's DocuSign account ID and base URI in the sync context.
* The userinfo lookup is expensive and identical across every page of a sync run, so it is
* cached on the shared `syncContext` (mirrors the Gmail connector's label cache pattern).
*/
async function resolveAccount(
accessToken: string,
syncContext: Record<string, unknown> | undefined,
retryOptions?: Parameters<typeof fetchWithRetry>[2]
): Promise<ResolvedAccount> {
const cacheKey = '_docusignAccount'
const cached = syncContext?.[cacheKey] as ResolvedAccount | undefined
if (cached) return cached
const response = await fetchWithRetry(
DOCUSIGN_USERINFO_URL,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
},
retryOptions
)
if (!response.ok) {
const errorText = await response.text().catch(() => '')
throw new Error(
`Failed to resolve DocuSign account: ${response.status}${
errorText ? `${errorText.slice(0, 200)}` : ''
}`
)
}
const data = (await response.json()) as DocuSignUserInfo
const accounts = Array.isArray(data.accounts) ? data.accounts : []
const account = accounts.find((a) => a.is_default) ?? accounts[0]
if (!account?.account_id || !account.base_uri) {
throw new Error('No accessible DocuSign account found for this user')
}
const resolved: ResolvedAccount = {
accountId: account.account_id,
baseUri: account.base_uri,
}
if (syncContext) syncContext[cacheKey] = resolved
return resolved
}
/**
* Builds the REST API base for an account, e.g.
* `https://demo.docusign.net/restapi/v2.1/accounts/{accountId}`.
*/
function apiBaseFor(account: ResolvedAccount): string {
return `${account.baseUri}/restapi/v2.1/accounts/${account.accountId}`
}
/**
* Builds a metadata-based content hash. Identical between the listDocuments stub and the
* getDocument result so the sync engine can detect changes without downloading content.
*/
function buildContentHash(envelope: DocuSignEnvelope): string {
const envelopeId = envelope.envelopeId ?? ''
const changeIndicator =
envelope.statusChangedDateTime ?? envelope.lastModifiedDateTime ?? envelope.status ?? ''
return `docusign:${envelopeId}:${changeIndicator}`
}
/**
* Builds a DocuSign web console link to the envelope, when an envelope ID is present.
*/
function buildSourceUrl(envelopeId: string | undefined): string | undefined {
if (!envelopeId) return undefined
return `${DOCUSIGN_WEB_BASE}/documents/details/${envelopeId}`
}
/**
* Resolves the sender's display name. The envelope summary exposes sender details only via
* the nested `sender` (userInfo) object — there is no top-level `senderName` field.
*/
function resolveSenderName(envelope: DocuSignEnvelope): string | undefined {
return envelope.sender?.userName ?? undefined
}
/**
* Resolves the sender's email. As with the name, this is only available on the nested
* `sender` (userInfo) object.
*/
function resolveSenderEmail(envelope: DocuSignEnvelope): string | undefined {
return envelope.sender?.email ?? undefined
}
/**
* Collects all named recipients across roles into a flat list of name/email/status entries.
*/
function collectRecipients(
recipients: DocuSignRecipients | undefined
): { name: string; email: string; status: string }[] {
if (!recipients) return []
const groups = [
recipients.signers,
recipients.agents,
recipients.editors,
recipients.carbonCopies,
recipients.certifiedDeliveries,
]
const out: { name: string; email: string; status: string }[] = []
for (const group of groups) {
if (!Array.isArray(group)) continue
for (const r of group) {
out.push({
name: r.name?.trim() ?? '',
email: r.email?.trim() ?? '',
status: r.status?.trim() ?? '',
})
}
}
return out
}
/**
* Builds shared metadata used by both the stub and the full document (and by mapTags).
*
* Every field consumed by `mapTags` is read from the envelope object that the list
* endpoint (`GET /envelopes?from_date=...`) returns directly on each list entry — the
* list response carries the full envelope summary (status, subject, sender, and all
* lifecycle timestamps), not a thin stub. The sync engine calls `mapTags` on this stub
* metadata, so tags are populated without any per-envelope detail fetch.
*/
function buildMetadata(envelope: DocuSignEnvelope): Record<string, unknown> {
const recipients = collectRecipients(envelope.recipients)
return {
status: envelope.status,
subject: envelope.emailSubject,
senderName: resolveSenderName(envelope),
senderEmail: resolveSenderEmail(envelope),
sentDate: envelope.sentDateTime,
completedDate: envelope.completedDateTime,
recipientNames: recipients.map((r) => r.name).filter(Boolean),
}
}
/**
* Creates a lightweight document stub from an envelope list entry. No per-envelope API
* calls are made — content is fetched lazily by getDocument for new/changed envelopes only.
*/
function envelopeToStub(envelope: DocuSignEnvelope): ExternalDocument {
return {
externalId: envelope.envelopeId ?? '',
title: envelope.emailSubject?.trim() || 'Untitled DocuSign Envelope',
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: buildSourceUrl(envelope.envelopeId),
contentHash: buildContentHash(envelope),
metadata: buildMetadata(envelope),
}
}
/**
* Renders the envelope's text metadata into a plain-text document. Envelope documents are
* PDFs/binary and are intentionally NOT downloaded or text-extracted — only the document
* names are indexed alongside subject, status, sender, recipients, custom fields, and form
* data field name/value pairs.
*/
function buildContent(
envelope: DocuSignEnvelope,
customFields: DocuSignCustomField[],
documents: DocuSignDocument[],
formValues: DocuSignFormValue[]
): string {
const parts: string[] = []
if (envelope.emailSubject) parts.push(`Subject: ${envelope.emailSubject}`)
if (envelope.emailBlurb) parts.push(`Message: ${envelope.emailBlurb}`)
if (envelope.status) parts.push(`Status: ${envelope.status}`)
const senderName = resolveSenderName(envelope)
const senderEmail = resolveSenderEmail(envelope)
if (senderName || senderEmail) {
parts.push(`Sender: ${[senderName, senderEmail].filter(Boolean).join(' ')}`.trim())
}
if (envelope.sentDateTime) parts.push(`Sent: ${envelope.sentDateTime}`)
if (envelope.completedDateTime) parts.push(`Completed: ${envelope.completedDateTime}`)
const recipients = collectRecipients(envelope.recipients)
if (recipients.length > 0) {
parts.push('')
parts.push('--- Recipients ---')
for (const r of recipients) {
const label = [r.name, r.email ? `<${r.email}>` : '', r.status ? `(${r.status})` : '']
.filter(Boolean)
.join(' ')
if (label) parts.push(label)
}
}
const fields = customFields.filter((f) => f.name?.trim())
if (fields.length > 0) {
parts.push('')
parts.push('--- Custom Fields ---')
for (const f of fields) {
parts.push(`${f.name}: ${f.value ?? ''}`)
}
}
const docNames = documents.map((d) => d.name?.trim()).filter((n): n is string => Boolean(n))
if (docNames.length > 0) {
parts.push('')
parts.push('--- Documents ---')
for (const name of docNames) parts.push(name)
}
const formPairs = formValues.filter((v) => v.name?.trim())
if (formPairs.length > 0) {
parts.push('')
parts.push('--- Form Data ---')
for (const v of formPairs) {
parts.push(`${v.name}: ${v.value ?? ''}`)
}
}
return parts.join('\n').trim()
}
/**
* Fetches the envelope's form data (signer-entered tab values). Returns an empty list on
* 404 or any error — form data is supplementary and a missing endpoint must not fail the doc.
*/
async function fetchFormValues(
apiBase: string,
accessToken: string,
envelopeId: string
): Promise<DocuSignFormValue[]> {
try {
const response = await fetchWithRetry(`${apiBase}/envelopes/${envelopeId}/form_data`, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) return []
const data = (await response.json()) as DocuSignFormData
const values: DocuSignFormValue[] = []
if (Array.isArray(data.formData)) values.push(...data.formData)
if (Array.isArray(data.recipientFormData)) {
for (const recipient of data.recipientFormData) {
if (Array.isArray(recipient.formData)) values.push(...recipient.formData)
}
}
return values
} catch (error) {
logger.warn('Failed to fetch DocuSign form data', {
envelopeId,
error: toError(error).message,
})
return []
}
}
export const docusignConnector: ConnectorConfig = {
...docusignConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>,
lastSyncAt?: Date
): Promise<ExternalDocumentList> => {
const account = await resolveAccount(accessToken, syncContext)
const apiBase = apiBaseFor(account)
const lookbackDays = computeLookbackDays(sourceConfig, lastSyncAt)
const maxEnvelopes = sourceConfig.maxEnvelopes
? Number(sourceConfig.maxEnvelopes)
: DEFAULT_MAX_ENVELOPES
const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
if (maxEnvelopes > 0 && prevFetched >= maxEnvelopes) {
return { documents: [], hasMore: false }
}
const startPosition = cursor ? Number(cursor) : 0
const cachedFromDate = syncContext?.docusignFromDate as string | undefined
const fromDate = cachedFromDate
? new Date(cachedFromDate)
: new Date(Date.now() - lookbackDays * MS_PER_DAY)
if (syncContext && !cachedFromDate) syncContext.docusignFromDate = fromDate.toISOString()
const queryParams = new URLSearchParams({
from_date: formatFromDate(fromDate),
include: 'recipients,custom_fields',
count: String(MAX_PAGE_SIZE),
start_position: String(startPosition),
})
const statusFilter = typeof sourceConfig.status === 'string' ? sourceConfig.status.trim() : ''
if (statusFilter) queryParams.set('status', statusFilter)
const url = `${apiBase}/envelopes?${queryParams.toString()}`
logger.info('Listing DocuSign envelopes', {
from: formatFromDate(fromDate),
startPosition,
incremental: Boolean(lastSyncAt),
})
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) {
const errorText = await response.text().catch(() => '')
logger.error('Failed to list DocuSign envelopes', {
status: response.status,
error: errorText.slice(0, 500),
})
throw new Error(`Failed to list DocuSign envelopes: ${response.status}`)
}
const data = (await response.json()) as DocuSignEnvelopesListResponse
const envelopes = (data.envelopes ?? []).filter((e) => e.envelopeId)
const pageDocuments = envelopes.map(envelopeToStub)
let documents = pageDocuments
if (maxEnvelopes > 0) {
const remaining = Math.max(0, maxEnvelopes - prevFetched)
if (pageDocuments.length > remaining) {
documents = pageDocuments.slice(0, remaining)
}
}
const totalFetched = prevFetched + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = maxEnvelopes > 0 && totalFetched >= maxEnvelopes
if (hitLimit && syncContext) syncContext.listingCapped = true
const endPosition = Number(data.endPosition)
const totalSetSize = Number(data.totalSetSize)
const hasNextPage =
pageDocuments.length === MAX_PAGE_SIZE &&
Number.isFinite(endPosition) &&
Number.isFinite(totalSetSize) &&
endPosition + 1 < totalSetSize
return {
documents,
nextCursor: !hitLimit && hasNextPage ? String(endPosition + 1) : undefined,
hasMore: !hitLimit && hasNextPage,
}
},
getDocument: async (
accessToken: string,
_sourceConfig: Record<string, unknown>,
externalId: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocument | null> => {
try {
if (!externalId) return null
const account = await resolveAccount(accessToken, syncContext)
const apiBase = apiBaseFor(account)
const response = await fetchWithRetry(
`${apiBase}/envelopes/${externalId}?include=recipients,custom_fields,documents`,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
}
)
if (!response.ok) {
if (response.status === 404 || response.status === 410) return null
throw new Error(`Failed to fetch DocuSign envelope: ${response.status}`)
}
const envelope = (await response.json()) as DocuSignEnvelope
if (!envelope.envelopeId) return null
const customFields: DocuSignCustomField[] = [
...(envelope.customFields?.textCustomFields ?? []),
...(envelope.customFields?.listCustomFields ?? []),
]
const documents = Array.isArray(envelope.envelopeDocuments) ? envelope.envelopeDocuments : []
const formValues = await fetchFormValues(apiBase, accessToken, externalId)
const content = buildContent(envelope, customFields, documents, formValues)
if (!content.trim()) return null
return {
externalId: envelope.envelopeId,
title: envelope.emailSubject?.trim() || 'Untitled DocuSign Envelope',
content,
contentDeferred: false,
mimeType: 'text/plain',
sourceUrl: buildSourceUrl(envelope.envelopeId),
contentHash: buildContentHash(envelope),
metadata: buildMetadata(envelope),
}
} catch (error) {
logger.warn('Failed to get DocuSign envelope', {
externalId,
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const maxEnvelopes = sourceConfig.maxEnvelopes as string | undefined
if (maxEnvelopes && (Number.isNaN(Number(maxEnvelopes)) || Number(maxEnvelopes) < 0)) {
return { valid: false, error: 'Max envelopes must be a non-negative number' }
}
try {
await resolveAccount(accessToken, undefined, VALIDATE_RETRY_OPTIONS)
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.status === 'string' && metadata.status.trim()) {
result.status = metadata.status
}
const sender = [metadata.senderName, metadata.senderEmail]
.filter((v): v is string => typeof v === 'string' && v.trim().length > 0)
.join(' ')
.trim()
if (sender) result.sender = sender
if (typeof metadata.subject === 'string' && metadata.subject.trim()) {
result.subject = metadata.subject.trim()
}
const sentDate = parseTagDate(metadata.sentDate)
if (sentDate) result.sentDate = sentDate
const completedDate = parseTagDate(metadata.completedDate)
if (completedDate) result.completedDate = completedDate
return result
},
}
+1
View File
@@ -0,0 +1 @@
export { docusignConnector } from '@/connectors/docusign/docusign'
+68
View File
@@ -0,0 +1,68 @@
import { DocuSignIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const docusignConnectorMeta: ConnectorMeta = {
id: 'docusign',
name: 'DocuSign',
description: 'Sync envelope and agreement metadata from DocuSign into your knowledge base',
version: '1.0.0',
icon: DocuSignIcon,
auth: {
mode: 'oauth',
provider: 'docusign',
requiredScopes: ['signature'],
},
supportsIncrementalSync: true,
configFields: [
{
id: 'lookback',
title: 'Date Range',
type: 'dropdown',
required: false,
options: [
{ label: 'Last 30 days', id: '30' },
{ label: 'Last 90 days (recommended)', id: '90' },
{ label: 'Last 6 months', id: '180' },
],
description:
'On initial sync only. Filters envelopes by when their status last changed (from_date).',
},
{
id: 'status',
title: 'Filter by Status',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. completed (or completed,sent)',
description:
'Only sync envelopes with these statuses (comma-separated: created, sent, delivered, completed, declined, voided). Leave blank to sync all.',
},
{
id: 'maxEnvelopes',
title: 'Max Envelopes',
type: 'short-input',
required: false,
placeholder: 'e.g. 200 (default: unlimited)',
},
],
/**
* Tag definitions are constrained by the document table's slot pools: 7 text slots but
* only 2 date slots (`date1`, `date2`). The two highest-value envelope dates — when it was
* sent and when it completed — claim both date slots. `createdDateTime` is intentionally
* NOT exposed as a date tag: it nearly always equals `sentDateTime` for sent envelopes, so
* adding it would consume a (non-existent) third date slot and be silently dropped by the
* slot allocator. `emailSubject` is exposed as a filterable text tag (distinct from the
* document title) since text slots are plentiful.
*/
tagDefinitions: [
{ id: 'status', displayName: 'Status', fieldType: 'text' },
{ id: 'sender', displayName: 'Sender', fieldType: 'text' },
{ id: 'subject', displayName: 'Subject', fieldType: 'text' },
{ id: 'sentDate', displayName: 'Sent Date', fieldType: 'date' },
{ id: 'completedDate', displayName: 'Completed Date', fieldType: 'date' },
],
}
+327
View File
@@ -0,0 +1,327 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { dropboxConnectorMeta } from '@/connectors/dropbox/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import {
CONNECTOR_MAX_FILE_BYTES,
ConnectorFileTooLargeError,
htmlToPlainText,
isSkippedDocument,
markSkipped,
parseTagDate,
readBodyWithLimit,
sizeLimitSkipReason,
stubOrSkipBySize,
takeIndexableWithinCap,
} from '@/connectors/utils'
const logger = createLogger('DropboxConnector')
const SUPPORTED_EXTENSIONS = new Set([
'.txt',
'.md',
'.markdown',
'.html',
'.htm',
'.csv',
'.json',
'.xml',
'.yaml',
'.yml',
'.log',
'.rst',
'.tsv',
])
const MAX_FILE_SIZE = CONNECTOR_MAX_FILE_BYTES
interface DropboxFileEntry {
'.tag': 'file' | 'folder' | 'deleted'
id: string
name: string
path_lower: string
path_display: string
client_modified?: string
server_modified?: string
size?: number
content_hash?: string
is_downloadable?: boolean
}
interface DropboxListFolderResponse {
entries: DropboxFileEntry[]
cursor: string
has_more: boolean
}
function hasSupportedExtension(name: string): boolean {
const lower = name.toLowerCase()
const dotIndex = lower.lastIndexOf('.')
if (dotIndex === -1) return false
return SUPPORTED_EXTENSIONS.has(lower.slice(dotIndex))
}
/** A downloadable file with a supported extension, regardless of size. */
function isDownloadableFile(entry: DropboxFileEntry): boolean {
return (
entry['.tag'] === 'file' && entry.is_downloadable !== false && hasSupportedExtension(entry.name)
)
}
async function downloadFileContent(accessToken: string, filePath: string): Promise<string> {
const response = await fetchWithRetry('https://content.dropboxapi.com/2/files/download', {
method: 'POST',
headers: {
Authorization: `Bearer ${accessToken}`,
'Dropbox-API-Arg': JSON.stringify({ path: filePath }),
},
})
if (!response.ok) {
throw new Error(`Failed to download file ${filePath}: ${response.status}`)
}
// Stream with a hard byte cap so a file whose listing metadata under-reported
// (or omitted) its size can never be fully buffered into memory. Oversize raises
// so getDocument can surface it as a skipped (failed) row rather than dropping it.
const buffer = await readBodyWithLimit(response, MAX_FILE_SIZE)
if (!buffer) {
throw new ConnectorFileTooLargeError(MAX_FILE_SIZE)
}
const text = buffer.toString('utf8')
if (filePath.endsWith('.html') || filePath.endsWith('.htm')) {
return htmlToPlainText(text)
}
return text
}
function fileToStub(entry: DropboxFileEntry): ExternalDocument {
return {
externalId: entry.id,
title: entry.name,
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: `https://www.dropbox.com/home${entry.path_display}`,
contentHash: `dropbox:${entry.id}:${entry.content_hash ?? entry.server_modified ?? ''}`,
metadata: {
path: entry.path_display,
lastModified: entry.server_modified || entry.client_modified,
fileSize: entry.size,
},
}
}
export const dropboxConnector: ConnectorConfig = {
...dropboxConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
let data: DropboxListFolderResponse
if (cursor) {
const response = await fetchWithRetry(
'https://api.dropboxapi.com/2/files/list_folder/continue',
{
method: 'POST',
headers: {
Authorization: `Bearer ${accessToken}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({ cursor }),
}
)
if (!response.ok) {
const errorText = await response.text()
logger.error('Failed to continue listing Dropbox folder', {
status: response.status,
error: errorText,
})
throw new Error(`Failed to continue listing Dropbox folder: ${response.status}`)
}
data = await response.json()
} else {
const folderPath = (sourceConfig.folderPath as string)?.trim() || ''
const path = folderPath.startsWith('/') ? folderPath : folderPath ? `/${folderPath}` : ''
logger.info('Listing Dropbox folder', { path: path || '(root)' })
const response = await fetchWithRetry('https://api.dropboxapi.com/2/files/list_folder', {
method: 'POST',
headers: {
Authorization: `Bearer ${accessToken}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
path,
recursive: true,
include_deleted: false,
include_non_downloadable_files: false,
limit: 2000,
}),
})
if (!response.ok) {
const errorText = await response.text()
logger.error('Failed to list Dropbox folder', {
status: response.status,
error: errorText,
})
throw new Error(`Failed to list Dropbox folder: ${response.status}`)
}
data = await response.json()
}
// Keep oversized files and surface them as skipped (failed) documents instead
// of dropping them silently at listing time.
const candidateFiles = data.entries.filter(isDownloadableFile)
const maxFiles = sourceConfig.maxFiles ? Number(sourceConfig.maxFiles) : 0
const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
const stubs = candidateFiles.map((entry) =>
stubOrSkipBySize(fileToStub(entry), entry.size, MAX_FILE_SIZE)
)
const { documents, indexableCount, capReached } = takeIndexableWithinCap(
stubs,
isSkippedDocument,
maxFiles,
previouslyFetched
)
const totalFetched = previouslyFetched + indexableCount
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = capReached
if (hitLimit && syncContext) syncContext.listingCapped = true
return {
documents,
nextCursor: hitLimit ? undefined : data.has_more ? data.cursor : undefined,
hasMore: hitLimit ? false : data.has_more,
}
},
getDocument: async (
accessToken: string,
_sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
try {
const response = await fetchWithRetry('https://api.dropboxapi.com/2/files/get_metadata', {
method: 'POST',
headers: {
Authorization: `Bearer ${accessToken}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({ path: externalId }),
})
if (!response.ok) {
if (response.status === 409) return null
throw new Error(`Failed to get metadata: ${response.status}`)
}
const entry = (await response.json()) as DropboxFileEntry
if (!isDownloadableFile(entry)) return null
const stub = fileToStub(entry)
if (entry.size && entry.size > MAX_FILE_SIZE) {
return markSkipped(stub, sizeLimitSkipReason(MAX_FILE_SIZE))
}
let content: string
try {
content = await downloadFileContent(accessToken, entry.path_lower)
} catch (error) {
if (error instanceof ConnectorFileTooLargeError) {
return markSkipped(stub, sizeLimitSkipReason(error.limitBytes))
}
throw error
}
if (!content.trim()) return null
return { ...stub, content, contentDeferred: false }
} catch (error) {
logger.warn(`Failed to fetch document ${externalId}`, {
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const maxFiles = sourceConfig.maxFiles as string | undefined
if (maxFiles && (Number.isNaN(Number(maxFiles)) || Number(maxFiles) <= 0)) {
return { valid: false, error: 'Max files must be a positive number' }
}
try {
const folderPath = (sourceConfig.folderPath as string)?.trim() || ''
const path = folderPath.startsWith('/') ? folderPath : folderPath ? `/${folderPath}` : ''
const response = await fetchWithRetry(
'https://api.dropboxapi.com/2/files/list_folder',
{
method: 'POST',
headers: {
Authorization: `Bearer ${accessToken}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
path,
limit: 1,
recursive: false,
}),
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
const errorText = await response.text()
if (errorText.includes('not_found')) {
return { valid: false, error: 'Folder not found. Check the path and try again.' }
}
return { valid: false, error: `Failed to access Dropbox: ${response.status}` }
}
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.path === 'string') {
result.path = metadata.path
}
const lastModified = parseTagDate(metadata.lastModified)
if (lastModified) result.lastModified = lastModified
if (metadata.fileSize != null) {
const num = Number(metadata.fileSize)
if (!Number.isNaN(num)) result.fileSize = num
}
return result
},
}
+1
View File
@@ -0,0 +1 @@
export { dropboxConnector } from '@/connectors/dropbox/dropbox'
+40
View File
@@ -0,0 +1,40 @@
import { DropboxIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const dropboxConnectorMeta: ConnectorMeta = {
id: 'dropbox',
name: 'Dropbox',
description: 'Sync text files from Dropbox',
version: '1.0.0',
icon: DropboxIcon,
auth: {
mode: 'oauth',
provider: 'dropbox',
requiredScopes: ['files.metadata.read', 'files.content.read'],
},
configFields: [
{
id: 'folderPath',
title: 'Folder Path',
type: 'short-input',
placeholder: 'e.g. /Documents (default: entire Dropbox)',
required: false,
description: 'Leave empty to sync all supported files',
},
{
id: 'maxFiles',
title: 'Max Files',
type: 'short-input',
required: false,
placeholder: 'e.g. 500 (default: unlimited)',
},
],
tagDefinitions: [
{ id: 'path', displayName: 'File Path', fieldType: 'text' },
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
{ id: 'fileSize', displayName: 'File Size (bytes)', fieldType: 'number' },
],
}
+545
View File
@@ -0,0 +1,545 @@
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import {
ThriftReader,
ThriftWriter,
TYPE_I32,
TYPE_I64,
TYPE_LIST,
TYPE_STRING,
TYPE_STRUCT,
} from '@/app/api/tools/evernote/lib/thrift'
import { evernoteConnectorMeta } from '@/connectors/evernote/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { htmlToPlainText, joinTagArray, parseTagDate } from '@/connectors/utils'
const logger = createLogger('EvernoteConnector')
const NOTES_PER_PAGE = 50
/**
* Extracts the shard ID from an Evernote developer token.
* Token format: "S=s1:U=12345:..." where s1 is the shard.
*/
function extractShardId(token: string): string {
const match = token.match(/S=s(\d+)/)
if (!match) {
throw new Error('Invalid Evernote token format: cannot extract shard ID')
}
return `s${match[1]}`
}
/**
* Extracts the user ID from an Evernote developer token.
* Token format: "S=s1:U=12345:..." where 12345 is the user ID.
*/
function extractUserId(token: string): string {
const match = token.match(/:U=(\d+)/)
if (!match) {
throw new Error('Invalid Evernote token format: cannot extract user ID')
}
return match[1]
}
/**
* Returns the Evernote API host based on the token type.
* Sandbox tokens contain `:Sandbox` and route to sandbox.evernote.com.
*/
function getHost(token: string): string {
return token.includes(':Sandbox') ? 'sandbox.evernote.com' : 'www.evernote.com'
}
/**
* Derives the NoteStore URL from a developer token.
*/
function getNoteStoreUrl(token: string): string {
const shardId = extractShardId(token)
return `https://${getHost(token)}/shard/${shardId}/notestore`
}
/**
* Sends a Thrift RPC call to the Evernote NoteStore via HTTP POST.
*/
async function callNoteStore(
token: string,
writer: ThriftWriter,
retryOptions?: Parameters<typeof fetchWithRetry>[2]
): Promise<ThriftReader> {
const url = getNoteStoreUrl(token)
const response = await fetchWithRetry(
url,
{
method: 'POST',
headers: {
'Content-Type': 'application/x-thrift',
Accept: 'application/x-thrift',
},
body: new Uint8Array(writer.toBuffer()),
},
retryOptions
)
if (!response.ok) {
throw new Error(`Evernote HTTP ${response.status}: ${response.statusText}`)
}
const reader = new ThriftReader(await response.arrayBuffer())
const msg = reader.readMessageBegin()
if (reader.isException(msg.type)) {
const ex = reader.readException()
throw new Error(`Evernote API error: ${ex.message}`)
}
return reader
}
/**
* Checks for Evernote-specific exceptions in response struct fields 1-3.
*/
function checkException(r: ThriftReader, fieldId: number, fieldType: number): boolean {
if ((fieldId === 1 || fieldId === 2) && fieldType === TYPE_STRUCT) {
let errorCode = 0
let message = ''
r.readStruct((r2, fid, ftype) => {
if (fid === 1 && ftype === TYPE_I32) errorCode = r2.readI32()
else if (fid === 2 && ftype === TYPE_STRING) message = r2.readString()
else r2.skip(ftype)
})
throw new Error(`Evernote error (${errorCode}): ${message}`)
}
if (fieldId === 3 && fieldType === TYPE_STRUCT) {
let identifier = ''
r.readStruct((r2, fid, ftype) => {
if (fid === 1 && ftype === TYPE_STRING) identifier = r2.readString()
else r2.skip(ftype)
})
throw new Error(`Evernote not found: ${identifier}`)
}
return false
}
interface Notebook {
guid: string
name: string
}
interface Tag {
guid: string
name: string
}
interface NoteMetadata {
guid: string
title: string
created: number
updated: number
notebookGuid: string
tagGuids: string[]
}
interface Note {
guid: string
title: string
content: string
created: number
updated: number
notebookGuid: string
tagGuids: string[]
}
async function apiListNotebooks(
token: string,
retryOptions?: Parameters<typeof fetchWithRetry>[2]
): Promise<Notebook[]> {
const w = new ThriftWriter()
w.writeMessageBegin('listNotebooks', 0)
w.writeStringField(1, token)
w.writeFieldStop()
const r = await callNoteStore(token, w, retryOptions)
const notebooks: Notebook[] = []
r.readStruct((r2, fid, ftype) => {
if (fid === 0 && ftype === TYPE_LIST) {
const { size } = r2.readListBegin()
for (let i = 0; i < size; i++) {
let guid = ''
let name = ''
r2.readStruct((r3, fid3, ftype3) => {
if (fid3 === 1 && ftype3 === TYPE_STRING) guid = r3.readString()
else if (fid3 === 2 && ftype3 === TYPE_STRING) name = r3.readString()
else r3.skip(ftype3)
})
notebooks.push({ guid, name })
}
} else if (!checkException(r2, fid, ftype)) {
r2.skip(ftype)
}
})
return notebooks
}
async function apiListTags(
token: string,
retryOptions?: Parameters<typeof fetchWithRetry>[2]
): Promise<Tag[]> {
const w = new ThriftWriter()
w.writeMessageBegin('listTags', 0)
w.writeStringField(1, token)
w.writeFieldStop()
const r = await callNoteStore(token, w, retryOptions)
const tags: Tag[] = []
r.readStruct((r2, fid, ftype) => {
if (fid === 0 && ftype === TYPE_LIST) {
const { size } = r2.readListBegin()
for (let i = 0; i < size; i++) {
let guid = ''
let name = ''
r2.readStruct((r3, fid3, ftype3) => {
if (fid3 === 1 && ftype3 === TYPE_STRING) guid = r3.readString()
else if (fid3 === 2 && ftype3 === TYPE_STRING) name = r3.readString()
else r3.skip(ftype3)
})
tags.push({ guid, name })
}
} else if (!checkException(r2, fid, ftype)) {
r2.skip(ftype)
}
})
return tags
}
/**
* Calls NoteStore.findNotesMetadata with offset-based pagination.
*
* Thrift field numbers (from NoteStore.thrift):
* findNotesMetadata(1:token, 2:NoteFilter, 3:offset, 4:maxNotes, 5:ResultSpec)
* NoteFilter: 4:notebookGuid
* NotesMetadataResultSpec: 2:includeTitle, 6:includeCreated, 7:includeUpdated,
* 11:includeNotebookGuid, 12:includeTagGuids
* NotesMetadataList: 1:startIndex, 2:totalNotes, 3:list<NoteMetadata>
* NoteMetadata: 1:guid, 2:title, 6:created(i64), 7:updated(i64),
* 11:notebookGuid, 12:list<tagGuids>
*/
async function apiFindNotesMetadata(
token: string,
offset: number,
maxNotes: number,
notebookGuid?: string,
retryOptions?: Parameters<typeof fetchWithRetry>[2]
): Promise<{ totalNotes: number; notes: NoteMetadata[] }> {
const w = new ThriftWriter()
w.writeMessageBegin('findNotesMetadata', 0)
w.writeStringField(1, token)
w.writeFieldBegin(TYPE_STRUCT, 2) // NoteFilter
if (notebookGuid) {
w.writeStringField(4, notebookGuid)
}
w.writeFieldStop()
w.writeI32Field(3, offset)
w.writeI32Field(4, maxNotes)
w.writeFieldBegin(TYPE_STRUCT, 5) // NotesMetadataResultSpec
w.writeBoolField(2, true) // includeTitle
w.writeBoolField(6, true) // includeCreated
w.writeBoolField(7, true) // includeUpdated
w.writeBoolField(11, true) // includeNotebookGuid
w.writeBoolField(12, true) // includeTagGuids
w.writeFieldStop()
w.writeFieldStop()
const r = await callNoteStore(token, w, retryOptions)
let totalNotes = 0
const notes: NoteMetadata[] = []
r.readStruct((r2, fid, ftype) => {
if (fid === 0 && ftype === TYPE_STRUCT) {
r2.readStruct((r3, fid3, ftype3) => {
if (fid3 === 1 && ftype3 === TYPE_I32) {
r3.readI32()
} else if (fid3 === 2 && ftype3 === TYPE_I32) {
totalNotes = r3.readI32()
} else if (fid3 === 3 && ftype3 === TYPE_LIST) {
const { size } = r3.readListBegin()
for (let i = 0; i < size; i++) {
let guid = ''
let title = ''
let created = 0
let updated = 0
let nbGuid = ''
const tagGuids: string[] = []
r3.readStruct((r4, fid4, ftype4) => {
if (fid4 === 1 && ftype4 === TYPE_STRING) guid = r4.readString()
else if (fid4 === 2 && ftype4 === TYPE_STRING) title = r4.readString()
else if (fid4 === 6 && ftype4 === TYPE_I64) created = Number(r4.readI64())
else if (fid4 === 7 && ftype4 === TYPE_I64) updated = Number(r4.readI64())
else if (fid4 === 11 && ftype4 === TYPE_STRING) nbGuid = r4.readString()
else if (fid4 === 12 && ftype4 === TYPE_LIST) {
const { size: tagCount } = r4.readListBegin()
for (let t = 0; t < tagCount; t++) tagGuids.push(r4.readString())
} else {
r4.skip(ftype4)
}
})
notes.push({ guid, title, created, updated, notebookGuid: nbGuid, tagGuids })
}
} else {
r3.skip(ftype3)
}
})
} else if (!checkException(r2, fid, ftype)) {
r2.skip(ftype)
}
})
return { totalNotes, notes }
}
/**
* Calls NoteStore.getNote to fetch a single note with content.
*
* Thrift: getNote(1:token, 2:guid, 3:withContent, 4:withResourcesData,
* 5:withResourcesRecognition, 6:withResourcesAlternateData)
* Note: 1:guid, 2:title, 3:content, 6:created, 7:updated, 11:notebookGuid, 12:tagGuids
*/
async function apiGetNote(
token: string,
guid: string,
retryOptions?: Parameters<typeof fetchWithRetry>[2]
): Promise<Note> {
const w = new ThriftWriter()
w.writeMessageBegin('getNote', 0)
w.writeStringField(1, token)
w.writeStringField(2, guid)
w.writeBoolField(3, true) // withContent
w.writeBoolField(4, false) // withResourcesData
w.writeBoolField(5, false) // withResourcesRecognition
w.writeBoolField(6, false) // withResourcesAlternateData
w.writeFieldStop()
const r = await callNoteStore(token, w, retryOptions)
let noteGuid = ''
let title = ''
let content = ''
let created = 0
let updated = 0
let notebookGuid = ''
const tagGuids: string[] = []
r.readStruct((r2, fid, ftype) => {
if (fid === 0 && ftype === TYPE_STRUCT) {
r2.readStruct((r3, fid3, ftype3) => {
if (fid3 === 1 && ftype3 === TYPE_STRING) noteGuid = r3.readString()
else if (fid3 === 2 && ftype3 === TYPE_STRING) title = r3.readString()
else if (fid3 === 3 && ftype3 === TYPE_STRING) content = r3.readString()
else if (fid3 === 6 && ftype3 === TYPE_I64) created = Number(r3.readI64())
else if (fid3 === 7 && ftype3 === TYPE_I64) updated = Number(r3.readI64())
else if (fid3 === 11 && ftype3 === TYPE_STRING) notebookGuid = r3.readString()
else if (fid3 === 12 && ftype3 === TYPE_LIST) {
const { size } = r3.readListBegin()
for (let t = 0; t < size; t++) tagGuids.push(r3.readString())
} else {
r3.skip(ftype3)
}
})
} else if (!checkException(r2, fid, ftype)) {
r2.skip(ftype)
}
})
return { guid: noteGuid || guid, title, content, created, updated, notebookGuid, tagGuids }
}
export const evernoteConnector: ConnectorConfig = {
...evernoteConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const notebookGuid = (sourceConfig.notebookGuid as string) || undefined
const retryOptions = { maxRetries: 3, initialDelayMs: 500 }
if (syncContext && !syncContext.tagMap) {
const tags = await apiListTags(accessToken, retryOptions)
syncContext.tagMap = Object.fromEntries(tags.map((t) => [t.guid, t.name]))
}
if (syncContext && !syncContext.notebookMap) {
const notebooks = await apiListNotebooks(accessToken, retryOptions)
syncContext.notebookMap = Object.fromEntries(notebooks.map((nb) => [nb.guid, nb.name]))
}
const tagMap = (syncContext?.tagMap as Record<string, string>) || {}
const notebookMap = (syncContext?.notebookMap as Record<string, string>) || {}
const offset = cursor ? Number(cursor) : 0
const shardId = extractShardId(accessToken)
const userId = extractUserId(accessToken)
const host = getHost(accessToken)
logger.info('Listing Evernote notes', { offset, maxNotes: NOTES_PER_PAGE })
const result = await apiFindNotesMetadata(
accessToken,
offset,
NOTES_PER_PAGE,
notebookGuid,
retryOptions
)
const documents: ExternalDocument[] = result.notes.map((meta) => {
const tagNames = meta.tagGuids.map((g) => tagMap[g]).filter(Boolean)
return {
externalId: meta.guid,
title: meta.title || 'Untitled',
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: `https://${host}/shard/${shardId}/nl/${userId}/${meta.guid}/`,
contentHash: `evernote:${meta.guid}:${meta.updated}`,
metadata: {
tags: tagNames,
notebook: notebookMap[meta.notebookGuid] || '',
createdAt: meta.created ? new Date(meta.created).toISOString() : undefined,
updatedAt: meta.updated ? new Date(meta.updated).toISOString() : undefined,
},
}
})
const nextOffset = offset + result.notes.length
const hasMore = nextOffset < result.totalNotes
return {
documents,
nextCursor: hasMore ? String(nextOffset) : undefined,
hasMore,
}
},
getDocument: async (
accessToken: string,
_sourceConfig: Record<string, unknown>,
externalId: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocument | null> => {
try {
const retryOptions = { maxRetries: 3, initialDelayMs: 500 }
const note = await apiGetNote(accessToken, externalId, retryOptions)
const plainText = htmlToPlainText(note.content)
const title = note.title || 'Untitled'
const content = plainText.trim() ? plainText : title
const shardId = extractShardId(accessToken)
const userId = extractUserId(accessToken)
const host = getHost(accessToken)
if (syncContext && !syncContext.tagMap) {
const tags = await apiListTags(accessToken, retryOptions)
syncContext.tagMap = Object.fromEntries(tags.map((t) => [t.guid, t.name]))
}
if (syncContext && !syncContext.notebookMap) {
const notebooks = await apiListNotebooks(accessToken, retryOptions)
syncContext.notebookMap = Object.fromEntries(notebooks.map((nb) => [nb.guid, nb.name]))
}
let tagMap: Record<string, string>
let notebookMap: Record<string, string>
if (syncContext) {
tagMap = syncContext.tagMap as Record<string, string>
notebookMap = syncContext.notebookMap as Record<string, string>
} else {
const tags = await apiListTags(accessToken, retryOptions)
tagMap = Object.fromEntries(tags.map((t) => [t.guid, t.name]))
const notebooks = await apiListNotebooks(accessToken, retryOptions)
notebookMap = Object.fromEntries(notebooks.map((nb) => [nb.guid, nb.name]))
}
const tagNames = note.tagGuids.map((g) => tagMap[g]).filter(Boolean)
const notebookName = notebookMap[note.notebookGuid] || ''
return {
externalId,
title,
content,
contentDeferred: false,
mimeType: 'text/plain',
sourceUrl: `https://${host}/shard/${shardId}/nl/${userId}/${externalId}/`,
contentHash: `evernote:${note.guid}:${note.updated}`,
metadata: {
tags: tagNames,
notebook: notebookName,
createdAt: note.created ? new Date(note.created).toISOString() : undefined,
updatedAt: note.updated ? new Date(note.updated).toISOString() : undefined,
},
}
} catch (error) {
logger.warn('Failed to get Evernote note', {
externalId,
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
try {
extractShardId(accessToken)
} catch {
return { valid: false, error: 'Invalid developer token format — must start with S=s{number}' }
}
try {
const notebooks = await apiListNotebooks(accessToken, VALIDATE_RETRY_OPTIONS)
const notebookGuid = (sourceConfig.notebookGuid as string) || ''
if (notebookGuid.trim()) {
const found = notebooks.some((nb) => nb.guid === notebookGuid.trim())
if (!found) {
return { valid: false, error: `Notebook with GUID "${notebookGuid}" not found` }
}
}
return { valid: true }
} catch (error) {
const message = toError(error).message || 'Failed to connect to Evernote'
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
const tags = joinTagArray(metadata.tags)
if (tags) result.tags = tags
if (typeof metadata.notebook === 'string' && metadata.notebook) {
result.notebook = metadata.notebook
}
const updatedAt = parseTagDate(metadata.updatedAt)
if (updatedAt) result.updatedAt = updatedAt
const createdAt = parseTagDate(metadata.createdAt)
if (createdAt) result.createdAt = createdAt
return result
},
}
+1
View File
@@ -0,0 +1 @@
export { evernoteConnector } from '@/connectors/evernote/evernote'
+34
View File
@@ -0,0 +1,34 @@
import { EvernoteIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const evernoteConnectorMeta: ConnectorMeta = {
id: 'evernote',
name: 'Evernote',
description: 'Sync notes from Evernote',
version: '1.0.0',
icon: EvernoteIcon,
auth: {
mode: 'apiKey',
label: 'Developer Token',
placeholder: 'Enter your Evernote developer token (starts with S=)',
},
configFields: [
{
id: 'notebookGuid',
title: 'Notebook GUID',
type: 'short-input',
placeholder: 'Leave empty to sync all notebooks',
required: false,
description: 'Sync only notes from this notebook (optional)',
},
],
tagDefinitions: [
{ id: 'tags', displayName: 'Tags', fieldType: 'text' },
{ id: 'notebook', displayName: 'Notebook', fieldType: 'text' },
{ id: 'updatedAt', displayName: 'Last Updated', fieldType: 'date' },
{ id: 'createdAt', displayName: 'Created', fieldType: 'date' },
],
}
+542
View File
@@ -0,0 +1,542 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { fathomConnectorMeta } from '@/connectors/fathom/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { parseTagDate } from '@/connectors/utils'
const logger = createLogger('FathomConnector')
const FATHOM_API_BASE = 'https://api.fathom.ai/external/v1'
const MS_PER_DAY = 24 * 60 * 60 * 1000
/**
* Days subtracted from `lastSyncAt` when computing the incremental `created_after`
* window. Fathom's list endpoint only filters by creation time (no update-based
* filter), so a meeting whose transcript was not yet ready on the sync that first
* saw it would otherwise never be re-listed. The overlap keeps recently-created
* meetings in the window long enough for late transcripts to be retried — the sync
* engine re-attempts meetings whose `getDocument` previously returned null, since
* those are never persisted. Matches the Gong connector's overlap approach.
*/
const INCREMENTAL_OVERLAP_DAYS = 14
/**
* Fathom authenticates external API requests with the `X-Api-Key` header.
* (The API also accepts `Authorization: Bearer` for OAuth-connected apps, but
* the api-key flow this connector uses requires `X-Api-Key`.)
*/
function buildHeaders(apiKey: string): Record<string, string> {
return {
'X-Api-Key': apiKey,
'Content-Type': 'application/json',
}
}
/**
* A meeting object as returned by `GET /meetings`. Only the fields this
* connector reads are typed; the API returns additional fields.
*/
interface FathomMeeting {
recording_id?: number
title?: string
meeting_title?: string | null
url?: string
share_url?: string
created_at?: string
scheduled_start_time?: string | null
scheduled_end_time?: string | null
recording_start_time?: string | null
recording_end_time?: string | null
transcript_language?: string
calendar_invitees_domains_type?: 'only_internal' | 'one_or_more_external' | null
recorded_by?: {
name?: string
email?: string
email_domain?: string
team?: string | null
} | null
}
interface FathomMeetingsListResponse {
items?: FathomMeeting[]
next_cursor?: string | null
}
/**
* A single transcript entry as returned by `GET /recordings/{id}/transcript`.
*/
interface FathomTranscriptEntry {
speaker?: {
display_name?: string
matched_calendar_invitee_email?: string | null
}
text?: string
timestamp?: string
}
interface FathomTranscriptResponse {
transcript?: FathomTranscriptEntry[]
}
interface FathomSummary {
template_name?: string | null
markdown_formatted?: string | null
}
interface FathomSummaryResponse {
summary?: FathomSummary | null
}
/**
* Header fields cached per recording during `listDocuments` so `getDocument`
* can render an identical document header. Fathom exposes no single-meeting
* GET and no `recording_ids` filter, so this metadata cannot be refetched once
* listing has moved past the page that contained it — it is carried forward in
* the shared `syncContext` instead.
*/
interface FathomMeetingHeader {
title: string
meetingDate?: string
durationSeconds?: number
recordedByEmail?: string
recordedByName?: string
team?: string
sourceUrl?: string
contentHash: string
metadata: FathomMeetingMetadata
}
/**
* Metadata describing a Fathom meeting, attached to the listing stub. The sync
* engine merges this onto the hydrated document, so `getDocument` never needs
* to reproduce it.
*/
interface FathomMeetingMetadata {
recordingId?: string
recordedByEmail?: string
recordedByName?: string
team?: string
meetingType?: string
meetingDate?: string
durationSeconds?: number
transcriptLanguage?: string
title?: string
}
/**
* Maps Fathom's `calendar_invitees_domains_type` to a human-readable meeting
* type. This is the only meeting-type signal the API exposes: `only_internal`
* means every invitee shares the recorder's domain; `one_or_more_external`
* means at least one external attendee (customer-facing).
*/
function resolveMeetingType(meeting: FathomMeeting): string | undefined {
switch (meeting.calendar_invitees_domains_type) {
case 'only_internal':
return 'internal'
case 'one_or_more_external':
return 'external'
default:
return undefined
}
}
/**
* Computes the meeting duration in whole seconds from the recording window,
* or undefined when either bound is missing or unparseable.
*/
function computeDurationSeconds(meeting: FathomMeeting): number | undefined {
const start = meeting.recording_start_time ?? meeting.scheduled_start_time ?? undefined
const end = meeting.recording_end_time ?? meeting.scheduled_end_time ?? undefined
if (!start || !end) return undefined
const startMs = new Date(start).getTime()
const endMs = new Date(end).getTime()
if (Number.isNaN(startMs) || Number.isNaN(endMs) || endMs < startMs) return undefined
return Math.round((endMs - startMs) / 1000)
}
/**
* Returns the best title for a meeting, falling back through the title fields.
*/
function resolveTitle(meeting: FathomMeeting): string {
const title = meeting.title?.trim() || meeting.meeting_title?.trim()
return title || 'Untitled Fathom Meeting'
}
/**
* Extracts the connector metadata bag attached to a listing stub.
*/
function buildMetadata(meeting: FathomMeeting): FathomMeetingMetadata {
return {
recordingId: meeting.recording_id != null ? String(meeting.recording_id) : undefined,
recordedByEmail: meeting.recorded_by?.email,
recordedByName: meeting.recorded_by?.name,
team: meeting.recorded_by?.team ?? undefined,
meetingType: resolveMeetingType(meeting),
meetingDate: meeting.recording_start_time ?? meeting.created_at ?? undefined,
durationSeconds: computeDurationSeconds(meeting),
transcriptLanguage: meeting.transcript_language,
title: resolveTitle(meeting),
}
}
/**
* Extracts the lightweight header fields cached for `getDocument`.
*/
function buildHeader(meeting: FathomMeeting): FathomMeetingHeader {
return {
title: resolveTitle(meeting),
meetingDate: meeting.recording_start_time ?? meeting.created_at ?? undefined,
durationSeconds: computeDurationSeconds(meeting),
recordedByEmail: meeting.recorded_by?.email,
recordedByName: meeting.recorded_by?.name,
team: meeting.recorded_by?.team ?? undefined,
sourceUrl: buildSourceUrl(meeting),
contentHash: buildContentHash(meeting),
metadata: buildMetadata(meeting),
}
}
/**
* Builds a metadata-based content hash. Fathom recordings are immutable once
* processed, so the recording id plus its end/creation timestamps fully identify
* a version. The same value is cached in the header and returned by `getDocument`,
* so the stub and hydrated document hash identically.
*/
function buildContentHash(meeting: FathomMeeting): string {
return `fathom:${meeting.recording_id ?? ''}:${meeting.recording_end_time ?? ''}:${meeting.created_at ?? ''}`
}
function buildSourceUrl(meeting: FathomMeeting): string | undefined {
return meeting.share_url || meeting.url || undefined
}
/**
* Reads the cached header for a recording out of the shared sync context.
*/
function readCachedHeader(
syncContext: Record<string, unknown> | undefined,
recordingId: string
): FathomMeetingHeader | undefined {
const cache = syncContext?.meetingHeaders as Record<string, FathomMeetingHeader> | undefined
return cache?.[recordingId]
}
/**
* Stores the header for a recording in the shared sync context.
*/
function cacheHeader(
syncContext: Record<string, unknown> | undefined,
recordingId: string,
header: FathomMeetingHeader
): void {
if (!syncContext) return
const cache =
(syncContext.meetingHeaders as Record<string, FathomMeetingHeader> | undefined) ?? {}
cache[recordingId] = header
syncContext.meetingHeaders = cache
}
/**
* Formats the meeting header, optional summary, and transcript into a single
* plain-text document with one `Speaker: text` line per transcript entry.
*/
function formatMeetingContent(
header: FathomMeetingHeader | undefined,
transcript: FathomTranscriptEntry[],
summary: FathomSummary | null
): string {
const parts: string[] = []
parts.push(`Meeting: ${header?.title ?? 'Untitled Fathom Meeting'}`)
if (header?.meetingDate) parts.push(`Date: ${header.meetingDate}`)
if (header?.durationSeconds != null) {
parts.push(`Duration: ${Math.round(header.durationSeconds / 60)} minutes`)
}
if (header?.recordedByEmail) {
parts.push(`Recorded by: ${header.recordedByName ?? header.recordedByEmail}`)
}
if (header?.team) parts.push(`Team: ${header.team}`)
if (summary?.markdown_formatted?.trim()) {
parts.push('')
parts.push('--- Summary ---')
parts.push(summary.markdown_formatted.trim())
}
if (transcript.length > 0) {
parts.push('')
parts.push('--- Transcript ---')
for (const entry of transcript) {
const speaker = entry.speaker?.display_name?.trim() || 'Unknown'
const text = entry.text?.trim()
if (text) parts.push(`${speaker}: ${text}`)
}
}
return parts.join('\n')
}
/**
* Converts a listing meeting into a deferred stub. Content is fetched lazily
* via `getDocument` only for new or changed meetings.
*/
function meetingToStub(meeting: FathomMeeting): ExternalDocument {
const metadata = buildMetadata(meeting)
return {
externalId: String(meeting.recording_id),
title: resolveTitle(meeting),
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: buildSourceUrl(meeting),
contentHash: buildContentHash(meeting),
metadata: { ...metadata },
}
}
export const fathomConnector: ConnectorConfig = {
...fathomConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>,
lastSyncAt?: Date
): Promise<ExternalDocumentList> => {
const recordedBy = (sourceConfig.recordedBy as string | undefined)?.trim()
const teams = (sourceConfig.teams as string | undefined)?.trim()
const meetingType = (sourceConfig.meetingType as string | undefined)?.trim()
const inviteeDomain = (sourceConfig.inviteeDomains as string | undefined)?.trim()
const maxMeetings = sourceConfig.maxMeetings ? Number(sourceConfig.maxMeetings) : 0
const url = new URL(`${FATHOM_API_BASE}/meetings`)
if (recordedBy) url.searchParams.append('recorded_by[]', recordedBy)
if (teams) url.searchParams.append('teams[]', teams)
if (meetingType && meetingType !== 'all') {
url.searchParams.append('calendar_invitees_domains_type', meetingType)
}
if (inviteeDomain) url.searchParams.append('calendar_invitees_domains[]', inviteeDomain)
if (cursor) url.searchParams.append('cursor', cursor)
if (lastSyncAt) {
const createdAfter = new Date(lastSyncAt.getTime() - INCREMENTAL_OVERLAP_DAYS * MS_PER_DAY)
url.searchParams.append('created_after', createdAfter.toISOString())
}
logger.info('Listing Fathom meetings', {
hasCursor: Boolean(cursor),
recordedBy,
teams,
meetingType,
inviteeDomain,
incremental: Boolean(lastSyncAt),
})
const response = await fetchWithRetry(url.toString(), {
method: 'GET',
headers: buildHeaders(accessToken),
})
if (!response.ok) {
const errorText = await response.text().catch(() => '')
logger.error('Failed to list Fathom meetings', {
status: response.status,
error: errorText.slice(0, 500),
})
throw new Error(`Failed to list Fathom meetings: ${response.status}`)
}
const data = (await response.json()) as FathomMeetingsListResponse
const meetings = data.items ?? []
const nextCursor = data.next_cursor?.trim() || undefined
const allDocuments: ExternalDocument[] = []
for (const meeting of meetings) {
if (meeting.recording_id == null) continue
const externalId = String(meeting.recording_id)
cacheHeader(syncContext, externalId, buildHeader(meeting))
allDocuments.push(meetingToStub(meeting))
}
const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
let documents = allDocuments
if (maxMeetings > 0) {
const remaining = Math.max(0, maxMeetings - prevFetched)
if (allDocuments.length > remaining) {
documents = allDocuments.slice(0, remaining)
}
}
const totalFetched = prevFetched + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = maxMeetings > 0 && totalFetched >= maxMeetings
if (hitLimit && syncContext) syncContext.listingCapped = true
const hasMore = !hitLimit && Boolean(nextCursor)
return {
documents,
nextCursor: hasMore ? nextCursor : undefined,
hasMore,
}
},
getDocument: async (
accessToken: string,
_sourceConfig: Record<string, unknown>,
externalId: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocument | null> => {
try {
if (!externalId) return null
const transcriptUrl = `${FATHOM_API_BASE}/recordings/${encodeURIComponent(externalId)}/transcript`
const transcriptResponse = await fetchWithRetry(transcriptUrl, {
method: 'GET',
headers: buildHeaders(accessToken),
})
if (!transcriptResponse.ok) {
if (transcriptResponse.status === 404) return null
throw new Error(`Failed to fetch Fathom transcript: ${transcriptResponse.status}`)
}
const transcriptData = (await transcriptResponse.json()) as FathomTranscriptResponse
const transcript = transcriptData.transcript ?? []
let summary: FathomSummary | null = null
try {
const summaryUrl = `${FATHOM_API_BASE}/recordings/${encodeURIComponent(externalId)}/summary`
const summaryResponse = await fetchWithRetry(summaryUrl, {
method: 'GET',
headers: buildHeaders(accessToken),
})
if (summaryResponse.ok) {
const summaryData = (await summaryResponse.json()) as FathomSummaryResponse
summary = summaryData.summary ?? null
}
} catch (summaryError) {
logger.warn('Failed to fetch Fathom summary', {
externalId,
error: toError(summaryError).message,
})
}
const hasTranscript = transcript.some((entry) => entry.text?.trim())
const hasSummary = Boolean(summary?.markdown_formatted?.trim())
if (!hasTranscript && !hasSummary) {
logger.info('No transcript or summary yet for Fathom meeting', { externalId })
return null
}
const header = readCachedHeader(syncContext, externalId)
if (!header) {
logger.warn(
'No cached header for Fathom meeting; skipping to avoid an un-refreshable record',
{
externalId,
}
)
return null
}
const content = formatMeetingContent(header, transcript, summary).trim()
if (!content) return null
return {
externalId,
title: header.title,
content,
contentDeferred: false,
mimeType: 'text/plain',
sourceUrl: header.sourceUrl,
contentHash: header.contentHash,
metadata: { ...header.metadata },
}
} catch (error) {
logger.warn('Failed to get Fathom meeting', {
externalId,
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const maxMeetings = sourceConfig.maxMeetings as string | undefined
if (maxMeetings && (Number.isNaN(Number(maxMeetings)) || Number(maxMeetings) < 0)) {
return { valid: false, error: 'Max meetings must be a non-negative number' }
}
try {
const response = await fetchWithRetry(
`${FATHOM_API_BASE}/meetings`,
{
method: 'GET',
headers: buildHeaders(accessToken),
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
const errorText = await response.text().catch(() => '')
return {
valid: false,
error: `Fathom access failed: ${response.status}${errorText ? `${errorText.slice(0, 200)}` : ''}`,
}
}
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.title === 'string' && metadata.title.trim()) {
result.title = metadata.title
}
if (typeof metadata.recordedByEmail === 'string' && metadata.recordedByEmail.trim()) {
result.recordedByEmail = metadata.recordedByEmail
}
if (typeof metadata.recordedByName === 'string' && metadata.recordedByName.trim()) {
result.recordedByName = metadata.recordedByName
}
if (typeof metadata.team === 'string' && metadata.team.trim()) {
result.team = metadata.team
}
if (typeof metadata.meetingType === 'string' && metadata.meetingType.trim()) {
result.meetingType = metadata.meetingType
}
if (typeof metadata.transcriptLanguage === 'string' && metadata.transcriptLanguage.trim()) {
result.transcriptLanguage = metadata.transcriptLanguage
}
if (metadata.durationSeconds != null) {
const num = Number(metadata.durationSeconds)
if (!Number.isNaN(num)) result.durationSeconds = num
}
const meetingDate = parseTagDate(metadata.meetingDate)
if (meetingDate) result.meetingDate = meetingDate
return result
},
}
+1
View File
@@ -0,0 +1 @@
export { fathomConnector } from '@/connectors/fathom/fathom'
+79
View File
@@ -0,0 +1,79 @@
import { FathomIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const fathomConnectorMeta: ConnectorMeta = {
id: 'fathom',
name: 'Fathom',
description: 'Sync meeting transcripts and summaries from Fathom',
version: '1.0.0',
icon: FathomIcon,
auth: {
mode: 'apiKey',
label: 'API Key',
placeholder: 'Enter your Fathom API key',
},
supportsIncrementalSync: true,
configFields: [
{
id: 'recordedBy',
title: 'Filter by Recorder Email',
type: 'short-input',
placeholder: 'e.g. john@example.com',
required: false,
description: 'Only sync meetings recorded by this email',
},
{
id: 'teams',
title: 'Filter by Team',
type: 'short-input',
placeholder: 'e.g. Sales',
required: false,
description: 'Only sync meetings belonging to this team',
},
{
id: 'meetingType',
title: 'Filter by Meeting Type',
type: 'dropdown',
mode: 'advanced',
required: false,
description:
'Only sync internal meetings (everyone shares the recorders domain) or external meetings (at least one outside attendee). Leave as All to sync both.',
options: [
{ id: 'all', label: 'All meetings' },
{ id: 'one_or_more_external', label: 'External (customer-facing) only' },
{ id: 'only_internal', label: 'Internal only' },
],
},
{
id: 'inviteeDomains',
title: 'Filter by Attendee Domain',
type: 'short-input',
mode: 'advanced',
placeholder: 'e.g. acme.com',
required: false,
description:
'Only sync meetings that include a calendar invitee from this company email domain (exact match).',
},
{
id: 'maxMeetings',
title: 'Max Meetings',
type: 'short-input',
required: false,
placeholder: 'e.g. 200 (default: unlimited)',
},
],
tagDefinitions: [
{ id: 'title', displayName: 'Title', fieldType: 'text' },
{ id: 'recordedByEmail', displayName: 'Recorded By (Email)', fieldType: 'text' },
{ id: 'recordedByName', displayName: 'Recorded By (Name)', fieldType: 'text' },
{ id: 'team', displayName: 'Team', fieldType: 'text' },
{ id: 'meetingType', displayName: 'Meeting Type', fieldType: 'text' },
{ id: 'transcriptLanguage', displayName: 'Language', fieldType: 'text' },
{ id: 'durationSeconds', displayName: 'Duration (seconds)', fieldType: 'number' },
{ id: 'meetingDate', displayName: 'Meeting Date', fieldType: 'date' },
],
}
+339
View File
@@ -0,0 +1,339 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { firefliesConnectorMeta } from '@/connectors/fireflies/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { parseTagDate } from '@/connectors/utils'
const logger = createLogger('FirefliesConnector')
const FIREFLIES_GRAPHQL_URL = 'https://api.fireflies.ai/graphql'
const TRANSCRIPTS_PER_PAGE = 50
interface FirefliesTranscript {
id: string
title: string
date: number
duration: number
host_email?: string
organizer_email?: string
participants?: string[]
transcript_url?: string
speakers?: { id: number; name: string }[]
sentences?: { index: number; speaker_name: string; text: string }[]
summary?: {
keywords?: string[]
action_items?: string
overview?: string
short_summary?: string
}
}
/**
* Executes a GraphQL query against the Fireflies API.
*/
async function firefliesGraphQL(
accessToken: string,
query: string,
variables: Record<string, unknown> = {},
retryOptions?: Parameters<typeof fetchWithRetry>[2]
): Promise<Record<string, unknown>> {
const response = await fetchWithRetry(
FIREFLIES_GRAPHQL_URL,
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${accessToken}`,
},
body: JSON.stringify({ query, variables }),
},
retryOptions
)
if (!response.ok) {
throw new Error(`Fireflies API HTTP error: ${response.status}`)
}
const data = await response.json()
if (data.errors) {
const message = (data.errors as { message: string }[])[0]?.message || 'Unknown GraphQL error'
throw new Error(`Fireflies API error: ${message}`)
}
return data.data as Record<string, unknown>
}
/**
* Formats transcript sentences into plain text content.
*/
function formatTranscriptContent(transcript: FirefliesTranscript): string {
const parts: string[] = []
if (transcript.title) {
parts.push(`Meeting: ${transcript.title}`)
}
if (transcript.date) {
parts.push(`Date: ${new Date(transcript.date).toISOString()}`)
}
if (transcript.duration) {
const minutes = Math.round(transcript.duration / 60)
parts.push(`Duration: ${minutes} minutes`)
}
if (transcript.host_email) {
parts.push(`Host: ${transcript.host_email}`)
}
if (transcript.participants && transcript.participants.length > 0) {
parts.push(`Participants: ${transcript.participants.join(', ')}`)
}
if (transcript.summary?.overview) {
parts.push('')
parts.push('--- Overview ---')
parts.push(transcript.summary.overview)
}
if (transcript.summary?.action_items) {
parts.push('')
parts.push('--- Action Items ---')
parts.push(transcript.summary.action_items)
}
if (transcript.summary?.keywords && transcript.summary.keywords.length > 0) {
parts.push('')
parts.push(`Keywords: ${transcript.summary.keywords.join(', ')}`)
}
if (transcript.sentences && transcript.sentences.length > 0) {
parts.push('')
parts.push('--- Transcript ---')
for (const sentence of transcript.sentences) {
parts.push(`${sentence.speaker_name}: ${sentence.text}`)
}
}
return parts.join('\n')
}
export const firefliesConnector: ConnectorConfig = {
...firefliesConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const hostEmail = (sourceConfig.hostEmail as string) || ''
const maxTranscripts = sourceConfig.maxTranscripts ? Number(sourceConfig.maxTranscripts) : 0
const skip = cursor ? Number(cursor) : 0
const variables: Record<string, unknown> = {
limit: TRANSCRIPTS_PER_PAGE,
skip,
}
if (hostEmail.trim()) {
variables.host_email = hostEmail.trim()
}
logger.info('Listing Fireflies transcripts', { skip, limit: TRANSCRIPTS_PER_PAGE, hostEmail })
const data = await firefliesGraphQL(
accessToken,
`query Transcripts(
$limit: Int
$skip: Int
$host_email: String
) {
transcripts(
limit: $limit
skip: $skip
host_email: $host_email
) {
id
title
date
duration
host_email
organizer_email
participants
transcript_url
speakers {
id
name
}
}
}`,
variables
)
const transcripts = (data.transcripts || []) as FirefliesTranscript[]
const documents: ExternalDocument[] = transcripts.map((transcript) => {
const meetingDate = transcript.date ? new Date(transcript.date).toISOString() : undefined
const speakerNames = transcript.speakers?.map((s) => s.name).filter(Boolean) ?? []
return {
externalId: transcript.id,
title: transcript.title || 'Untitled Meeting',
content: '',
contentDeferred: true,
mimeType: 'text/plain' as const,
sourceUrl: transcript.transcript_url || undefined,
contentHash: `fireflies:${transcript.id}:${transcript.date ?? ''}:${transcript.duration ?? ''}`,
metadata: {
hostEmail: transcript.host_email,
duration: transcript.duration,
meetingDate,
participants: transcript.participants,
speakers: speakerNames,
},
}
})
const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = maxTranscripts > 0 && totalFetched >= maxTranscripts
const hasMore = !hitLimit && transcripts.length === TRANSCRIPTS_PER_PAGE
return {
documents,
nextCursor: hasMore ? String(skip + transcripts.length) : undefined,
hasMore,
}
},
getDocument: async (
accessToken: string,
_sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
try {
const data = await firefliesGraphQL(
accessToken,
`query Transcript($id: String!) {
transcript(id: $id) {
id
title
date
duration
host_email
organizer_email
participants
transcript_url
speakers {
id
name
}
sentences {
index
speaker_name
text
}
summary {
keywords
action_items
overview
short_summary
}
}
}`,
{ id: externalId }
)
const transcript = data.transcript as FirefliesTranscript | null
if (!transcript) return null
const content = formatTranscriptContent(transcript)
const contentHash = `fireflies:${transcript.id}:${transcript.date ?? ''}:${transcript.duration ?? ''}`
const meetingDate = transcript.date ? new Date(transcript.date).toISOString() : undefined
const speakerNames = transcript.speakers?.map((s) => s.name).filter(Boolean) ?? []
return {
externalId: transcript.id,
title: transcript.title || 'Untitled Meeting',
content,
contentDeferred: false,
mimeType: 'text/plain',
sourceUrl: transcript.transcript_url || undefined,
contentHash,
metadata: {
hostEmail: transcript.host_email,
duration: transcript.duration,
meetingDate,
participants: transcript.participants,
speakers: speakerNames,
keywords: transcript.summary?.keywords,
},
}
} catch (error) {
logger.warn('Failed to get Fireflies transcript', {
externalId,
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const maxTranscripts = sourceConfig.maxTranscripts as string | undefined
if (maxTranscripts && (Number.isNaN(Number(maxTranscripts)) || Number(maxTranscripts) < 0)) {
return { valid: false, error: 'Max transcripts must be a non-negative number' }
}
try {
await firefliesGraphQL(
accessToken,
`query User {
user {
user_id
name
email
}
}`,
{},
VALIDATE_RETRY_OPTIONS
)
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.hostEmail === 'string') {
result.hostEmail = metadata.hostEmail
}
const speakers = Array.isArray(metadata.speakers) ? (metadata.speakers as string[]) : []
if (speakers.length > 0) {
result.speakers = speakers.join(', ')
}
if (metadata.duration != null) {
const num = Number(metadata.duration)
if (!Number.isNaN(num)) result.duration = num
}
const meetingDate = parseTagDate(metadata.meetingDate)
if (meetingDate) result.meetingDate = meetingDate
return result
},
}
+1
View File
@@ -0,0 +1 @@
export { firefliesConnector } from '@/connectors/fireflies/fireflies'
+41
View File
@@ -0,0 +1,41 @@
import { FirefliesIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const firefliesConnectorMeta: ConnectorMeta = {
id: 'fireflies',
name: 'Fireflies',
description: 'Sync meeting transcripts from Fireflies.ai',
version: '1.0.0',
icon: FirefliesIcon,
auth: {
mode: 'apiKey',
label: 'API Key',
placeholder: 'Enter your Fireflies API key',
},
configFields: [
{
id: 'hostEmail',
title: 'Filter by Host Email',
type: 'short-input',
placeholder: 'e.g. john@example.com',
required: false,
description: 'Only sync transcripts hosted by this email',
},
{
id: 'maxTranscripts',
title: 'Max Transcripts',
type: 'short-input',
required: false,
placeholder: 'e.g. 100 (default: unlimited)',
},
],
tagDefinitions: [
{ id: 'hostEmail', displayName: 'Host Email', fieldType: 'text' },
{ id: 'speakers', displayName: 'Speakers', fieldType: 'text' },
{ id: 'duration', displayName: 'Duration (seconds)', fieldType: 'number' },
{ id: 'meetingDate', displayName: 'Meeting Date', fieldType: 'date' },
],
}
+448
View File
@@ -0,0 +1,448 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { githubConnectorMeta } from '@/connectors/github/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import {
CONNECTOR_MAX_FILE_BYTES,
markSkipped,
parseTagDate,
sizeLimitSkipReason,
stubOrSkipBySize,
takeIndexableWithinCap,
} from '@/connectors/utils'
const logger = createLogger('GitHubConnector')
const GITHUB_API_URL = 'https://api.github.com'
const BATCH_SIZE = 30
const GIT_SHA_PREFIX = 'git-sha:'
const MAX_FILE_SIZE = CONNECTOR_MAX_FILE_BYTES
const BINARY_SNIFF_BYTES = 8000
/**
* Heuristic binary detection: Git treats files containing a NUL byte in the
* first 8000 bytes as binary. Matches `git diff` / `git grep` semantics.
*/
function isBinaryBuffer(buf: Buffer): boolean {
const len = Math.min(buf.length, BINARY_SNIFF_BYTES)
for (let i = 0; i < len; i++) {
if (buf[i] === 0) return true
}
return false
}
/**
* Parses the repository string into owner and repo.
*/
function parseRepo(repository: string): { owner: string; repo: string } {
const cleaned = repository.replace(/^https?:\/\/github\.com\//, '').replace(/\.git$/, '')
const parts = cleaned.split('/')
if (parts.length < 2 || !parts[0] || !parts[1]) {
throw new Error(`Invalid repository format: "${repository}". Use "owner/repo".`)
}
return { owner: parts[0], repo: parts[1] }
}
/**
* File extension filter set from user config. Returns null if no filter (accept all).
*/
function parseExtensions(extensions: string): Set<string> | null {
const trimmed = extensions.trim()
if (!trimmed) return null
const exts = trimmed
.split(',')
.map((e) => e.trim().toLowerCase())
.filter(Boolean)
.map((e) => (e.startsWith('.') ? e : `.${e}`))
return exts.length > 0 ? new Set(exts) : null
}
/**
* Checks whether a file path matches the extension filter.
*/
function matchesExtension(filePath: string, extSet: Set<string> | null): boolean {
if (!extSet) return true
const lastDot = filePath.lastIndexOf('.')
if (lastDot === -1) return false
return extSet.has(filePath.slice(lastDot).toLowerCase())
}
interface TreeItem {
path: string
mode: string
type: string
sha: string
size?: number
}
/**
* Fetches the full recursive tree for a branch.
*/
async function fetchTree(
accessToken: string,
owner: string,
repo: string,
branch: string
): Promise<TreeItem[]> {
const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/git/trees/${encodeURIComponent(branch)}?recursive=1`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/vnd.github+json',
Authorization: `Bearer ${accessToken}`,
'X-GitHub-Api-Version': '2022-11-28',
},
})
if (!response.ok) {
const errorText = await response.text()
logger.error('Failed to fetch GitHub tree', { status: response.status, error: errorText })
throw new Error(`Failed to fetch repository tree: ${response.status}`)
}
const data = await response.json()
if (data.truncated) {
logger.warn('GitHub tree was truncated — some files may be missing', { owner, repo, branch })
}
return (data.tree || []).filter((item: TreeItem) => item.type === 'blob')
}
/**
* Fetches blob content via the Git Blobs API. Used as a fallback when the
* `/contents/` endpoint cannot return the file body (files larger than 1 MB
* return `content: ""` and `encoding: "none"`). Supports blobs up to 100 MB.
*/
async function fetchBlobContent(
accessToken: string,
owner: string,
repo: string,
sha: string
): Promise<string | null> {
const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/git/blobs/${encodeURIComponent(sha)}`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/vnd.github+json',
Authorization: `Bearer ${accessToken}`,
'X-GitHub-Api-Version': '2022-11-28',
},
})
if (!response.ok) {
throw new Error(`Failed to fetch git blob ${sha}: ${response.status}`)
}
const data = await response.json()
const content = (data.content as string) || ''
const encoding = data.encoding as string | undefined
if (encoding === 'base64') {
const buf = Buffer.from(content, 'base64')
if (isBinaryBuffer(buf)) return null
return buf.toString('utf8')
}
/**
* Per https://docs.github.com/en/rest/git/blobs the Blobs API only ever
* returns base64. Refuse to silently persist empty content for an
* unexpected encoding so a sync surfaces the error instead.
*/
throw new Error(`Unexpected git blob encoding for ${sha}: ${encoding ?? 'undefined'}`)
}
/**
* Creates a lightweight stub ExternalDocument from a tree item.
* Uses the Git blob SHA as contentHash for change detection, avoiding
* the need to fetch blob content for every file during listing.
* Content is deferred and only fetched for new/changed documents.
*/
function treeItemToStub(
owner: string,
repo: string,
branch: string,
item: TreeItem
): ExternalDocument {
return {
externalId: item.path,
title: item.path.split('/').pop() || item.path,
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: `https://github.com/${owner}/${repo}/blob/${branch.split('/').map(encodeURIComponent).join('/')}/${item.path.split('/').map(encodeURIComponent).join('/')}`,
contentHash: `${GIT_SHA_PREFIX}${item.sha}`,
metadata: {
path: item.path,
sha: item.sha,
size: item.size,
branch,
repository: `${owner}/${repo}`,
},
}
}
export const githubConnector: ConnectorConfig = {
...githubConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const { owner, repo } = parseRepo(sourceConfig.repository as string)
const branch = ((sourceConfig.branch as string) || 'main').trim()
const pathPrefix = ((sourceConfig.pathPrefix as string) || '').trim()
const extSet = parseExtensions((sourceConfig.extensions as string) || '')
const maxFiles = sourceConfig.maxFiles ? Number(sourceConfig.maxFiles) : 0
let capped: TreeItem[]
if (syncContext?.filteredTree) {
capped = syncContext.filteredTree as TreeItem[]
} else {
const tree = await fetchTree(accessToken, owner, repo, branch)
// Filter by path prefix and extensions. Oversized files are kept here and
// surfaced as skipped (failed) documents at stub time so they stay visible.
const filtered = tree.filter((item) => {
if (pathPrefix && !item.path.startsWith(pathPrefix)) return false
if (!matchesExtension(item.path, extSet)) return false
return true
})
// Apply the max-files limit to indexable files only; oversized files within
// the capped window are kept (and surfaced as skipped) but never consume the cap.
capped =
maxFiles > 0
? takeIndexableWithinCap(
filtered,
(item) => Boolean(item.size && item.size > MAX_FILE_SIZE),
maxFiles,
0
).documents
: filtered
if (syncContext) syncContext.filteredTree = capped
}
// Paginate using offset cursor
const offset = cursor ? Number(cursor) : 0
const batch = capped.slice(offset, offset + BATCH_SIZE)
logger.info('Listing GitHub files', {
owner,
repo,
branch,
totalFiltered: capped.length,
offset,
batchSize: batch.length,
})
const documents = batch.map((item) =>
stubOrSkipBySize(treeItemToStub(owner, repo, branch, item), item.size, MAX_FILE_SIZE)
)
const nextOffset = offset + BATCH_SIZE
const hasMore = nextOffset < capped.length
return {
documents,
nextCursor: hasMore ? String(nextOffset) : undefined,
hasMore,
}
},
getDocument: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
externalId: string,
_syncContext?: Record<string, unknown>
): Promise<ExternalDocument | null> => {
const { owner, repo } = parseRepo(sourceConfig.repository as string)
const branch = ((sourceConfig.branch as string) || 'main').trim()
// externalId is the file path
const path = externalId
try {
const encodedPath = path.split('/').map(encodeURIComponent).join('/')
const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/contents/${encodedPath}?ref=${encodeURIComponent(branch)}`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/vnd.github+json',
Authorization: `Bearer ${accessToken}`,
'X-GitHub-Api-Version': '2022-11-28',
},
})
if (!response.ok) {
if (response.status === 404) return null
if (response.status === 403) {
logger.info('Skipping GitHub file rejected by Contents API', {
path,
status: response.status,
})
return null
}
throw new Error(`Failed to fetch file ${path}: ${response.status}`)
}
const lastModifiedHeader = response.headers.get('last-modified') || undefined
const data = await response.json()
const size = typeof data.size === 'number' ? data.size : 0
if (size > MAX_FILE_SIZE) {
logger.info('Skipping GitHub file exceeding size limit', {
path,
size,
limit: MAX_FILE_SIZE,
})
return markSkipped(
{
externalId,
title: path.split('/').pop() || path,
content: '',
mimeType: 'text/plain',
sourceUrl: `https://github.com/${owner}/${repo}/blob/${branch.split('/').map(encodeURIComponent).join('/')}/${path.split('/').map(encodeURIComponent).join('/')}`,
contentHash: `${GIT_SHA_PREFIX}${data.sha as string}`,
metadata: {
path,
sha: data.sha as string,
size,
branch,
repository: `${owner}/${repo}`,
},
},
sizeLimitSkipReason(MAX_FILE_SIZE)
)
}
const rawContent = (data.content as string) || ''
const encoding = data.encoding as string | undefined
let content: string
if (encoding === 'base64' && rawContent.length > 0) {
const buf = Buffer.from(rawContent, 'base64')
if (isBinaryBuffer(buf)) {
logger.info('Skipping binary GitHub file', { path, size })
return null
}
content = buf.toString('utf8')
} else if (encoding === 'none' && data.sha && size > 0) {
const blobContent = await fetchBlobContent(accessToken, owner, repo, data.sha as string)
if (blobContent === null) {
logger.info('Skipping binary GitHub file', { path, size })
return null
}
content = blobContent
} else {
content = ''
}
return {
externalId,
title: path.split('/').pop() || path,
content,
contentDeferred: false,
mimeType: 'text/plain',
sourceUrl: `https://github.com/${owner}/${repo}/blob/${branch.split('/').map(encodeURIComponent).join('/')}/${path.split('/').map(encodeURIComponent).join('/')}`,
contentHash: `${GIT_SHA_PREFIX}${data.sha as string}`,
metadata: {
path,
sha: data.sha as string,
size: data.size as number,
branch,
repository: `${owner}/${repo}`,
lastModified: lastModifiedHeader,
},
}
} catch (error) {
logger.warn(`Failed to fetch GitHub document ${externalId}`, {
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const repository = (sourceConfig.repository as string)?.trim()
if (!repository) {
return { valid: false, error: 'Repository is required' }
}
let owner: string
let repo: string
try {
const parsed = parseRepo(repository)
owner = parsed.owner
repo = parsed.repo
} catch (error) {
return {
valid: false,
error: getErrorMessage(error, 'Invalid repository format'),
}
}
const maxFiles = sourceConfig.maxFiles as string | undefined
if (maxFiles && (Number.isNaN(Number(maxFiles)) || Number(maxFiles) <= 0)) {
return { valid: false, error: 'Max files must be a positive number' }
}
const branch = ((sourceConfig.branch as string) || 'main').trim()
try {
// Verify repo and branch are accessible
const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/branches/${encodeURIComponent(branch)}`
const response = await fetchWithRetry(
url,
{
method: 'GET',
headers: {
Accept: 'application/vnd.github+json',
Authorization: `Bearer ${accessToken}`,
'X-GitHub-Api-Version': '2022-11-28',
},
},
VALIDATE_RETRY_OPTIONS
)
if (response.status === 404) {
return {
valid: false,
error: `Repository "${owner}/${repo}" or branch "${branch}" not found`,
}
}
if (!response.ok) {
return { valid: false, error: `Cannot access repository: ${response.status}` }
}
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.path === 'string') result.path = metadata.path
if (typeof metadata.repository === 'string') result.repository = metadata.repository
if (typeof metadata.branch === 'string') result.branch = metadata.branch
if (metadata.size != null) {
const num = Number(metadata.size)
if (!Number.isNaN(num)) result.size = num
}
const lastModified = parseTagDate(metadata.lastModified)
if (lastModified) result.lastModified = lastModified
return result
},
}
+1
View File
@@ -0,0 +1 @@
export { githubConnector } from '@/connectors/github/github'
+62
View File
@@ -0,0 +1,62 @@
import { GithubIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const githubConnectorMeta: ConnectorMeta = {
id: 'github',
name: 'GitHub',
description: 'Sync files from a GitHub repository',
version: '1.0.0',
icon: GithubIcon,
auth: {
mode: 'apiKey',
label: 'Personal Access Token',
placeholder: 'ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx',
},
configFields: [
{
id: 'repository',
title: 'Repository',
type: 'short-input',
placeholder: 'owner/repo',
required: true,
},
{
id: 'branch',
title: 'Branch',
type: 'short-input',
placeholder: 'main (default)',
required: false,
},
{
id: 'pathPrefix',
title: 'Path Filter',
type: 'short-input',
placeholder: 'e.g. docs/, src/components/',
required: false,
},
{
id: 'extensions',
title: 'File Extensions',
type: 'short-input',
placeholder: 'e.g. .md, .txt, .mdx',
required: false,
},
{
id: 'maxFiles',
title: 'Max Files',
type: 'short-input',
required: false,
placeholder: 'e.g. 500 (default: unlimited)',
},
],
tagDefinitions: [
{ id: 'path', displayName: 'File Path', fieldType: 'text' },
{ id: 'repository', displayName: 'Repository', fieldType: 'text' },
{ id: 'branch', displayName: 'Branch', fieldType: 'text' },
{ id: 'size', displayName: 'File Size', fieldType: 'number' },
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
],
}
File diff suppressed because it is too large Load Diff
+1
View File
@@ -0,0 +1 @@
export { gitlabConnector } from '@/connectors/gitlab/gitlab'
+140
View File
@@ -0,0 +1,140 @@
import { GitLabIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const gitlabConnectorMeta: ConnectorMeta = {
id: 'gitlab',
name: 'GitLab',
description:
'Sync repository files, wiki pages, and issues from a GitLab project into your knowledge base',
version: '1.0.0',
icon: GitLabIcon,
/**
* Incremental sync applies to issues only (via the `updated_after` filter
* derived from lastSyncAt). Wikis and repository files lack a change timestamp
* on listing, so they are always re-listed in full and reconciled by content
* hash (wiki: content digest, file: git blob SHA) — unchanged docs are skipped.
*/
supportsIncrementalSync: true,
auth: {
mode: 'apiKey',
label: 'Personal Access Token',
placeholder: 'Enter your GitLab PAT',
},
configFields: [
{
id: 'host',
title: 'Host',
type: 'short-input',
placeholder: 'gitlab.com',
required: false,
description: 'Self-managed GitLab host. Leave blank for gitlab.com.',
},
{
id: 'project',
title: 'Project',
type: 'short-input',
placeholder: 'group/project or numeric ID',
required: true,
description: 'Project path (e.g. my-group/my-repo) or numeric project ID.',
},
{
id: 'contentTypes',
title: 'Content',
type: 'dropdown',
required: false,
options: [
{ label: 'Code, Wiki & Issues', id: 'all' },
{ label: 'Code (repository files) only', id: 'repo' },
{ label: 'Wiki only', id: 'wiki' },
{ label: 'Issues only', id: 'issues' },
{ label: 'Wiki & Issues', id: 'both' },
],
description: 'Which content to index. "Code" syncs repository files (READMEs, docs, source).',
},
{
id: 'ref',
title: 'Branch',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'Default branch',
description: 'Branch or tag to sync repository files from. Applies only when syncing Code.',
},
{
id: 'pathPrefix',
title: 'Path Filter',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. docs/',
description:
'Only sync repository files under this path prefix. Applies only when syncing Code.',
},
{
id: 'fileExtensions',
title: 'File Extensions',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. .md, .txt, .mdx',
description:
'Only sync repository files with these extensions (comma-separated). Leave blank for all text files. Applies only when syncing Code.',
},
{
id: 'issueState',
title: 'Issue State',
type: 'dropdown',
required: false,
mode: 'advanced',
options: [
{ label: 'All', id: 'all' },
{ label: 'Open only', id: 'opened' },
{ label: 'Closed only', id: 'closed' },
],
description: 'Which issues to sync by state. Applies only when syncing issues.',
},
{
id: 'issueLabels',
title: 'Issue Labels',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. bug,docs (comma-separated)',
description:
'Only sync issues with all of these labels (comma-separated). Applies only when syncing issues.',
},
{
id: 'issueMilestone',
title: 'Issue Milestone',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. v1.0 (milestone title)',
description:
'Only sync issues assigned to this milestone (exact title). Applies only when syncing issues.',
},
{
id: 'maxItems',
title: 'Max Items',
type: 'short-input',
required: false,
placeholder: 'e.g. 500 (default: unlimited)',
},
],
tagDefinitions: [
{ id: 'contentType', displayName: 'Content Type', fieldType: 'text' },
{ id: 'title', displayName: 'Title', fieldType: 'text' },
{ id: 'state', displayName: 'State', fieldType: 'text' },
{ id: 'author', displayName: 'Author', fieldType: 'text' },
{ id: 'labels', displayName: 'Labels', fieldType: 'text' },
{ id: 'milestone', displayName: 'Milestone', fieldType: 'text' },
{ id: 'path', displayName: 'File Path', fieldType: 'text' },
{ id: 'size', displayName: 'File Size (bytes)', fieldType: 'number' },
{ id: 'createdAt', displayName: 'Created At', fieldType: 'date' },
{ id: 'updatedAt', displayName: 'Updated At', fieldType: 'date' },
],
}
+575
View File
@@ -0,0 +1,575 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { DEFAULT_MAX_THREADS, gmailConnectorMeta } from '@/connectors/gmail/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { htmlToPlainText, joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils'
const logger = createLogger('GmailConnector')
const GMAIL_API_BASE = 'https://gmail.googleapis.com/gmail/v1/users/me'
const THREADS_PER_PAGE = 100
interface GmailHeader {
name: string
value: string
}
interface GmailMessagePart {
mimeType?: string
body?: { data?: string; size?: number }
parts?: GmailMessagePart[]
headers?: GmailHeader[]
}
interface GmailMessage {
id: string
threadId: string
internalDate?: string
payload?: GmailMessagePart
labelIds?: string[]
snippet?: string
}
interface GmailThread {
id: string
historyId?: string
messages?: GmailMessage[]
snippet?: string
}
interface GmailLabel {
id: string
name: string
type?: string
}
/**
* Formats a single Gmail label name for use in a `label:` operator.
* Gmail search syntax accepts quoted strings for labels containing spaces;
* unquoted label tokens have spaces replaced with hyphens.
*/
function formatLabelToken(name: string): string {
const trimmed = name.trim()
if (!trimmed) return ''
if (/\s/.test(trimmed)) {
const escaped = trimmed.replace(/\\/g, '\\\\').replace(/"/g, '\\"')
return `label:"${escaped}"`
}
return `label:${trimmed}`
}
/**
* Builds a Gmail search query string from the source config.
* Combines the user's custom query with the label and date range filters.
* When multiple labels are provided, they are OR-joined: `(label:A OR label:B)`.
*/
function buildSearchQuery(sourceConfig: Record<string, unknown>): string {
const parts: string[] = []
const labelNames = parseMultiValue(sourceConfig.label)
if (labelNames.length === 1) {
const token = formatLabelToken(labelNames[0])
if (token) parts.push(token)
} else if (labelNames.length > 1) {
const tokens = labelNames.map(formatLabelToken).filter(Boolean)
if (tokens.length === 1) {
parts.push(tokens[0])
} else if (tokens.length > 1) {
parts.push(`(${tokens.join(' OR ')})`)
}
}
const dateRange = (sourceConfig.dateRange as string) || 'all'
const now = new Date()
switch (dateRange) {
case '7d':
parts.push(`after:${formatGmailDate(daysAgo(now, 7))}`)
break
case '30d':
parts.push(`after:${formatGmailDate(daysAgo(now, 30))}`)
break
case '90d':
parts.push(`after:${formatGmailDate(daysAgo(now, 90))}`)
break
case '6m':
parts.push(`after:${formatGmailDate(daysAgo(now, 180))}`)
break
case '1y':
parts.push(`after:${formatGmailDate(daysAgo(now, 365))}`)
break
}
const excludePromotions = sourceConfig.excludePromotions !== 'false'
if (excludePromotions) {
parts.push('-category:promotions')
}
const excludeSocial = sourceConfig.excludeSocial !== 'false'
if (excludeSocial) {
parts.push('-category:social')
}
const customQuery = sourceConfig.query as string | undefined
const trimmedCustom = customQuery?.trim()
if (trimmedCustom) {
/**
* Wrap the user-supplied query in parentheses whenever it contains an OR
* so it's AND-joined as a single clause with the preceding label / category
* / date filters. Always wrap (rather than try to detect existing outer
* parens) because a regex like /^\(.*\)$/ misclassifies inputs such as
* `(from:alice) OR (from:bob)` where the parens don't bracket the whole
* expression. Double-wrapping is a no-op in Gmail search syntax.
*/
const needsGroup = /\bOR\b/i.test(trimmedCustom)
parts.push(needsGroup ? `(${trimmedCustom})` : trimmedCustom)
}
return parts.join(' ')
}
function daysAgo(now: Date, days: number): Date {
return new Date(now.getTime() - days * 24 * 60 * 60 * 1000)
}
function formatGmailDate(date: Date): string {
const y = date.getFullYear()
const m = String(date.getMonth() + 1).padStart(2, '0')
const d = String(date.getDate()).padStart(2, '0')
return `${y}/${m}/${d}`
}
/**
* Decodes base64url-encoded content from the Gmail API.
* Uses Buffer to correctly handle multi-byte UTF-8 characters.
*/
function decodeBase64Url(data: string): string {
return Buffer.from(data, 'base64url').toString('utf-8')
}
/**
* Extracts the plain text body from a Gmail message payload.
* Prefers text/plain, falls back to text/html with tag stripping.
*/
function extractBody(part: GmailMessagePart): string {
if (part.mimeType === 'text/plain' && part.body?.data) {
return decodeBase64Url(part.body.data)
}
if (part.parts) {
// Prefer text/plain from multipart
for (const child of part.parts) {
if (child.mimeType === 'text/plain' && child.body?.data) {
return decodeBase64Url(child.body.data)
}
}
// Fall back to text/html
for (const child of part.parts) {
if (child.mimeType === 'text/html' && child.body?.data) {
return htmlToPlainText(decodeBase64Url(child.body.data))
}
}
// Recurse into nested multipart
for (const child of part.parts) {
const result = extractBody(child)
if (result) return result
}
}
if (part.mimeType === 'text/html' && part.body?.data) {
return htmlToPlainText(decodeBase64Url(part.body.data))
}
return ''
}
/**
* Gets a header value from a Gmail message payload.
*/
function getHeader(payload: GmailMessagePart | undefined, name: string): string | undefined {
if (!payload?.headers) return undefined
const header = payload.headers.find((h) => h.name.toLowerCase() === name.toLowerCase())
return header?.value
}
/**
* Formats a thread's messages into a single document string.
*/
function formatThread(thread: GmailThread): {
content: string
subject: string
metadata: Record<string, unknown>
} {
const messages = thread.messages || []
if (messages.length === 0) {
return { content: '', subject: 'Untitled Thread', metadata: {} }
}
const firstMessage = messages[0]
const lastMessage = messages[messages.length - 1]
const subject = getHeader(firstMessage.payload, 'Subject') || 'No Subject'
const from = getHeader(firstMessage.payload, 'From') || 'Unknown'
const to = getHeader(firstMessage.payload, 'To') || ''
const labelIds = firstMessage.labelIds || []
const lines: string[] = []
lines.push(`Subject: ${subject}`)
lines.push(`From: ${from}`)
if (to) lines.push(`To: ${to}`)
lines.push(`Messages: ${messages.length}`)
lines.push('')
for (const msg of messages) {
const msgFrom = getHeader(msg.payload, 'From') || 'Unknown'
const msgDate = getHeader(msg.payload, 'Date') || ''
const body = msg.payload ? extractBody(msg.payload) : ''
lines.push(`--- ${msgFrom} (${msgDate}) ---`)
lines.push(body.trim())
lines.push('')
}
const firstDate = firstMessage.internalDate
? new Date(Number(firstMessage.internalDate)).toISOString()
: undefined
const lastDate = lastMessage.internalDate
? new Date(Number(lastMessage.internalDate)).toISOString()
: undefined
return {
content: lines.join('\n').trim(),
subject,
metadata: {
from,
to,
subject,
messageCount: messages.length,
labelIds,
firstMessageDate: firstDate,
lastMessageDate: lastDate,
},
}
}
/**
* Fetches a full thread with all its messages.
*/
async function fetchThread(accessToken: string, threadId: string): Promise<GmailThread | null> {
const url = `${GMAIL_API_BASE}/threads/${threadId}?format=FULL`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) {
if (response.status === 404) return null
throw new Error(`Failed to fetch thread ${threadId}: ${response.status}`)
}
return (await response.json()) as GmailThread
}
/**
* Resolves label IDs to human-readable label names using a cache.
*/
async function resolveLabelNames(
accessToken: string,
labelIds: string[],
syncContext?: Record<string, unknown>
): Promise<string[]> {
const cacheKey = '_gmailLabelCache'
if (syncContext && !syncContext[cacheKey]) {
try {
const url = `${GMAIL_API_BASE}/labels`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (response.ok) {
const data = await response.json()
const labels = (data.labels || []) as GmailLabel[]
const labelMap: Record<string, string> = {}
for (const label of labels) {
labelMap[label.id] = label.name
}
syncContext[cacheKey] = labelMap
}
} catch {
syncContext[cacheKey] = {}
}
}
const cache = (syncContext?.[cacheKey] as Record<string, string>) ?? {}
return labelIds
.map((id) => cache[id] || id)
.filter((name) => !name.startsWith('CATEGORY_') && name !== 'UNREAD')
}
/**
* Creates a lightweight document stub from a thread list entry.
* Uses metadata-based contentHash for change detection without downloading content.
*/
function threadToStub(thread: {
id: string
snippet?: string
historyId?: string
}): ExternalDocument {
return {
externalId: thread.id,
title: thread.snippet || 'Untitled Thread',
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: `https://mail.google.com/mail/u/0/#inbox/${thread.id}`,
contentHash: `gmail:${thread.id}:${thread.historyId ?? ''}`,
metadata: {},
}
}
export const gmailConnector: ConnectorConfig = {
...gmailConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const searchQuery = buildSearchQuery(sourceConfig)
const maxThreads = sourceConfig.maxThreads
? Number(sourceConfig.maxThreads)
: DEFAULT_MAX_THREADS
const totalFetched = (syncContext?.totalThreadsFetched as number) ?? 0
if (totalFetched >= maxThreads) {
return { documents: [], hasMore: false }
}
const remaining = maxThreads - totalFetched
const pageSize = Math.min(THREADS_PER_PAGE, remaining)
const queryParams = new URLSearchParams({
maxResults: String(pageSize),
})
if (searchQuery) {
queryParams.set('q', searchQuery)
}
if (cursor) {
queryParams.set('pageToken', cursor)
}
const url = `${GMAIL_API_BASE}/threads?${queryParams.toString()}`
logger.info('Listing Gmail threads', {
query: searchQuery,
cursor: cursor ?? 'initial',
maxThreads,
})
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) {
const errorText = await response.text()
logger.error('Failed to list Gmail threads', { status: response.status, error: errorText })
throw new Error(`Failed to list Gmail threads: ${response.status}`)
}
const data = await response.json()
const threads = (data.threads || []) as { id: string; snippet?: string; historyId?: string }[]
if (threads.length === 0) {
return { documents: [], hasMore: false }
}
const documents = threads.map(threadToStub)
const newTotal = totalFetched + documents.length
if (syncContext) syncContext.totalThreadsFetched = newTotal
const nextPageToken = data.nextPageToken as string | undefined
const hitLimit = newTotal >= maxThreads
if (hitLimit && syncContext) syncContext.listingCapped = true
return {
documents,
nextCursor: hitLimit ? undefined : nextPageToken,
hasMore: hitLimit ? false : Boolean(nextPageToken),
}
},
getDocument: async (
accessToken: string,
_sourceConfig: Record<string, unknown>,
externalId: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocument | null> => {
try {
const thread = await fetchThread(accessToken, externalId)
if (!thread) return null
const { content, subject, metadata } = formatThread(thread)
if (!content.trim()) return null
const labelIds = (metadata.labelIds as string[]) || []
const labelNames = await resolveLabelNames(accessToken, labelIds, syncContext)
metadata.labels = labelNames
return {
externalId: thread.id,
title: subject,
content,
contentDeferred: false,
mimeType: 'text/plain',
sourceUrl: `https://mail.google.com/mail/u/0/#inbox/${thread.id}`,
contentHash: `gmail:${thread.id}:${thread.historyId ?? ''}`,
metadata,
}
} catch (error) {
logger.warn('Failed to get Gmail thread', {
externalId,
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const maxThreads = sourceConfig.maxThreads as string | undefined
if (maxThreads && (Number.isNaN(Number(maxThreads)) || Number(maxThreads) <= 0)) {
return { valid: false, error: 'Max threads must be a positive number' }
}
try {
// Verify Gmail API access by fetching profile
const profileUrl = `${GMAIL_API_BASE}/profile`
const profileResponse = await fetchWithRetry(
profileUrl,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
},
VALIDATE_RETRY_OPTIONS
)
if (!profileResponse.ok) {
return { valid: false, error: `Failed to access Gmail: ${profileResponse.status}` }
}
// If labels are specified, verify each one exists
const labelNames = parseMultiValue(sourceConfig.label)
if (labelNames.length > 0) {
const labelsUrl = `${GMAIL_API_BASE}/labels`
const labelsResponse = await fetchWithRetry(
labelsUrl,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
},
VALIDATE_RETRY_OPTIONS
)
if (!labelsResponse.ok) {
return { valid: false, error: 'Failed to fetch labels' }
}
const labelsData = await labelsResponse.json()
const labels = (labelsData.labels || []) as GmailLabel[]
const labelNameSet = new Set(labels.map((l) => l.name.toLowerCase()))
const missing = labelNames.filter((name) => !labelNameSet.has(name.toLowerCase()))
if (missing.length > 0) {
return {
valid: false,
error: `Label(s) not found: ${missing.join(', ')}. Available labels: ${labels
.filter(
(l) =>
l.type !== 'system' ||
['INBOX', 'IMPORTANT', 'STARRED', 'SENT', 'DRAFT'].includes(l.id)
)
.map((l) => l.name)
.slice(0, 15)
.join(', ')}`,
}
}
}
// If a custom query is specified, verify it's valid by doing a dry-run
const query = sourceConfig.query as string | undefined
if (query?.trim()) {
const searchQuery = buildSearchQuery(sourceConfig)
const testUrl = `${GMAIL_API_BASE}/threads?q=${encodeURIComponent(searchQuery)}&maxResults=1`
const testResponse = await fetchWithRetry(
testUrl,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
},
VALIDATE_RETRY_OPTIONS
)
if (!testResponse.ok) {
return { valid: false, error: 'Invalid search query. Check Gmail search syntax.' }
}
}
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.from === 'string') {
result.from = metadata.from
}
const labels = joinTagArray(metadata.labels)
if (labels) {
result.labels = labels
}
if (typeof metadata.messageCount === 'number') {
result.messageCount = metadata.messageCount
}
const lastMessageDate = parseTagDate(metadata.lastMessageDate)
if (lastMessageDate) {
result.lastMessageDate = lastMessageDate
}
return result
},
}
+1
View File
@@ -0,0 +1 @@
export { gmailConnector } from '@/connectors/gmail/gmail'
+100
View File
@@ -0,0 +1,100 @@
import { GmailIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const DEFAULT_MAX_THREADS = 500
export const gmailConnectorMeta: ConnectorMeta = {
id: 'gmail',
name: 'Gmail',
description: 'Sync email threads from Gmail',
version: '1.0.0',
icon: GmailIcon,
auth: {
mode: 'oauth',
provider: 'google-email',
requiredScopes: ['https://www.googleapis.com/auth/gmail.modify'],
},
configFields: [
{
id: 'labelSelector',
title: 'Labels',
type: 'selector',
selectorKey: 'gmail.labels',
canonicalParamId: 'label',
mode: 'basic',
multi: true,
placeholder: 'Select one or more labels',
required: false,
description: 'Only sync emails matching any of these labels. Leave empty for all mail.',
},
{
id: 'label',
title: 'Labels',
type: 'short-input',
canonicalParamId: 'label',
mode: 'advanced',
multi: true,
placeholder: 'e.g. INBOX, IMPORTANT (comma-separated; commas in label names not supported)',
required: false,
description: 'Only sync emails matching any of these labels. Leave empty for all mail.',
},
{
id: 'dateRange',
title: 'Date Range',
type: 'dropdown',
required: false,
options: [
{ label: 'Last 7 days', id: '7d' },
{ label: 'Last 30 days', id: '30d' },
{ label: 'Last 90 days', id: '90d' },
{ label: 'Last 6 months', id: '6m' },
{ label: 'Last year', id: '1y' },
{ label: 'All time', id: 'all' },
],
},
{
id: 'excludePromotions',
title: 'Exclude Promotions',
type: 'dropdown',
required: false,
options: [
{ label: 'Yes (recommended)', id: 'true' },
{ label: 'No', id: 'false' },
],
},
{
id: 'excludeSocial',
title: 'Exclude Social',
type: 'dropdown',
required: false,
options: [
{ label: 'Yes (recommended)', id: 'true' },
{ label: 'No', id: 'false' },
],
},
{
id: 'query',
title: 'Search Filter',
type: 'short-input',
placeholder: 'e.g. from:boss@company.com subject:report has:attachment',
required: false,
description: 'Additional Gmail search filter. Uses the same syntax as the Gmail search bar.',
},
{
id: 'maxThreads',
title: 'Max Threads',
type: 'short-input',
required: false,
placeholder: `e.g. 200 (default: ${DEFAULT_MAX_THREADS})`,
},
],
tagDefinitions: [
{ id: 'from', displayName: 'From', fieldType: 'text' },
{ id: 'labels', displayName: 'Labels', fieldType: 'text' },
{ id: 'messageCount', displayName: 'Messages in Thread', fieldType: 'number' },
{ id: 'lastMessageDate', displayName: 'Last Message', fieldType: 'date' },
],
}
+540
View File
@@ -0,0 +1,540 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { gongConnectorMeta } from '@/connectors/gong/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { parseTagDate } from '@/connectors/utils'
const logger = createLogger('GongConnector')
const GONG_API_BASE = 'https://api.gong.io/v2'
const DEFAULT_LOOKBACK_DAYS = 90
const MAX_LOOKBACK_DAYS = 180
/**
* Days of overlap added when computing the incremental sync window. Gong call data
* (parties, transcript) can finish processing minutes to hours — occasionally a
* day or two — after a call ends. The sync engine re-attempts calls whose
* transcript was not yet ready (a null getDocument result is never persisted, so
* the call is re-listed and re-fetched on the next sync), but only while the call
* stays inside the incremental window. A two-week overlap keeps recently-ended
* calls in that window long enough for late transcripts to be picked up, at the
* cost of re-listing already-synced calls (skipped downstream by content hash).
*/
const INCREMENTAL_OVERLAP_DAYS = 14
const MS_PER_DAY = 24 * 60 * 60 * 1000
/**
* Metadata for a single call participant. `speakerId` cross-references the
* `speakerId` field on transcript monologues, letting the connector attribute
* each spoken line to a named participant.
*/
interface GongParty {
id?: string
name?: string
emailAddress?: string
speakerId?: string
affiliation?: string
}
/**
* Core call metadata returned by the extensive calls endpoint. Mirrors Gong's
* `CallBasicData` (the `metaData` object) — every field here is present on the
* `/v2/calls/extensive` stub response and never requires the transcript fetch.
*/
interface GongCallMetaData {
id?: string
title?: string
scheduled?: string
started?: string
duration?: number
url?: string
workspaceId?: string
primaryUserId?: string
direction?: string
scope?: string
system?: string
language?: string
purpose?: string
isPrivate?: boolean
}
/**
* A single call object from POST /v2/calls/extensive.
*/
interface GongExtensiveCall {
metaData?: GongCallMetaData
parties?: GongParty[]
}
interface GongRecords {
cursor?: string
totalRecords?: number
currentPageSize?: number
}
interface GongExtensiveCallsResponse {
calls?: GongExtensiveCall[]
records?: GongRecords
}
/**
* A single sentence within a transcript monologue. Gong returns timing in
* `startMs`/`endMs`; only `text` is used for the formatted transcript.
*/
interface GongTranscriptSentence {
text?: string
}
/**
* A monologue (one speaker turn) within a call transcript.
*/
interface GongMonologue {
speakerId?: string
topic?: string
sentences?: GongTranscriptSentence[]
}
interface GongCallTranscript {
callId?: string
transcript?: GongMonologue[]
}
interface GongTranscriptResponse {
callTranscripts?: GongCallTranscript[]
records?: GongRecords
}
/**
* Builds the Authorization header value for Gong's Basic auth scheme.
*
* Gong authenticates with `Basic base64(accessKey:accessKeySecret)`. The sync
* engine passes the user's stored key as `accessToken`. To support both raw
* `accessKey:accessKeySecret` pairs and pre-encoded credentials, the raw form
* (containing a colon) is base64-encoded here; an already-encoded value is sent
* as-is.
*/
function buildAuthHeader(accessToken: string): string {
const token = accessToken.includes(':')
? Buffer.from(accessToken, 'utf8').toString('base64')
: accessToken
return `Basic ${token}`
}
function buildHeaders(accessToken: string): Record<string, string> {
return {
'Content-Type': 'application/json',
Accept: 'application/json',
Authorization: buildAuthHeader(accessToken),
}
}
/**
* Parses a comma- or newline-separated list of Gong IDs into a trimmed,
* de-duplicated, non-empty array. Returns `undefined` when nothing usable
* remains so the caller can omit the filter key entirely.
*/
function parseIdList(raw: unknown): string[] | undefined {
if (typeof raw !== 'string') return undefined
const ids = Array.from(
new Set(
raw
.split(/[\n,]/)
.map((value) => value.trim())
.filter((value) => value.length > 0)
)
)
return ids.length > 0 ? ids : undefined
}
/**
* Metadata-based content hash shared by `listDocuments` stubs and `getDocument`
* results. Derived purely from call identity and its start time so the value is
* identical across both paths — guaranteeing the sync engine only re-fetches a
* transcript when the call's metadata actually changes.
*/
function buildContentHash(callId: string, started: string | undefined): string {
return `gong:${callId}:${started ?? ''}`
}
function buildCallTitle(metaData: GongCallMetaData | undefined): string {
return metaData?.title?.trim() || 'Untitled Gong Call'
}
/**
* Extracts named participant labels from a call's parties for tag mapping and
* the transcript header.
*/
function buildParticipantNames(parties: GongParty[] | undefined): string[] {
if (!parties) return []
const names: string[] = []
for (const party of parties) {
const label = party.name?.trim() || party.emailAddress?.trim()
if (label) names.push(label)
}
return names
}
/**
* Builds a `speakerId` → display-name map from a call's parties so transcript
* monologues (keyed by `speakerId`) can be attributed to a named speaker.
*/
function buildSpeakerMap(parties: GongParty[] | undefined): Record<string, string> {
const map: Record<string, string> = {}
if (!parties) return map
for (const party of parties) {
if (!party.speakerId) continue
const label = party.name?.trim() || party.emailAddress?.trim()
if (label) map[party.speakerId] = label
}
return map
}
function buildMetadata(
metaData: GongCallMetaData | undefined,
participants: string[]
): Record<string, unknown> {
return {
callId: metaData?.id,
callTitle: metaData?.title,
callDate: metaData?.started,
scheduledDate: metaData?.scheduled,
duration: metaData?.duration,
workspaceId: metaData?.workspaceId,
primaryUserId: metaData?.primaryUserId,
direction: metaData?.direction,
scope: metaData?.scope,
system: metaData?.system,
language: metaData?.language,
purpose: metaData?.purpose,
isPrivate: metaData?.isPrivate,
participants,
}
}
/**
* Formats a call's transcript into speaker-attributed plain text with a header
* describing the call (title, date, duration, participants).
*/
function formatTranscriptContent(
metaData: GongCallMetaData | undefined,
participants: string[],
speakerMap: Record<string, string>,
monologues: GongMonologue[]
): string {
const parts: string[] = []
parts.push(`Call: ${buildCallTitle(metaData)}`)
if (metaData?.started) parts.push(`Date: ${metaData.started}`)
if (metaData?.duration != null) {
const minutes = Math.round(metaData.duration / 60)
parts.push(`Duration: ${minutes} minutes`)
}
if (participants.length > 0) parts.push(`Participants: ${participants.join(', ')}`)
parts.push('')
parts.push('--- Transcript ---')
for (const monologue of monologues) {
const speaker = (monologue.speakerId && speakerMap[monologue.speakerId]) || 'Unknown Speaker'
const text = (monologue.sentences ?? [])
.map((sentence) => sentence.text?.trim())
.filter((value): value is string => Boolean(value))
.join(' ')
if (text) parts.push(`${speaker}: ${text}`)
}
return parts.join('\n')
}
/**
* Computes the effective lookback window in days, narrowing to the time since
* the last successful sync (plus an overlap to catch transcripts that finished
* processing late) when incremental sync is active.
*/
function computeLookbackDays(
sourceConfig: Record<string, unknown>,
lastSyncAt: Date | undefined
): number {
const raw = sourceConfig.lookback as string | undefined
const configured = Number(raw)
const baseline =
Number.isFinite(configured) && configured > 0
? Math.min(Math.floor(configured), MAX_LOOKBACK_DAYS)
: DEFAULT_LOOKBACK_DAYS
if (!lastSyncAt) return baseline
const sinceLastSync = Math.ceil((Date.now() - lastSyncAt.getTime()) / MS_PER_DAY)
const incremental = Math.max(sinceLastSync + INCREMENTAL_OVERLAP_DAYS, INCREMENTAL_OVERLAP_DAYS)
return Math.min(incremental, baseline)
}
/**
* Fetches a single page of calls from POST /v2/calls/extensive with parties
* exposed (needed to resolve transcript speaker IDs to names).
*/
async function fetchExtensiveCalls(
accessToken: string,
filter: Record<string, unknown>,
cursor: string | undefined,
retryOptions?: Parameters<typeof fetchWithRetry>[2]
): Promise<GongExtensiveCallsResponse> {
const body: Record<string, unknown> = {
filter,
contentSelector: { exposedFields: { parties: true } },
}
if (cursor) body.cursor = cursor
const response = await fetchWithRetry(
`${GONG_API_BASE}/calls/extensive`,
{
method: 'POST',
headers: buildHeaders(accessToken),
body: JSON.stringify(body),
},
retryOptions
)
if (!response.ok) {
const errorText = await response.text().catch(() => '')
throw new Error(
`Failed to list Gong calls: ${response.status}${errorText ? `${errorText.slice(0, 200)}` : ''}`
)
}
return (await response.json()) as GongExtensiveCallsResponse
}
export const gongConnector: ConnectorConfig = {
...gongConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>,
lastSyncAt?: Date
): Promise<ExternalDocumentList> => {
const lookbackDays = computeLookbackDays(sourceConfig, lastSyncAt)
const maxCalls = sourceConfig.maxCalls ? Number(sourceConfig.maxCalls) : 0
const workspaceId = (sourceConfig.workspaceId as string | undefined)?.trim()
const primaryUserIds = parseIdList(sourceConfig.primaryUserIds)
const cachedWindow = syncContext?.gongDateWindow as
| { fromDateTime: string; toDateTime: string }
| undefined
const now = new Date()
const window = cachedWindow ?? {
fromDateTime: new Date(now.getTime() - lookbackDays * MS_PER_DAY).toISOString(),
toDateTime: now.toISOString(),
}
if (syncContext && !cachedWindow) syncContext.gongDateWindow = window
const { fromDateTime, toDateTime } = window
const filter: Record<string, unknown> = { fromDateTime, toDateTime }
if (workspaceId) filter.workspaceId = workspaceId
if (primaryUserIds) filter.primaryUserIds = primaryUserIds
logger.info('Listing Gong calls', {
fromDateTime,
toDateTime,
hasCursor: Boolean(cursor),
incremental: Boolean(lastSyncAt),
})
const data = await fetchExtensiveCalls(accessToken, filter, cursor)
const calls = data.calls ?? []
const nextPageCursor = data.records?.cursor?.trim() || undefined
const allDocuments: ExternalDocument[] = []
for (const call of calls) {
const callId = call.metaData?.id
if (!callId) continue
const participants = buildParticipantNames(call.parties)
allDocuments.push({
externalId: callId,
title: buildCallTitle(call.metaData),
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: call.metaData?.url || undefined,
contentHash: buildContentHash(callId, call.metaData?.started),
metadata: buildMetadata(call.metaData, participants),
})
}
const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
let documents = allDocuments
let capDroppedDocs = false
if (maxCalls > 0) {
const remaining = Math.max(0, maxCalls - prevFetched)
if (allDocuments.length > remaining) {
documents = allDocuments.slice(0, remaining)
capDroppedDocs = true
}
}
const totalFetched = prevFetched + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = maxCalls > 0 && totalFetched >= maxCalls
const hasMore = !hitLimit && Boolean(nextPageCursor)
/**
* Only flag the listing as capped when the `maxCalls` limit actually
* truncated calls that still exist in the source — either by dropping calls
* from the current page or by stopping while another page remains. Reaching
* the limit exactly at source exhaustion (no dropped calls, no further
* cursor) yields a complete listing, so deletion reconciliation must still
* run for calls removed in Gong.
*/
if (syncContext && (capDroppedDocs || (hitLimit && Boolean(nextPageCursor)))) {
syncContext.listingCapped = true
}
return {
documents,
nextCursor: hasMore ? nextPageCursor : undefined,
hasMore,
}
},
getDocument: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
try {
if (!externalId) return null
const workspaceId = (sourceConfig.workspaceId as string | undefined)?.trim()
const filter: Record<string, unknown> = { callIds: [externalId] }
if (workspaceId) filter.workspaceId = workspaceId
const callData = await fetchExtensiveCalls(accessToken, filter, undefined)
const call = callData.calls?.[0]
if (!call?.metaData?.id) {
logger.warn('Gong call not found', { externalId })
return null
}
const metaData = call.metaData
const participants = buildParticipantNames(call.parties)
const speakerMap = buildSpeakerMap(call.parties)
const transcriptResponse = await fetchWithRetry(`${GONG_API_BASE}/calls/transcript`, {
method: 'POST',
headers: buildHeaders(accessToken),
body: JSON.stringify({ filter: { callIds: [externalId] } }),
})
if (!transcriptResponse.ok) {
if (transcriptResponse.status === 404) return null
throw new Error(`Failed to fetch Gong transcript: ${transcriptResponse.status}`)
}
const transcriptData = (await transcriptResponse.json()) as GongTranscriptResponse
const callTranscript = transcriptData.callTranscripts?.find(
(entry) => entry.callId === externalId
)
const monologues = callTranscript?.transcript ?? []
if (monologues.length === 0) {
logger.info('Transcript not available for Gong call', { externalId })
return null
}
const hasSpokenText = monologues.some((monologue) =>
(monologue.sentences ?? []).some((sentence) => Boolean(sentence.text?.trim()))
)
if (!hasSpokenText) return null
const content = formatTranscriptContent(metaData, participants, speakerMap, monologues)
return {
externalId: metaData.id ?? externalId,
title: buildCallTitle(metaData),
content,
contentDeferred: false,
mimeType: 'text/plain',
sourceUrl: metaData.url || undefined,
contentHash: buildContentHash(metaData.id ?? externalId, metaData.started),
metadata: buildMetadata(metaData, participants),
}
} catch (error) {
logger.warn('Failed to get Gong call transcript', {
externalId,
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const maxCalls = sourceConfig.maxCalls as string | undefined
if (maxCalls && (Number.isNaN(Number(maxCalls)) || Number(maxCalls) < 0)) {
return { valid: false, error: 'Max calls must be a non-negative number' }
}
try {
const response = await fetchWithRetry(
`${GONG_API_BASE}/users`,
{
method: 'GET',
headers: buildHeaders(accessToken),
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
const errorText = await response.text().catch(() => '')
return {
valid: false,
error: `Gong access failed: ${response.status}${errorText ? `${errorText.slice(0, 200)}` : ''}`,
}
}
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.callTitle === 'string' && metadata.callTitle.trim()) {
result.callTitle = metadata.callTitle
}
const participants = Array.isArray(metadata.participants)
? (metadata.participants as string[])
: []
if (participants.length > 0) {
result.participants = participants.join(', ')
}
if (metadata.duration != null) {
const num = Number(metadata.duration)
if (!Number.isNaN(num)) result.duration = num
}
const callDate = parseTagDate(metadata.callDate)
if (callDate) result.callDate = callDate
const scheduledDate = parseTagDate(metadata.scheduledDate)
if (scheduledDate) result.scheduledDate = scheduledDate
const textTags = ['direction', 'scope', 'system', 'language', 'purpose'] as const
for (const key of textTags) {
const value = metadata[key]
if (typeof value === 'string' && value.trim()) result[key] = value.trim()
}
if (typeof metadata.isPrivate === 'boolean') result.isPrivate = metadata.isPrivate
return result
},
}
+1
View File
@@ -0,0 +1 @@
export { gongConnector } from '@/connectors/gong/gong'
+72
View File
@@ -0,0 +1,72 @@
import { GongIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const gongConnectorMeta: ConnectorMeta = {
id: 'gong',
name: 'Gong',
description: 'Sync call transcripts from Gong revenue intelligence',
version: '1.0.0',
icon: GongIcon,
auth: {
mode: 'apiKey',
label: 'Access Key & Secret',
placeholder: 'accessKey:accessKeySecret',
},
supportsIncrementalSync: true,
configFields: [
{
id: 'lookback',
title: 'Date Range',
type: 'dropdown',
required: false,
options: [
{ label: 'Last 30 days', id: '30' },
{ label: 'Last 90 days (recommended)', id: '90' },
{ label: 'Last 6 months', id: '180' },
],
description:
'On initial sync only. Controls how far back to look for calls with transcripts.',
},
{
id: 'maxCalls',
title: 'Max Calls',
type: 'short-input',
required: false,
placeholder: 'e.g. 200 (default: unlimited)',
},
{
id: 'workspaceId',
title: 'Workspace ID',
type: 'short-input',
required: false,
placeholder: 'Optional — limit to a single Gong workspace',
},
{
id: 'primaryUserIds',
title: 'Host User IDs',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'Optional — comma-separated Gong user IDs (call hosts)',
description:
'Only sync calls hosted by these users. Find IDs in Gong under Company Settings → Users, or via the API.',
},
],
tagDefinitions: [
{ id: 'callTitle', displayName: 'Call Title', fieldType: 'text' },
{ id: 'participants', displayName: 'Participants', fieldType: 'text' },
{ id: 'duration', displayName: 'Duration (seconds)', fieldType: 'number' },
{ id: 'callDate', displayName: 'Call Date', fieldType: 'date' },
{ id: 'scheduledDate', displayName: 'Scheduled Date', fieldType: 'date' },
{ id: 'direction', displayName: 'Direction', fieldType: 'text' },
{ id: 'scope', displayName: 'Scope', fieldType: 'text' },
{ id: 'system', displayName: 'System', fieldType: 'text' },
{ id: 'language', displayName: 'Language', fieldType: 'text' },
{ id: 'purpose', displayName: 'Purpose', fieldType: 'text' },
{ id: 'isPrivate', displayName: 'Private', fieldType: 'boolean' },
],
}
@@ -0,0 +1,510 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { DEFAULT_MAX_EVENTS, googleCalendarConnectorMeta } from '@/connectors/google-calendar/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { parseMultiValue, parseTagDate } from '@/connectors/utils'
const logger = createLogger('GoogleCalendarConnector')
const CALENDAR_API_BASE = 'https://www.googleapis.com/calendar/v3'
const DEFAULT_RANGE_DAYS = 30
const PAGE_SIZE = 250
interface CalendarEventTime {
date?: string
dateTime?: string
timeZone?: string
}
interface CalendarAttendee {
email?: string
displayName?: string
responseStatus?: string
self?: boolean
resource?: boolean
optional?: boolean
}
interface CalendarEvent {
id: string
status?: string
htmlLink?: string
created?: string
updated?: string
summary?: string
description?: string
location?: string
creator?: { email?: string; displayName?: string }
organizer?: { email?: string; displayName?: string; self?: boolean }
start?: CalendarEventTime
end?: CalendarEventTime
attendees?: CalendarAttendee[]
recurringEventId?: string
eventType?: string
}
/**
* Formats a CalendarEventTime into a human-readable string.
* All-day events use the date field; timed events use dateTime.
*/
function formatEventTime(eventTime?: CalendarEventTime): string {
if (!eventTime) return 'Unknown'
if (eventTime.dateTime) {
const date = new Date(eventTime.dateTime)
return date.toLocaleString('en-US', {
weekday: 'long',
year: 'numeric',
month: 'long',
day: 'numeric',
hour: 'numeric',
minute: '2-digit',
timeZoneName: 'short',
timeZone: eventTime.timeZone || undefined,
})
}
if (eventTime.date) {
const date = new Date(`${eventTime.date}T00:00:00`)
return date.toLocaleDateString('en-US', {
weekday: 'long',
year: 'numeric',
month: 'long',
day: 'numeric',
})
}
return 'Unknown'
}
/**
* Determines whether the event is all-day based on whether `date` (not `dateTime`) is used.
*/
function isAllDayEvent(event: CalendarEvent): boolean {
return Boolean(event.start?.date && !event.start?.dateTime)
}
/**
* Formats attendees into a comma-separated list of names/emails.
*/
function formatAttendees(attendees?: CalendarAttendee[]): string {
if (!attendees || attendees.length === 0) return ''
return attendees
.filter((a) => !a.resource)
.map((a) => a.displayName || a.email || 'Unknown')
.join(', ')
}
/**
* Formats an organizer into a display string.
*/
function formatOrganizer(organizer?: { email?: string; displayName?: string }): string {
if (!organizer) return ''
if (organizer.displayName && organizer.email) {
return `${organizer.displayName} (${organizer.email})`
}
return organizer.displayName || organizer.email || ''
}
/**
* Builds a readable content string from a calendar event.
*/
function eventToContent(event: CalendarEvent): string {
const parts: string[] = []
parts.push(`Event: ${event.summary || 'Untitled Event'}`)
if (isAllDayEvent(event)) {
parts.push(`Date: ${formatEventTime(event.start)} (All Day)`)
} else {
parts.push(`Date: ${formatEventTime(event.start)} - ${formatEventTime(event.end)}`)
}
if (event.location) {
parts.push(`Location: ${event.location}`)
}
const organizer = formatOrganizer(event.organizer)
if (organizer) {
parts.push(`Organizer: ${organizer}`)
}
const attendees = formatAttendees(event.attendees)
if (attendees) {
parts.push(`Attendees: ${attendees}`)
}
if (event.description) {
parts.push('')
parts.push('Description:')
parts.push(
event.description
.replace(/<[^>]*>/g, ' ')
.replace(/\s+/g, ' ')
.trim()
)
}
return parts.join('\n')
}
/**
* Computes the default time range boundaries: 30 days in the past to 30 days in the future.
*/
function getDefaultTimeRange(): { timeMin: string; timeMax: string } {
const now = new Date()
const past = new Date(now)
past.setDate(past.getDate() - DEFAULT_RANGE_DAYS)
const future = new Date(now)
future.setDate(future.getDate() + DEFAULT_RANGE_DAYS)
return {
timeMin: past.toISOString(),
timeMax: future.toISOString(),
}
}
/**
* Parses the date range config into timeMin/timeMax values.
*/
function getTimeRange(sourceConfig: Record<string, unknown>): { timeMin: string; timeMax: string } {
const dateRange = (sourceConfig.dateRange as string) || 'default'
const now = new Date()
switch (dateRange) {
case 'past_only': {
const past = new Date(now)
past.setDate(past.getDate() - DEFAULT_RANGE_DAYS)
return { timeMin: past.toISOString(), timeMax: now.toISOString() }
}
case 'future_only': {
const future = new Date(now)
future.setDate(future.getDate() + DEFAULT_RANGE_DAYS)
return { timeMin: now.toISOString(), timeMax: future.toISOString() }
}
case 'past_90': {
const past = new Date(now)
past.setDate(past.getDate() - 90)
const future = new Date(now)
future.setDate(future.getDate() + 90)
return { timeMin: past.toISOString(), timeMax: future.toISOString() }
}
default:
return getDefaultTimeRange()
}
}
/**
* Converts a CalendarEvent to an ExternalDocument.
*
* Backward compatibility: when only a single calendar is configured (the only
* code path that existed before multi-calendar support), externalId and
* contentHash use the legacy non-namespaced format so existing connectors see
* zero churn on re-sync. When 2+ calendars are configured, we namespace by
* calendarId because Google Calendar event IDs are only unique within a
* single calendar.
*/
function eventToDocument(
event: CalendarEvent,
calendarId: string,
isMultiCalendar: boolean
): ExternalDocument | null {
if (event.status === 'cancelled') return null
const content = eventToContent(event)
if (!content.trim()) return null
const startTime = event.start?.dateTime || event.start?.date || ''
const attendeeCount = event.attendees?.filter((a) => !a.resource).length || 0
const externalId = isMultiCalendar ? `${calendarId}:${event.id}` : event.id
const contentHash = isMultiCalendar
? `gcal:${calendarId}:${event.id}:${event.updated ?? ''}`
: `gcal:${event.id}:${event.updated ?? ''}`
return {
externalId,
title: event.summary || 'Untitled Event',
content,
mimeType: 'text/plain',
sourceUrl: event.htmlLink || `https://calendar.google.com/calendar/event?eid=${event.id}`,
contentHash,
metadata: {
calendarId,
startTime,
endTime: event.end?.dateTime || event.end?.date || '',
location: event.location || '',
organizer: formatOrganizer(event.organizer),
attendeeCount,
isAllDay: isAllDayEvent(event),
eventDate: startTime,
updatedTime: event.updated,
createdTime: event.created,
},
}
}
export const googleCalendarConnector: ConnectorConfig = {
...googleCalendarConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const parsedCalendarIds = parseMultiValue(sourceConfig.calendarId)
const calendarIds = parsedCalendarIds.length > 0 ? parsedCalendarIds : ['primary']
const { timeMin, timeMax } = getTimeRange(sourceConfig)
const searchQuery = (sourceConfig.searchQuery as string) || ''
/**
* Cursor format:
* - For a single calendar with legacy cursors: the raw pageToken string
* - For multi-calendar walking: JSON-encoded { calendarIndex, pageToken }
*/
let calendarIndex = 0
let pageToken: string | undefined
if (cursor) {
try {
const parsed = JSON.parse(cursor) as { calendarIndex: number; pageToken?: string }
if (typeof parsed.calendarIndex === 'number') {
calendarIndex = parsed.calendarIndex
pageToken = parsed.pageToken
} else {
pageToken = cursor
}
} catch {
pageToken = cursor
}
}
if (calendarIndex >= calendarIds.length) {
return { documents: [], hasMore: false }
}
const calendarId = calendarIds[calendarIndex]
const queryParams = new URLSearchParams({
singleEvents: 'true',
orderBy: 'startTime',
maxResults: String(PAGE_SIZE),
timeMin,
timeMax,
})
if (searchQuery.trim()) {
queryParams.set('q', searchQuery.trim())
}
if (pageToken) {
queryParams.set('pageToken', pageToken)
}
const url = `${CALENDAR_API_BASE}/calendars/${encodeURIComponent(calendarId)}/events?${queryParams.toString()}`
logger.info('Listing Google Calendar events', {
calendarId,
calendarIndex,
calendarCount: calendarIds.length,
timeMin,
timeMax,
hasPageToken: Boolean(pageToken),
})
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) {
const errorText = await response.text()
logger.error('Failed to list Google Calendar events', {
status: response.status,
calendarId,
error: errorText,
})
throw new Error(`Failed to list Google Calendar events: ${response.status}`)
}
const data = await response.json()
const events = (data.items || []) as CalendarEvent[]
const isMultiCalendar = calendarIds.length > 1
const documents: ExternalDocument[] = []
for (const event of events) {
const doc = eventToDocument(event, calendarId, isMultiCalendar)
if (doc) documents.push(doc)
}
const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const maxEvents = sourceConfig.maxEvents ? Number(sourceConfig.maxEvents) : DEFAULT_MAX_EVENTS
const hitLimit = maxEvents > 0 && totalFetched >= maxEvents
const nextPageToken = data.nextPageToken as string | undefined
if (hitLimit) {
return { documents, hasMore: false }
}
if (nextPageToken) {
return {
documents,
nextCursor: JSON.stringify({ calendarIndex, pageToken: nextPageToken }),
hasMore: true,
}
}
const nextCalendarIndex = calendarIndex + 1
if (nextCalendarIndex < calendarIds.length) {
return {
documents,
nextCursor: JSON.stringify({ calendarIndex: nextCalendarIndex }),
hasMore: true,
}
}
return { documents, hasMore: false }
},
getDocument: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
/**
* externalId format depends on connector configuration:
* - Single-calendar (1 calendar configured): externalId = eventId (legacy
* and current single-calendar format).
* - Multi-calendar (2+ calendars configured): externalId =
* `calendarId:eventId`. The first `:` is the separator — event IDs never
* contain `:` while calendar IDs (e.g. `user@group.calendar.google.com`)
* may include URL-safe chars but not `:`.
*
* Legacy in-flight rows that lack a separator fall back to the configured
* calendar (or `primary`).
*/
const parsedCalendarIds = parseMultiValue(sourceConfig.calendarId)
const calendarIds = parsedCalendarIds.length > 0 ? parsedCalendarIds : ['primary']
/**
* Derive `isMultiCalendar` from the externalId itself, not from the current
* config. If a row was synced under a multi-calendar config and the user
* later removed calendars, the row's externalId still has the prefix —
* returning a doc without the prefix would mint a duplicate via the sync
* engine's externalId-keyed matching.
*/
const separatorIndex = externalId.indexOf(':')
const isMultiCalendar = separatorIndex !== -1
let calendarId: string
let eventId: string
if (separatorIndex === -1) {
calendarId = calendarIds[0] ?? 'primary'
eventId = externalId
} else {
calendarId = externalId.slice(0, separatorIndex)
eventId = externalId.slice(separatorIndex + 1)
}
const url = `${CALENDAR_API_BASE}/calendars/${encodeURIComponent(calendarId)}/events/${encodeURIComponent(eventId)}`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) {
if (response.status === 404) return null
throw new Error(`Failed to get Google Calendar event: ${response.status}`)
}
const event = (await response.json()) as CalendarEvent
if (event.status === 'cancelled') return null
return eventToDocument(event, calendarId, isMultiCalendar) ?? null
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const maxEvents = sourceConfig.maxEvents as string | undefined
if (maxEvents && (Number.isNaN(Number(maxEvents)) || Number(maxEvents) <= 0)) {
return { valid: false, error: 'Max events must be a positive number' }
}
const parsedCalendarIds = parseMultiValue(sourceConfig.calendarId)
const calendarIds = parsedCalendarIds.length > 0 ? parsedCalendarIds : ['primary']
try {
for (const calendarId of calendarIds) {
const url = `${CALENDAR_API_BASE}/calendars/${encodeURIComponent(calendarId)}/events?maxResults=1&singleEvents=true&orderBy=startTime&timeMin=${encodeURIComponent(new Date().toISOString())}`
const response = await fetchWithRetry(
url,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
if (response.status === 404) {
return {
valid: false,
error: `Calendar not found: ${calendarId}. Check the calendar ID.`,
}
}
return {
valid: false,
error: `Failed to access Google Calendar "${calendarId}": ${response.status}`,
}
}
}
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.organizer === 'string' && metadata.organizer) {
result.organizer = metadata.organizer
}
if (typeof metadata.attendeeCount === 'number') {
result.attendeeCount = metadata.attendeeCount
}
if (typeof metadata.location === 'string' && metadata.location) {
result.location = metadata.location
}
const eventDate = parseTagDate(metadata.eventDate)
if (eventDate) result.eventDate = eventDate
const lastModified = parseTagDate(metadata.updatedTime)
if (lastModified) result.lastModified = lastModified
const createdAt = parseTagDate(metadata.createdTime)
if (createdAt) result.createdAt = createdAt
return result
},
}
@@ -0,0 +1 @@
export { googleCalendarConnector } from '@/connectors/google-calendar/google-calendar'
@@ -0,0 +1,81 @@
import { GoogleCalendarIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const DEFAULT_MAX_EVENTS = 500
export const googleCalendarConnectorMeta: ConnectorMeta = {
id: 'google_calendar',
name: 'Google Calendar',
description: 'Sync calendar events from Google Calendar',
version: '1.0.0',
icon: GoogleCalendarIcon,
auth: {
mode: 'oauth',
provider: 'google-calendar',
requiredScopes: ['https://www.googleapis.com/auth/calendar'],
},
configFields: [
{
id: 'calendarSelector',
title: 'Calendars',
type: 'selector',
selectorKey: 'google.calendar',
canonicalParamId: 'calendarId',
mode: 'basic',
multi: true,
placeholder: 'Select one or more calendars',
required: false,
description: 'Calendars to sync from. Defaults to your primary calendar.',
},
{
id: 'calendarId',
title: 'Calendar IDs',
type: 'short-input',
canonicalParamId: 'calendarId',
mode: 'advanced',
multi: true,
placeholder: 'e.g. primary, team@group.calendar.google.com (comma-separated for multiple)',
required: false,
description:
'Calendars to sync from. Use "primary" for your main calendar. Defaults to "primary".',
},
{
id: 'dateRange',
title: 'Date Range',
type: 'dropdown',
required: false,
options: [
{ label: 'Last 30 days + next 30 days (default)', id: 'default' },
{ label: 'Past events only (last 30 days)', id: 'past_only' },
{ label: 'Future events only (next 30 days)', id: 'future_only' },
{ label: 'Extended range (90 days each way)', id: 'past_90' },
],
},
{
id: 'searchQuery',
title: 'Search Query',
type: 'short-input',
placeholder: 'e.g. standup, sprint review (optional)',
required: false,
description: 'Filter events by text search across all fields.',
},
{
id: 'maxEvents',
title: 'Max Events',
type: 'short-input',
required: false,
placeholder: `e.g. 500 (default: ${DEFAULT_MAX_EVENTS})`,
},
],
tagDefinitions: [
{ id: 'organizer', displayName: 'Organizer', fieldType: 'text' },
{ id: 'attendeeCount', displayName: 'Attendee Count', fieldType: 'number' },
{ id: 'location', displayName: 'Location', fieldType: 'text' },
{ id: 'eventDate', displayName: 'Event Date', fieldType: 'date' },
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
{ id: 'createdAt', displayName: 'Created', fieldType: 'date' },
],
}
@@ -0,0 +1,382 @@
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { googleDocsConnectorMeta } from '@/connectors/google-docs/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import {
buildDriveParentsClause,
joinTagArray,
parseMultiValue,
parseTagDate,
} from '@/connectors/utils'
const logger = createLogger('GoogleDocsConnector')
/**
* Represents a Google Drive file entry returned by the Drive API.
*/
interface DriveFile {
id: string
name: string
mimeType: string
modifiedTime?: string
createdTime?: string
webViewLink?: string
owners?: { displayName?: string; emailAddress?: string }[]
}
/**
* Represents a structural element within a Google Docs document body.
*/
interface DocsStructuralElement {
paragraph?: {
paragraphStyle?: {
namedStyleType?: string
}
elements?: {
textRun?: {
content?: string
}
}[]
}
}
/**
* Represents the response from the Google Docs API for a single document.
*/
interface DocsDocument {
documentId: string
title: string
body?: {
content?: DocsStructuralElement[]
}
}
/**
* Maps a Google Docs heading style to a Markdown heading prefix.
*/
function headingPrefix(namedStyleType?: string): string {
switch (namedStyleType) {
case 'HEADING_1':
return '# '
case 'HEADING_2':
return '## '
case 'HEADING_3':
return '### '
case 'HEADING_4':
return '#### '
case 'HEADING_5':
return '##### '
case 'HEADING_6':
return '###### '
default:
return ''
}
}
/**
* Extracts plain text from a Google Docs API structured document response.
* Headings are prefixed with Markdown-style `#` markers.
*/
function extractTextFromDocsBody(doc: DocsDocument): string {
const elements = doc.body?.content
if (!elements) return ''
const parts: string[] = []
for (const element of elements) {
const paragraph = element.paragraph
if (!paragraph?.elements) continue
const prefix = headingPrefix(paragraph.paragraphStyle?.namedStyleType)
/**
* Each paragraph's final `textRun.content` already ends with `\n`. Strip
* it before joining with `\n` so a heading followed by a body paragraph
* is separated by a single newline, not two.
*/
const text = paragraph.elements
.map((el) => el.textRun?.content ?? '')
.join('')
.replace(/\n+$/, '')
if (text.trim()) {
parts.push(`${prefix}${text}`)
}
}
return parts.join('\n').trim()
}
/**
* Fetches the structured content of a Google Doc via the Docs API and
* extracts it as plain text.
*/
async function fetchDocContent(accessToken: string, documentId: string): Promise<string> {
const url = `https://docs.googleapis.com/v1/documents/${documentId}?fields=body.content`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) {
throw new Error(`Failed to fetch Google Doc content ${documentId}: ${response.status}`)
}
const doc = (await response.json()) as DocsDocument
return extractTextFromDocsBody(doc)
}
/**
* Creates a lightweight stub from a Drive file entry. Content is deferred
* and only fetched via getDocument for new or changed documents.
*/
function fileToStub(file: DriveFile): ExternalDocument {
return {
externalId: file.id,
title: file.name || 'Untitled',
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: file.webViewLink || `https://docs.google.com/document/d/${file.id}/edit`,
contentHash: `gdocs:${file.id}:${file.modifiedTime ?? ''}`,
metadata: {
modifiedTime: file.modifiedTime,
createdTime: file.createdTime,
owners: file.owners?.map((o) => o.displayName || o.emailAddress).filter(Boolean),
},
}
}
/**
* Builds the Drive API query string for listing Google Docs.
*/
function buildQuery(sourceConfig: Record<string, unknown>): string {
const parts: string[] = ['trashed = false', "mimeType = 'application/vnd.google-apps.document'"]
const parentsClause = buildDriveParentsClause(parseMultiValue(sourceConfig.folderId))
if (parentsClause) parts.push(parentsClause)
return parts.join(' and ')
}
export const googleDocsConnector: ConnectorConfig = {
...googleDocsConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const query = buildQuery(sourceConfig)
const pageSize = 100
const queryParams = new URLSearchParams({
q: query,
pageSize: String(pageSize),
fields: 'nextPageToken,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners)',
supportsAllDrives: 'true',
includeItemsFromAllDrives: 'true',
})
if (cursor) {
queryParams.set('pageToken', cursor)
}
const url = `https://www.googleapis.com/drive/v3/files?${queryParams.toString()}`
logger.info('Listing Google Docs', { query, cursor: cursor ?? 'initial' })
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) {
const errorText = await response.text()
logger.error('Failed to list Google Docs', {
status: response.status,
error: errorText,
})
throw new Error(`Failed to list Google Docs: ${response.status}`)
}
const data = await response.json()
const files = (data.files || []) as DriveFile[]
/**
* Drive sets `incompleteSearch` when it could not search every corpus (it
* arises with the `allDrives` scope enabled by `includeItemsFromAllDrives`).
* A partial listing drops still-existing docs, so reconciliation must be
* suppressed to avoid hard-deleting valid documents.
*/
const incompleteSearch = data.incompleteSearch === true
const maxDocs = sourceConfig.maxDocs ? Number(sourceConfig.maxDocs) : 0
const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
let documents = files.map(fileToStub)
let slicedSome = false
if (maxDocs > 0) {
const remaining = maxDocs - previouslyFetched
if (documents.length > remaining) {
slicedSome = true
documents = documents.slice(0, remaining)
}
}
const totalFetched = previouslyFetched + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = maxDocs > 0 && totalFetched >= maxDocs
const nextPageToken = data.nextPageToken as string | undefined
/**
* Mark the listing as incomplete so the sync engine skips deletion
* reconciliation when this page does not represent the full source set:
* - `slicedSome`: the page held more docs than the `maxDocs` cap allowed.
* - `hitLimit` with a next page: the cap was reached while more pages remain.
* - `incompleteSearch`: Drive could not search every corpus, so the page is
* partial and may omit still-existing docs.
* Reconciliation against any of these would hard-delete valid documents.
*/
if (syncContext && (slicedSome || (hitLimit && Boolean(nextPageToken)) || incompleteSearch)) {
syncContext.listingCapped = true
}
return {
documents,
nextCursor: hitLimit ? undefined : nextPageToken,
hasMore: hitLimit ? false : Boolean(nextPageToken),
}
},
getDocument: async (
accessToken: string,
_sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
const fields = 'id,name,mimeType,modifiedTime,createdTime,webViewLink,owners,trashed'
const url = `https://www.googleapis.com/drive/v3/files/${externalId}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) {
if (response.status === 404) return null
throw new Error(`Failed to get Google Doc metadata: ${response.status}`)
}
const file = (await response.json()) as DriveFile & { trashed?: boolean }
if (file.trashed) return null
if (file.mimeType !== 'application/vnd.google-apps.document') return null
try {
const content = await fetchDocContent(accessToken, file.id)
if (!content.trim()) return null
return { ...fileToStub(file), content, contentDeferred: false }
} catch (error) {
logger.warn(`Failed to extract content from document: ${file.name} (${file.id})`, {
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const folderIds = parseMultiValue(sourceConfig.folderId)
const maxDocs = sourceConfig.maxDocs as string | undefined
if (maxDocs && (Number.isNaN(Number(maxDocs)) || Number(maxDocs) <= 0)) {
return { valid: false, error: 'Max documents must be a positive number' }
}
try {
if (folderIds.length > 0) {
for (const folderId of folderIds) {
const url = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(folderId)}?fields=id,name,mimeType&supportsAllDrives=true`
const response = await fetchWithRetry(
url,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
if (response.status === 404) {
return {
valid: false,
error: `Folder "${folderId}" not found. Check the folder ID and permissions.`,
}
}
return {
valid: false,
error: `Failed to access folder "${folderId}": ${response.status}`,
}
}
const folder = await response.json()
if (folder.mimeType !== 'application/vnd.google-apps.folder') {
return { valid: false, error: `"${folderId}" is not a folder` }
}
}
} else {
const url =
"https://www.googleapis.com/drive/v3/files?pageSize=1&q=mimeType%3D'application%2Fvnd.google-apps.document'&fields=files(id)"
const response = await fetchWithRetry(
url,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
return { valid: false, error: `Failed to access Google Docs: ${response.status}` }
}
}
return { valid: true }
} catch (error) {
return { valid: false, error: toError(error).message || 'Failed to validate configuration' }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
const owners = joinTagArray(metadata.owners)
if (owners) result.owners = owners
const lastModified = parseTagDate(metadata.modifiedTime)
if (lastModified) result.lastModified = lastModified
return result
},
}
+1
View File
@@ -0,0 +1 @@
export { googleDocsConnector } from '@/connectors/google-docs/google-docs'
+53
View File
@@ -0,0 +1,53 @@
import { GoogleDocsIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const googleDocsConnectorMeta: ConnectorMeta = {
id: 'google_docs',
name: 'Google Docs',
description: 'Sync Google Docs documents',
version: '1.0.0',
icon: GoogleDocsIcon,
auth: {
mode: 'oauth',
provider: 'google-docs',
requiredScopes: ['https://www.googleapis.com/auth/drive'],
},
configFields: [
{
id: 'folderSelector',
title: 'Folders',
type: 'selector',
selectorKey: 'google.drive',
mimeType: 'application/vnd.google-apps.folder',
canonicalParamId: 'folderId',
mode: 'basic',
multi: true,
placeholder: 'Select one or more folders (optional)',
required: false,
},
{
id: 'folderId',
title: 'Folder IDs',
type: 'short-input',
canonicalParamId: 'folderId',
mode: 'advanced',
multi: true,
placeholder: 'e.g. 1aBcDeFg…, 2cDeFgHi… (comma-separated for multiple)',
required: false,
},
{
id: 'maxDocs',
title: 'Max Documents',
type: 'short-input',
required: false,
placeholder: 'e.g. 500 (default: unlimited)',
},
],
tagDefinitions: [
{ id: 'owners', displayName: 'Owner', fieldType: 'text' },
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
],
}
@@ -0,0 +1,421 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { googleDriveConnectorMeta } from '@/connectors/google-drive/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import {
buildDriveParentsClause,
CONNECTOR_MAX_FILE_BYTES,
ConnectorFileTooLargeError,
htmlToPlainText,
joinTagArray,
markSkipped,
parseMultiValue,
parseTagDate,
readBodyWithLimit,
sizeLimitSkipReason,
} from '@/connectors/utils'
const logger = createLogger('GoogleDriveConnector')
const GOOGLE_WORKSPACE_MIME_TYPES: Record<string, string> = {
'application/vnd.google-apps.document': 'text/plain',
'application/vnd.google-apps.spreadsheet': 'text/csv',
'application/vnd.google-apps.presentation': 'text/plain',
}
const SUPPORTED_TEXT_MIME_TYPES = [
'text/plain',
'text/csv',
'text/html',
'text/markdown',
'application/json',
'application/xml',
]
// Google Drive's `files.export` API rejects exports over 10 MB (exportSizeLimitExceeded),
// so this is a hard external limit for Google Workspace docs — not the connector cap.
const MAX_EXPORT_SIZE = 10 * 1024 * 1024
function isGoogleWorkspaceFile(mimeType: string): boolean {
return mimeType in GOOGLE_WORKSPACE_MIME_TYPES
}
function isSupportedTextFile(mimeType: string): boolean {
return SUPPORTED_TEXT_MIME_TYPES.some((t) => mimeType.startsWith(t))
}
async function exportGoogleWorkspaceFile(
accessToken: string,
fileId: string,
sourceMimeType: string
): Promise<string> {
const exportMimeType = GOOGLE_WORKSPACE_MIME_TYPES[sourceMimeType]
if (!exportMimeType) {
throw new Error(`Unsupported Google Workspace MIME type: ${sourceMimeType}`)
}
const url = `https://www.googleapis.com/drive/v3/files/${fileId}/export?mimeType=${encodeURIComponent(exportMimeType)}`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: { Authorization: `Bearer ${accessToken}` },
})
if (!response.ok) {
// Google rejects exports over its 10 MB limit with a 403 exportSizeLimitExceeded
// before streaming any bytes — surface that as an oversize skip, not a hard error.
if (response.status === 403) {
const body = await response.text().catch(() => '')
if (body.includes('exportSizeLimitExceeded')) {
throw new ConnectorFileTooLargeError(MAX_EXPORT_SIZE)
}
}
throw new Error(`Failed to export file ${fileId}: ${response.status}`)
}
const buffer = await readBodyWithLimit(response, MAX_EXPORT_SIZE)
if (!buffer) {
throw new ConnectorFileTooLargeError(MAX_EXPORT_SIZE)
}
return buffer.toString('utf8')
}
async function downloadTextFile(accessToken: string, fileId: string): Promise<string> {
const url = `https://www.googleapis.com/drive/v3/files/${fileId}?alt=media`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: { Authorization: `Bearer ${accessToken}` },
})
if (!response.ok) {
throw new Error(`Failed to download file ${fileId}: ${response.status}`)
}
// Stream with a hard byte cap so a file with missing/under-reported listing
// size metadata is never fully buffered into memory. Oversized files raise
// DriveFileTooLargeError so getDocument can surface them as skipped (failed) rows.
const buffer = await readBodyWithLimit(response, CONNECTOR_MAX_FILE_BYTES)
if (!buffer) {
throw new ConnectorFileTooLargeError(CONNECTOR_MAX_FILE_BYTES)
}
return buffer.toString('utf8')
}
async function fetchFileContent(
accessToken: string,
fileId: string,
mimeType: string
): Promise<string> {
if (isGoogleWorkspaceFile(mimeType)) {
return exportGoogleWorkspaceFile(accessToken, fileId, mimeType)
}
if (mimeType === 'text/html') {
const html = await downloadTextFile(accessToken, fileId)
return htmlToPlainText(html)
}
if (isSupportedTextFile(mimeType)) {
return downloadTextFile(accessToken, fileId)
}
throw new Error(`Unsupported MIME type for content extraction: ${mimeType}`)
}
interface DriveFile {
id: string
name: string
mimeType: string
modifiedTime?: string
createdTime?: string
webViewLink?: string
parents?: string[]
owners?: { displayName?: string; emailAddress?: string }[]
size?: string
starred?: boolean
trashed?: boolean
}
function buildQuery(sourceConfig: Record<string, unknown>): string {
const parts: string[] = ['trashed = false']
const parentsClause = buildDriveParentsClause(parseMultiValue(sourceConfig.folderId))
if (parentsClause) parts.push(parentsClause)
const fileType = (sourceConfig.fileType as string) || 'all'
switch (fileType) {
case 'documents':
parts.push("mimeType = 'application/vnd.google-apps.document'")
break
case 'spreadsheets':
parts.push("mimeType = 'application/vnd.google-apps.spreadsheet'")
break
case 'presentations':
parts.push("mimeType = 'application/vnd.google-apps.presentation'")
break
case 'text':
parts.push(`(${SUPPORTED_TEXT_MIME_TYPES.map((t) => `mimeType = '${t}'`).join(' or ')})`)
break
default: {
// Include Google Workspace files + plain text files, exclude folders
const allMimeTypes = [
...Object.keys(GOOGLE_WORKSPACE_MIME_TYPES),
...SUPPORTED_TEXT_MIME_TYPES,
]
parts.push(`(${allMimeTypes.map((t) => `mimeType = '${t}'`).join(' or ')})`)
break
}
}
return parts.join(' and ')
}
function fileToStub(file: DriveFile): ExternalDocument {
return {
externalId: file.id,
title: file.name || 'Untitled',
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: file.webViewLink || `https://drive.google.com/file/d/${file.id}/view`,
contentHash: `gdrive:${file.id}:${file.modifiedTime ?? ''}`,
metadata: {
originalMimeType: file.mimeType,
modifiedTime: file.modifiedTime,
createdTime: file.createdTime,
owners: file.owners?.map((o) => o.displayName || o.emailAddress).filter(Boolean),
starred: file.starred,
fileSize: file.size ? Number(file.size) : undefined,
},
}
}
export const googleDriveConnector: ConnectorConfig = {
...googleDriveConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const query = buildQuery(sourceConfig)
const pageSize = 100
const maxFiles = sourceConfig.maxFiles ? Number(sourceConfig.maxFiles) : 0
const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
if (maxFiles > 0 && previouslyFetched >= maxFiles) {
return { documents: [], hasMore: false }
}
const remaining = maxFiles > 0 ? maxFiles - previouslyFetched : 0
const effectivePageSize = maxFiles > 0 ? Math.min(pageSize, remaining) : pageSize
const queryParams = new URLSearchParams({
q: query,
pageSize: String(effectivePageSize),
orderBy: 'modifiedTime desc',
fields:
'nextPageToken,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,parents,owners,size,starred)',
supportsAllDrives: 'true',
includeItemsFromAllDrives: 'true',
})
if (cursor) {
queryParams.set('pageToken', cursor)
}
const url = `https://www.googleapis.com/drive/v3/files?${queryParams.toString()}`
logger.info('Listing Google Drive files', { query, cursor: cursor ?? 'initial' })
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) {
const errorText = await response.text()
logger.error('Failed to list Google Drive files', {
status: response.status,
error: errorText,
})
throw new Error(`Failed to list Google Drive files: ${response.status}`)
}
const data = await response.json()
const files = (data.files || []) as DriveFile[]
/**
* Drive sets `incompleteSearch` when it could not search every corpus (it
* arises with the `allDrives` scope enabled by `includeItemsFromAllDrives`).
* A partial listing drops still-existing files, so reconciliation must be
* suppressed to avoid hard-deleting valid documents.
*/
const incompleteSearch = data.incompleteSearch === true
const documents = files
.filter((f) => isGoogleWorkspaceFile(f.mimeType) || isSupportedTextFile(f.mimeType))
.map(fileToStub)
const totalFetched = previouslyFetched + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = maxFiles > 0 && totalFetched >= maxFiles
if (syncContext && (hitLimit || incompleteSearch)) syncContext.listingCapped = true
const nextPageToken = data.nextPageToken as string | undefined
return {
documents,
nextCursor: hitLimit ? undefined : nextPageToken,
hasMore: hitLimit ? false : Boolean(nextPageToken),
}
},
getDocument: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
const fields =
'id,name,mimeType,modifiedTime,createdTime,webViewLink,parents,owners,size,starred,trashed'
const url = `https://www.googleapis.com/drive/v3/files/${externalId}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) {
if (response.status === 404) return null
throw new Error(`Failed to get Google Drive file: ${response.status}`)
}
const file = (await response.json()) as DriveFile
if (file.trashed) return null
try {
const content = await fetchFileContent(accessToken, file.id, file.mimeType)
if (!content.trim()) return null
const stub = fileToStub(file)
return { ...stub, content, contentDeferred: false }
} catch (error) {
if (error instanceof ConnectorFileTooLargeError) {
logger.info('Skipping oversized Google Drive file', { fileId: file.id, name: file.name })
return markSkipped(fileToStub(file), sizeLimitSkipReason(error.limitBytes))
}
logger.warn(`Failed to fetch content for file: ${file.name} (${file.id})`, {
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const folderIds = parseMultiValue(sourceConfig.folderId)
const maxFiles = sourceConfig.maxFiles as string | undefined
if (maxFiles && (Number.isNaN(Number(maxFiles)) || Number(maxFiles) <= 0)) {
return { valid: false, error: 'Max files must be a positive number' }
}
// Verify access to Drive API
try {
if (folderIds.length > 0) {
// Verify each folder exists, is accessible, and is actually a folder
for (const folderId of folderIds) {
const url = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(folderId)}?fields=id,name,mimeType&supportsAllDrives=true`
const response = await fetchWithRetry(
url,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
if (response.status === 404) {
return {
valid: false,
error: `Folder "${folderId}" not found. Check the folder ID and permissions.`,
}
}
return {
valid: false,
error: `Failed to access folder "${folderId}": ${response.status}`,
}
}
const folder = await response.json()
if (folder.mimeType !== 'application/vnd.google-apps.folder') {
return { valid: false, error: `"${folderId}" is not a folder` }
}
}
} else {
// Verify basic Drive access by listing one file
const url = 'https://www.googleapis.com/drive/v3/files?pageSize=1&fields=files(id)'
const response = await fetchWithRetry(
url,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
return { valid: false, error: `Failed to access Google Drive: ${response.status}` }
}
}
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
const owners = joinTagArray(metadata.owners)
if (owners) result.owners = owners
if (typeof metadata.originalMimeType === 'string') {
const mimeType = metadata.originalMimeType
if (mimeType.includes('document')) result.fileType = 'Google Doc'
else if (mimeType.includes('spreadsheet')) result.fileType = 'Google Sheet'
else if (mimeType.includes('presentation')) result.fileType = 'Google Slides'
else if (mimeType.startsWith('text/')) result.fileType = 'Text File'
else result.fileType = mimeType
}
const lastModified = parseTagDate(metadata.modifiedTime)
if (lastModified) result.lastModified = lastModified
if (typeof metadata.starred === 'boolean') {
result.starred = metadata.starred
}
return result
},
}
@@ -0,0 +1 @@
export { googleDriveConnector } from '@/connectors/google-drive/google-drive'
+68
View File
@@ -0,0 +1,68 @@
import { GoogleDriveIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const googleDriveConnectorMeta: ConnectorMeta = {
id: 'google_drive',
name: 'Google Drive',
description: 'Sync documents from Google Drive',
version: '1.0.0',
icon: GoogleDriveIcon,
auth: {
mode: 'oauth',
provider: 'google-drive',
requiredScopes: ['https://www.googleapis.com/auth/drive'],
},
configFields: [
{
id: 'folderSelector',
title: 'Folders',
type: 'selector',
selectorKey: 'google.drive',
mimeType: 'application/vnd.google-apps.folder',
canonicalParamId: 'folderId',
mode: 'basic',
multi: true,
placeholder: 'Select one or more folders (optional)',
required: false,
},
{
id: 'folderId',
title: 'Folder IDs',
type: 'short-input',
canonicalParamId: 'folderId',
mode: 'advanced',
multi: true,
placeholder: 'e.g. 1aBcDeFg…, 2cDeFgHi… (comma-separated for multiple)',
required: false,
},
{
id: 'fileType',
title: 'File Type',
type: 'dropdown',
required: false,
options: [
{ label: 'All supported files', id: 'all' },
{ label: 'Google Docs only', id: 'documents' },
{ label: 'Google Sheets only', id: 'spreadsheets' },
{ label: 'Google Slides only', id: 'presentations' },
{ label: 'Plain text files only', id: 'text' },
],
},
{
id: 'maxFiles',
title: 'Max Files',
type: 'short-input',
required: false,
placeholder: 'e.g. 500 (default: unlimited)',
},
],
tagDefinitions: [
{ id: 'owners', displayName: 'Owner', fieldType: 'text' },
{ id: 'fileType', displayName: 'File Type', fieldType: 'text' },
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
{ id: 'starred', displayName: 'Starred', fieldType: 'boolean' },
],
}
@@ -0,0 +1,765 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { googleFormsConnectorMeta, MAX_RESPONSES_PER_FORM } from '@/connectors/google-forms/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import {
buildDriveParentsClause,
joinTagArray,
parseMultiValue,
parseTagDate,
} from '@/connectors/utils'
const logger = createLogger('GoogleFormsConnector')
const DRIVE_API_BASE = 'https://www.googleapis.com/drive/v3'
const FORMS_API_BASE = 'https://forms.googleapis.com/v1'
const FORM_MIME_TYPE = 'application/vnd.google-apps.form'
const FOLDER_MIME_TYPE = 'application/vnd.google-apps.folder'
/**
* Drive API page size when listing forms. The Drive API caps pageSize at 100.
*/
const DRIVE_PAGE_SIZE = 100
/**
* Maximum responses returned per Forms API page (API caps and defaults to 5000).
*/
const RESPONSES_PAGE_SIZE = 5000
/**
* Number of forms whose change indicators are fetched concurrently during
* listing. Keeps the Forms API call volume bounded while still parallelizing.
*/
const LIST_CONCURRENCY = 4
/**
* Content scope for a form document. `both` indexes the form's questions and its
* submitted responses; `structure` indexes only the questions (no response reads,
* so the responses scope is never exercised for that connector instance).
*/
type ContentScope = 'both' | 'structure'
/**
* Resolves the content scope from sourceConfig, defaulting to `both`.
*/
function resolveContentScope(value: unknown): ContentScope {
return value === 'structure' ? 'structure' : 'both'
}
/**
* Represents a Google Drive file entry for a form, returned by the Drive API.
*/
interface DriveFormFile {
id: string
name: string
mimeType: string
modifiedTime?: string
createdTime?: string
webViewLink?: string
owners?: { displayName?: string; emailAddress?: string }[]
trashed?: boolean
}
/**
* A single answer entry inside a response answer container.
*/
interface FormTextAnswer {
value?: string
}
/**
* A single question's answers within a form response. The Forms API keys the
* `answers` map by questionId and stores text values under
* `textAnswers.answers[].value`.
*/
interface FormAnswer {
questionId?: string
textAnswers?: { answers?: FormTextAnswer[] }
}
/**
* A single submitted response to a form.
*/
interface FormResponse {
responseId?: string
createTime?: string
lastSubmittedTime?: string
respondentEmail?: string
answers?: Record<string, FormAnswer>
}
/**
* Paginated response list from the Forms API.
*/
interface FormResponseList {
responses?: FormResponse[]
nextPageToken?: string
}
/**
* A question item within a form's structure.
*/
interface FormQuestionItem {
question?: {
questionId?: string
required?: boolean
}
}
/**
* A single structural item within a form (question, section, image, etc.).
*/
interface FormItem {
itemId?: string
title?: string
description?: string
questionItem?: FormQuestionItem
}
/**
* The form structure returned by the Forms API `forms.get` endpoint.
*/
interface FormStructure {
formId?: string
info?: {
title?: string
description?: string
documentTitle?: string
}
items?: FormItem[]
revisionId?: string
responderUri?: string
}
/**
* Lightweight metadata captured during listing, sufficient to build a stub
* and detect changes without downloading the full form content.
*/
interface FormStubInput {
file: DriveFormFile
formTitle?: string
revisionId?: string
latestResponseTime?: string
contentScope: ContentScope
responseCap: number
}
/**
* Resolves the effective per-form response cap applied when rendering content:
* the user-configured `maxResponsesPerForm` clamped to the hard
* `MAX_RESPONSES_PER_FORM` ceiling. Part of the content hash so changing the
* cap re-syncs every form (the rendered content depends on it).
*/
function resolveResponseCap(sourceConfig: Record<string, unknown>): number {
const configured = parsePositiveInt(sourceConfig.maxResponsesPerForm)
return configured > 0 ? Math.min(configured, MAX_RESPONSES_PER_FORM) : MAX_RESPONSES_PER_FORM
}
/**
* Parses an optional positive-integer config value, returning 0 when unset/invalid.
*/
function parsePositiveInt(value: unknown): number {
if (value == null || value === '') return 0
const num = Number(value)
return Number.isNaN(num) || num <= 0 ? 0 : Math.floor(num)
}
/**
* Maps a small array over an async worker with a bounded concurrency, preserving
* input order in the returned results.
*/
async function mapWithConcurrency<T, R>(
items: T[],
limit: number,
worker: (item: T, index: number) => Promise<R>
): Promise<R[]> {
const results = new Array<R>(items.length)
let next = 0
async function run(): Promise<void> {
while (next < items.length) {
const current = next++
results[current] = await worker(items[current], current)
}
}
const runners = Array.from({ length: Math.min(limit, items.length) }, run)
await Promise.all(runners)
return results
}
/**
* Fetches the form structure via the Forms API. Returns null on 404 (form
* deleted or inaccessible).
*/
async function fetchFormStructure(
accessToken: string,
formId: string
): Promise<FormStructure | null> {
const url = `${FORMS_API_BASE}/forms/${encodeURIComponent(formId)}`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) {
if (response.status === 404) return null
throw new Error(`Failed to fetch form structure ${formId}: ${response.status}`)
}
return (await response.json()) as FormStructure
}
/**
* Result of fetching a form's responses: the collected responses (capped at
* `MAX_RESPONSES_PER_FORM` for rendering) plus the greatest submission timestamp
* across ALL response pages.
*
* `latestSubmittedTime` is tracked separately from the capped `responses` so the
* content hash computed in getDocument stays identical to the one computed during
* listing, which scans the same full set via `fetchLatestResponseTime`. If it
* were derived from the capped slice alone, a form with more than
* `MAX_RESPONSES_PER_FORM` responses could hash differently between the two paths
* and re-sync on every run.
*/
interface FetchedResponses {
responses: FormResponse[]
latestSubmittedTime?: string
}
/**
* Fetches form responses, retaining up to `MAX_RESPONSES_PER_FORM` for rendering.
* Every page is scanned for the latest submission timestamp even after the
* render cap is reached — the Forms API does not guarantee response order, so
* the newest submission may sit on any page. `fetchLatestResponseTime` scans
* the same full set during listing, keeping the content hash identical across
* the listing and getDocument paths regardless of form size.
*/
async function fetchFormResponses(accessToken: string, formId: string): Promise<FetchedResponses> {
const collected: FormResponse[] = []
let latest = ''
let pageToken: string | undefined
do {
const url = new URL(`${FORMS_API_BASE}/forms/${encodeURIComponent(formId)}/responses`)
url.searchParams.set('pageSize', String(RESPONSES_PAGE_SIZE))
if (pageToken) url.searchParams.set('pageToken', pageToken)
const response = await fetchWithRetry(url.toString(), {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) {
throw new Error(`Failed to list responses for form ${formId}: ${response.status}`)
}
const data = (await response.json()) as FormResponseList
const responses = data.responses ?? []
const pageLatest = latestResponseTime(responses)
if (pageLatest && pageLatest > latest) latest = pageLatest
for (const r of responses) {
if (collected.length >= MAX_RESPONSES_PER_FORM) break
collected.push(r)
}
pageToken = data.nextPageToken
} while (pageToken)
return { responses: collected, latestSubmittedTime: latest || undefined }
}
/**
* Reads the latest response submission time for change detection without
* retaining responses. Scans every page — the Forms API does not guarantee
* response order, so the newest submission may sit on any page. Returns the
* greatest `lastSubmittedTime` (falling back to `createTime`), or undefined
* when there are none. Throws on a failed read so the caller skips the form
* for this run instead of computing a hash from incomplete data — a swallowed
* error would poison the stub's content hash and re-process the form on every
* sync, while throwing routes into the per-form catch that sets
* `skippedOnError` → `listingCapped`.
*/
async function fetchLatestResponseTime(
accessToken: string,
formId: string
): Promise<string | undefined> {
let latest = ''
let pageToken: string | undefined
do {
const url = new URL(`${FORMS_API_BASE}/forms/${encodeURIComponent(formId)}/responses`)
url.searchParams.set('pageSize', String(RESPONSES_PAGE_SIZE))
if (pageToken) url.searchParams.set('pageToken', pageToken)
const response = await fetchWithRetry(url.toString(), {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) {
throw new Error(
`Failed to read responses for change detection on form ${formId}: ${response.status}`
)
}
const data = (await response.json()) as FormResponseList
const pageLatest = latestResponseTime(data.responses ?? [])
if (pageLatest && pageLatest > latest) latest = pageLatest
pageToken = data.nextPageToken
} while (pageToken)
return latest || undefined
}
/**
* Returns the greatest submission timestamp across the given responses, or
* undefined when the list is empty.
*/
function latestResponseTime(responses: FormResponse[]): string | undefined {
let latest = ''
for (const r of responses) {
const t = r.lastSubmittedTime || r.createTime || ''
if (t > latest) latest = t
}
return latest || undefined
}
/**
* Builds the content hash for a form. The hash must change when either the form
* structure (revisionId) or, when responses are indexed, the set of responses
* (latest submission time) changes. Drive `modifiedTime` alone is insufficient
* because new response submissions do not update the form's Drive modifiedTime.
* The content scope is part of the hash so that toggling response indexing
* forces a re-sync of every document.
*/
function formContentHash(input: FormStubInput): string {
const responsePart =
input.contentScope === 'both'
? `${input.latestResponseTime ?? ''}:${input.responseCap}`
: 'none'
return `gforms:${input.file.id}:${input.contentScope}:${input.revisionId ?? ''}:${responsePart}`
}
/**
* Creates a lightweight stub from a form's Drive file and change indicators.
* Content is deferred and only fetched via getDocument for new/changed forms.
*/
function formToStub(input: FormStubInput): ExternalDocument {
const { file } = input
const title = input.formTitle?.trim() || file.name || 'Untitled Form'
return {
externalId: file.id,
title,
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: file.webViewLink || `https://docs.google.com/forms/d/${file.id}/edit`,
contentHash: formContentHash(input),
metadata: {
formTitle: title,
modifiedTime: file.modifiedTime,
createdTime: file.createdTime,
latestResponseTime: input.contentScope === 'both' ? input.latestResponseTime : undefined,
owners: file.owners?.map((o) => o.displayName || o.emailAddress).filter(Boolean),
},
}
}
/**
* Extracts the answer values for a single question from a response.
*/
function extractAnswerText(answer: FormAnswer | undefined): string {
const values = answer?.textAnswers?.answers
?.map((a) => a.value)
.filter((v): v is string => typeof v === 'string' && v.trim().length > 0)
return values && values.length > 0 ? values.join(', ') : ''
}
/**
* Builds a question-id → title map from the form structure, so responses can be
* rendered with human-readable question labels instead of opaque IDs.
*/
function buildQuestionTitleMap(form: FormStructure): Map<string, string> {
const map = new Map<string, string>()
for (const item of form.items ?? []) {
const questionId = item.questionItem?.question?.questionId
if (questionId && item.title) {
map.set(questionId, item.title)
}
}
return map
}
/**
* Renders the full form document: its structure (title, description, questions)
* followed by each response's question/answer pairs when responses are included.
*/
function renderFormDocument(form: FormStructure, responses: FormResponse[]): string {
const parts: string[] = []
const title = form.info?.title || form.info?.documentTitle
if (title) parts.push(`# ${title}`)
if (form.info?.description?.trim()) parts.push(form.info.description.trim())
const questionTitles = buildQuestionTitleMap(form)
const questionLines: string[] = []
for (const item of form.items ?? []) {
if (!item.title?.trim()) continue
const required = item.questionItem?.question?.required ? ' (required)' : ''
questionLines.push(`- ${item.title.trim()}${required}`)
if (item.description?.trim()) questionLines.push(` ${item.description.trim()}`)
}
if (questionLines.length > 0) {
parts.push('## Questions')
parts.push(questionLines.join('\n'))
}
if (responses.length > 0) {
parts.push(`## Responses (${responses.length})`)
responses.forEach((response, index) => {
const responseLines: string[] = []
const submitted = response.lastSubmittedTime || response.createTime
const header = submitted
? `### Response ${index + 1}${submitted}`
: `### Response ${index + 1}`
responseLines.push(header)
if (response.respondentEmail) {
responseLines.push(`Respondent: ${response.respondentEmail}`)
}
for (const [questionId, answer] of Object.entries(response.answers ?? {})) {
const label = questionTitles.get(questionId) || questionId
const value = extractAnswerText(answer)
if (value) responseLines.push(`${label}: ${value}`)
}
parts.push(responseLines.join('\n'))
})
}
return parts.join('\n\n').trim()
}
/**
* Builds the Drive `q` query that selects form files, optionally scoped to one
* or more folders. Single quotes and backslashes in folder IDs are escaped to
* prevent query injection.
*/
function buildDriveQuery(folderIds: string[]): string {
const parts = ['trashed = false', `mimeType = '${FORM_MIME_TYPE}'`]
const parentsClause = buildDriveParentsClause(folderIds)
if (parentsClause) parts.push(parentsClause)
return parts.join(' and ')
}
export const googleFormsConnector: ConnectorConfig = {
...googleFormsConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const maxForms = parsePositiveInt(sourceConfig.maxForms)
const contentScope = resolveContentScope(sourceConfig.contentScope)
const responseCap = resolveResponseCap(sourceConfig)
const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
if (maxForms > 0 && previouslyFetched >= maxForms) {
return { documents: [], hasMore: false }
}
const folderIds = parseMultiValue(sourceConfig.folderId)
const queryParams = new URLSearchParams({
q: buildDriveQuery(folderIds),
pageSize: String(DRIVE_PAGE_SIZE),
orderBy: 'modifiedTime desc',
fields: 'nextPageToken,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners)',
supportsAllDrives: 'true',
includeItemsFromAllDrives: 'true',
})
if (cursor) queryParams.set('pageToken', cursor)
const url = `${DRIVE_API_BASE}/files?${queryParams.toString()}`
logger.info('Listing Google Forms', {
folderId: folderIds.length > 0 ? folderIds.join(',') : 'all',
contentScope,
cursor: cursor ?? 'initial',
})
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) {
const errorText = await response.text()
logger.error('Failed to list Google Forms', { status: response.status, error: errorText })
throw new Error(`Failed to list Google Forms: ${response.status}`)
}
const data = await response.json()
let files = (data.files || []) as DriveFormFile[]
/**
* Drive sets `incompleteSearch` when it could not search every corpus (it
* arises with the `allDrives` scope enabled by `includeItemsFromAllDrives`).
* A partial listing drops still-existing forms, so reconciliation must be
* suppressed to avoid hard-deleting valid documents.
*/
const incompleteSearch = data.incompleteSearch === true
let slicedSome = false
if (maxForms > 0) {
const remaining = maxForms - previouslyFetched
if (files.length > remaining) {
slicedSome = true
files = files.slice(0, remaining)
}
}
/**
* Build stubs with metadata-based change indicators. Each form needs its
* revisionId (structure changes) and, when responses are indexed, the latest
* response time (new submissions) so the sync engine can detect changes
* without downloading full content. Forms are processed with bounded
* concurrency; a transient per-form failure is skipped rather than aborting
* the whole page, but it is recorded so the listing is marked incomplete.
*/
let skippedOnError = false
const stubs = await mapWithConcurrency(files, LIST_CONCURRENCY, async (file) => {
try {
const form = await fetchFormStructure(accessToken, file.id)
if (!form) return null
const latest =
contentScope === 'both' ? await fetchLatestResponseTime(accessToken, file.id) : undefined
return formToStub({
file,
formTitle: form.info?.title || form.info?.documentTitle,
revisionId: form.revisionId,
latestResponseTime: latest,
contentScope,
responseCap,
})
} catch (error) {
skippedOnError = true
logger.warn(`Skipping form during listing: ${file.name} (${file.id})`, {
error: toError(error).message,
})
return null
}
})
const documents = stubs.filter((s): s is ExternalDocument => s !== null)
const totalFetched = previouslyFetched + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = maxForms > 0 && totalFetched >= maxForms
const nextPageToken = data.nextPageToken as string | undefined
/**
* Mark the listing as incomplete so the sync engine skips deletion
* reconciliation. Three cases drop still-existing forms from the listing:
* - `slicedSome`: this page held more forms than the `maxForms` cap allowed,
* so forms beyond the slice were truncated. This is independent of
* `hitLimit`, which counts successfully fetched stubs and can fall below
* the cap when 404s or errors null out items even though real forms were
* sliced off.
* - `hitLimit` with a next page: the cap was reached while more pages of
* forms remain in the source.
* - `skippedOnError`: a transient error dropped a still-present form.
* - `incompleteSearch`: Drive could not search every corpus, so the page
* itself is partial and may omit still-existing forms.
* Deleting any of those would wipe valid documents from the knowledge base.
* When the cap merely coincides with source exhaustion (no slice, no next
* page), reconciliation stays enabled so deleted forms are cleaned up.
*/
if (
syncContext &&
(slicedSome || (hitLimit && Boolean(nextPageToken)) || skippedOnError || incompleteSearch)
) {
syncContext.listingCapped = true
}
return {
documents,
nextCursor: hitLimit ? undefined : nextPageToken,
hasMore: hitLimit ? false : Boolean(nextPageToken),
}
},
getDocument: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
const contentScope = resolveContentScope(sourceConfig.contentScope)
const fields = 'id,name,mimeType,modifiedTime,createdTime,webViewLink,owners,trashed'
const metadataUrl = `${DRIVE_API_BASE}/files/${encodeURIComponent(externalId)}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true`
const metadataResponse = await fetchWithRetry(metadataUrl, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!metadataResponse.ok) {
if (metadataResponse.status === 404) return null
throw new Error(`Failed to get form metadata: ${metadataResponse.status}`)
}
const file = (await metadataResponse.json()) as DriveFormFile
if (file.trashed) return null
if (file.mimeType !== FORM_MIME_TYPE) return null
try {
const form = await fetchFormStructure(accessToken, file.id)
if (!form) return null
const responseCap = resolveResponseCap(sourceConfig)
const fetched =
contentScope === 'both'
? await fetchFormResponses(accessToken, file.id)
: { responses: [], latestSubmittedTime: undefined }
const responses = fetched.responses
const cappedResponses =
responses.length > responseCap ? responses.slice(0, responseCap) : responses
const content = renderFormDocument(form, cappedResponses)
if (!content.trim()) return null
const stub = formToStub({
file,
formTitle: form.info?.title || form.info?.documentTitle,
revisionId: form.revisionId,
latestResponseTime: fetched.latestSubmittedTime,
contentScope,
responseCap,
})
return { ...stub, content, contentDeferred: false }
} catch (error) {
logger.warn(`Failed to fetch content for form: ${file.name} (${file.id})`, {
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const folderIds = parseMultiValue(sourceConfig.folderId)
const maxForms = sourceConfig.maxForms as string | undefined
const maxResponsesPerForm = sourceConfig.maxResponsesPerForm as string | undefined
if (maxForms && (Number.isNaN(Number(maxForms)) || Number(maxForms) <= 0)) {
return { valid: false, error: 'Max forms must be a positive number' }
}
if (
maxResponsesPerForm &&
(Number.isNaN(Number(maxResponsesPerForm)) || Number(maxResponsesPerForm) <= 0)
) {
return { valid: false, error: 'Max responses per form must be a positive number' }
}
try {
if (folderIds.length > 0) {
for (const folderId of folderIds) {
const url = `${DRIVE_API_BASE}/files/${encodeURIComponent(folderId)}?fields=id,name,mimeType&supportsAllDrives=true`
const response = await fetchWithRetry(
url,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
if (response.status === 404) {
return {
valid: false,
error: `Folder "${folderId}" not found. Check the folder ID and permissions.`,
}
}
return {
valid: false,
error: `Failed to access folder "${folderId}": ${response.status}`,
}
}
const folder = await response.json()
if (folder.mimeType !== FOLDER_MIME_TYPE) {
return { valid: false, error: `"${folderId}" is not a folder` }
}
}
} else {
const url = `${DRIVE_API_BASE}/files?pageSize=1&q=${encodeURIComponent(`mimeType = '${FORM_MIME_TYPE}'`)}&fields=files(id)&supportsAllDrives=true&includeItemsFromAllDrives=true`
const response = await fetchWithRetry(
url,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
return { valid: false, error: `Failed to access Google Forms: ${response.status}` }
}
}
return { valid: true }
} catch (error) {
return { valid: false, error: getErrorMessage(error, 'Failed to validate configuration') }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.formTitle === 'string' && metadata.formTitle.trim()) {
result.formTitle = metadata.formTitle.trim()
}
const owners = joinTagArray(metadata.owners)
if (owners) result.owners = owners
const lastModified = parseTagDate(metadata.modifiedTime)
if (lastModified) result.lastModified = lastModified
const lastResponse = parseTagDate(metadata.latestResponseTime)
if (lastResponse) result.lastResponse = lastResponse
return result
},
}
@@ -0,0 +1 @@
export { googleFormsConnector } from '@/connectors/google-forms/google-forms'
+87
View File
@@ -0,0 +1,87 @@
import { GoogleFormsIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
/**
* Hard cap on the number of responses appended to a single form document.
* Keeps individual documents within a reasonable size for embedding/indexing.
*/
export const MAX_RESPONSES_PER_FORM = 500
export const googleFormsConnectorMeta: ConnectorMeta = {
id: 'google_forms',
name: 'Google Forms',
description: 'Sync Google Forms questions and responses into your knowledge base',
version: '1.0.0',
icon: GoogleFormsIcon,
auth: {
mode: 'oauth',
provider: 'google-forms',
requiredScopes: [
'https://www.googleapis.com/auth/drive',
'https://www.googleapis.com/auth/forms.body',
'https://www.googleapis.com/auth/forms.responses.readonly',
],
},
configFields: [
{
id: 'folderSelector',
title: 'Folders',
type: 'selector',
selectorKey: 'google.drive',
mimeType: 'application/vnd.google-apps.folder',
canonicalParamId: 'folderId',
mode: 'basic',
multi: true,
placeholder: 'Select one or more folders (optional)',
required: false,
description: 'Only sync forms inside these Drive folders. Leave blank to sync all forms.',
},
{
id: 'folderId',
title: 'Folder IDs',
type: 'short-input',
canonicalParamId: 'folderId',
mode: 'advanced',
multi: true,
placeholder: 'e.g. 1aBcDeFg…, 2cDeFgHi… (comma-separated for multiple)',
required: false,
description: 'Only sync forms inside these Drive folders. Leave blank to sync all forms.',
},
{
id: 'contentScope',
title: 'Content',
type: 'dropdown',
required: false,
options: [
{ label: 'Questions & responses', id: 'both' },
{ label: 'Questions only', id: 'structure' },
],
description: 'Whether to index submitted responses alongside each forms questions.',
},
{
id: 'maxForms',
title: 'Max Forms',
type: 'short-input',
required: false,
placeholder: 'e.g. 100 (default: unlimited)',
},
{
id: 'maxResponsesPerForm',
title: 'Max Responses Per Form',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: `e.g. 100 (default: ${MAX_RESPONSES_PER_FORM})`,
description: 'Cap on responses indexed per form. Applies only when indexing responses.',
},
],
tagDefinitions: [
{ id: 'formTitle', displayName: 'Form Title', fieldType: 'text' },
{ id: 'owners', displayName: 'Owner', fieldType: 'text' },
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
{ id: 'lastResponse', displayName: 'Last Response', fieldType: 'date' },
],
}
@@ -0,0 +1,537 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { googleMeetConnectorMeta } from '@/connectors/google-meet/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { joinTagArray, parseTagDate } from '@/connectors/utils'
const logger = createLogger('GoogleMeetConnector')
const MEET_API_BASE = 'https://meet.googleapis.com/v2'
const MS_PER_DAY = 24 * 60 * 60 * 1000
/** Conference records list page size (Meet API max is 100). */
const RECORDS_PAGE_SIZE = 100
/** Transcripts list page size (Meet API max is 100). */
const TRANSCRIPTS_PAGE_SIZE = 100
/** Transcript entries page size (Meet API max is 100). */
const ENTRIES_PAGE_SIZE = 100
/** Max concurrent participant-name lookups during a single getDocument call. */
const PARTICIPANT_FETCH_CONCURRENCY = 5
/**
* A conference record as returned by the Meet REST API v2. A conference record
* represents a single meeting session and is immutable once it has ended. Only the
* fields the connector reads are modeled.
*/
interface ConferenceRecord {
name: string
startTime?: string
endTime?: string | null
expireTime?: string
space?: string
}
interface ConferenceRecordsListResponse {
conferenceRecords?: ConferenceRecord[]
nextPageToken?: string
}
/**
* The Google Doc a transcript is exported to once its `state` reaches
* `FILE_GENERATED`. Used to link the synced document back to the source transcript.
*/
interface DocsDestination {
document?: string
exportUri?: string
}
/**
* A transcript of a conference record. `state` progresses STARTED → ENDED →
* FILE_GENERATED; entries are only complete once the session has ended.
*/
interface Transcript {
name: string
state?: 'STATE_UNSPECIFIED' | 'STARTED' | 'ENDED' | 'FILE_GENERATED'
startTime?: string
endTime?: string
docsDestination?: DocsDestination
}
interface TranscriptsListResponse {
transcripts?: Transcript[]
nextPageToken?: string
}
/**
* A single speaker-attributed segment of a transcript. `participant` is the resource
* name of the speaking participant (resolved to a display name separately).
*/
interface TranscriptEntry {
name: string
participant?: string
text?: string
languageCode?: string
startTime?: string
endTime?: string
}
interface TranscriptEntriesListResponse {
transcriptEntries?: TranscriptEntry[]
nextPageToken?: string
}
/**
* A meeting participant. The Meet API uses a oneof for the identity — exactly one of
* `signedinUser`, `anonymousUser`, or `phoneUser` is populated, each carrying a
* `displayName`.
*/
interface Participant {
name: string
signedinUser?: { user?: string; displayName?: string }
anonymousUser?: { displayName?: string }
phoneUser?: { displayName?: string }
}
function meetHeaders(accessToken: string): Record<string, string> {
return { Authorization: `Bearer ${accessToken}` }
}
/**
* Normalizes a conference record identifier to its full resource name
* (`conferenceRecords/{id}`), tolerating a bare id.
*/
function conferenceResourceName(externalId: string): string {
const trimmed = externalId.trim()
return trimmed.startsWith('conferenceRecords/') ? trimmed : `conferenceRecords/${trimmed}`
}
/**
* Derives a stable, human-readable title for a meeting. Conference records carry no
* title, so the meeting's start date is used.
*/
function recordTitle(record: ConferenceRecord): string {
const date = record.startTime?.slice(0, 10)
return date ? `Google Meet — ${date}` : 'Google Meet meeting'
}
/**
* Computes the meeting duration in whole minutes, or undefined when the meeting has
* not ended or timestamps are missing.
*/
function recordDurationMinutes(record: ConferenceRecord): number | undefined {
if (!record.startTime || !record.endTime) return undefined
const start = new Date(record.startTime).getTime()
const end = new Date(record.endTime).getTime()
if (Number.isNaN(start) || Number.isNaN(end) || end <= start) return undefined
return Math.round((end - start) / 60000)
}
/**
* Computes the metadata-based change-detection hash for a conference record. Records
* are immutable once ended, so the end time fully captures the final state; an
* in-progress meeting (no end time) re-syncs once it ends and the hash changes. The
* identical formula is used for both the listing stub and the fetched document.
*/
function buildContentHash(record: ConferenceRecord): string {
return `gmeet:${record.name}:${record.endTime ?? ''}`
}
/**
* Builds the deferred listing stub for a conference record. Transcript content is
* fetched lazily in getDocument; only metadata and the change hash are computed here.
*/
function recordToStub(record: ConferenceRecord): ExternalDocument {
return {
externalId: record.name,
title: recordTitle(record),
content: '',
contentDeferred: true,
mimeType: 'text/plain',
contentHash: buildContentHash(record),
metadata: {
meetingDate: record.startTime,
duration: recordDurationMinutes(record),
},
}
}
/**
* Returns a transcript entry's start time as epoch milliseconds for chronological
* sorting. Entries without a parseable start time sort last (stably).
*/
function entryStartMs(entry: TranscriptEntry): number {
if (!entry.startTime) return Number.POSITIVE_INFINITY
const ms = new Date(entry.startTime).getTime()
return Number.isNaN(ms) ? Number.POSITIVE_INFINITY : ms
}
/**
* Resolves a participant's display name across the identity oneof, falling back to a
* stable placeholder when no name is exposed (e.g. anonymous joins).
*/
function participantDisplayName(participant: Participant): string {
return (
participant.signedinUser?.displayName?.trim() ||
participant.anonymousUser?.displayName?.trim() ||
participant.phoneUser?.displayName?.trim() ||
'Unknown'
)
}
/**
* Fetches a single conference record. Returns null on 404 (record expired/deleted).
*/
async function fetchConferenceRecord(
accessToken: string,
name: string
): Promise<ConferenceRecord | null> {
const response = await fetchWithRetry(`${MEET_API_BASE}/${name}`, {
method: 'GET',
headers: meetHeaders(accessToken),
})
if (!response.ok) {
if (response.status === 404) return null
throw new Error(`Failed to fetch Google Meet conference record: ${response.status}`)
}
return (await response.json()) as ConferenceRecord
}
/**
* Lists every transcript belonging to a conference record, following pagination.
*/
async function fetchTranscripts(accessToken: string, recordName: string): Promise<Transcript[]> {
const transcripts: Transcript[] = []
let pageToken: string | undefined
do {
const params = new URLSearchParams({ pageSize: String(TRANSCRIPTS_PAGE_SIZE) })
if (pageToken) params.set('pageToken', pageToken)
const response = await fetchWithRetry(
`${MEET_API_BASE}/${recordName}/transcripts?${params.toString()}`,
{ method: 'GET', headers: meetHeaders(accessToken) }
)
if (!response.ok) {
if (response.status === 404) break
throw new Error(`Failed to list Google Meet transcripts: ${response.status}`)
}
const data = (await response.json()) as TranscriptsListResponse
if (data.transcripts) transcripts.push(...data.transcripts)
pageToken = data.nextPageToken
} while (pageToken)
return transcripts
}
/**
* Lists every entry of a transcript, following pagination. Entries are returned in
* chronological order by the API.
*/
async function fetchTranscriptEntries(
accessToken: string,
transcriptName: string
): Promise<TranscriptEntry[]> {
const entries: TranscriptEntry[] = []
let pageToken: string | undefined
do {
const params = new URLSearchParams({ pageSize: String(ENTRIES_PAGE_SIZE) })
if (pageToken) params.set('pageToken', pageToken)
const response = await fetchWithRetry(
`${MEET_API_BASE}/${transcriptName}/entries?${params.toString()}`,
{ method: 'GET', headers: meetHeaders(accessToken) }
)
if (!response.ok) {
if (response.status === 404) break
throw new Error(`Failed to list Google Meet transcript entries: ${response.status}`)
}
const data = (await response.json()) as TranscriptEntriesListResponse
if (data.transcriptEntries) entries.push(...data.transcriptEntries)
pageToken = data.nextPageToken
} while (pageToken)
return entries
}
/**
* Resolves the display names for a set of participant resource names, returning a map
* keyed by resource name. Participants that fail to resolve are omitted so the caller
* falls back to a placeholder.
*/
async function resolveParticipantNames(
accessToken: string,
participantNames: string[]
): Promise<Map<string, string>> {
const map = new Map<string, string>()
for (let i = 0; i < participantNames.length; i += PARTICIPANT_FETCH_CONCURRENCY) {
const batch = participantNames.slice(i, i + PARTICIPANT_FETCH_CONCURRENCY)
await Promise.all(
batch.map(async (name) => {
try {
const response = await fetchWithRetry(`${MEET_API_BASE}/${name}`, {
method: 'GET',
headers: meetHeaders(accessToken),
})
if (!response.ok) return
const participant = (await response.json()) as Participant
map.set(name, participantDisplayName(participant))
} catch (error) {
logger.warn('Failed to resolve Google Meet participant', {
participant: name,
error: toError(error).message,
})
}
})
)
}
return map
}
/**
* Formats a meeting header plus speaker-attributed transcript lines into plain text.
*/
function formatTranscriptContent(
record: ConferenceRecord,
entries: TranscriptEntry[],
participantNames: Map<string, string>
): string {
const parts: string[] = []
parts.push(`Meeting: ${recordTitle(record)}`)
if (record.startTime) parts.push(`Date: ${record.startTime}`)
const minutes = recordDurationMinutes(record)
if (minutes != null) parts.push(`Duration: ${minutes} minutes`)
const speakers = Array.from(
new Set(
entries
.map((entry) => (entry.participant ? participantNames.get(entry.participant) : undefined))
.filter((name): name is string => Boolean(name))
)
)
if (speakers.length > 0) parts.push(`Participants: ${speakers.join(', ')}`)
parts.push('')
parts.push('--- Transcript ---')
for (const entry of entries) {
const text = entry.text?.trim()
if (!text) continue
const speaker = (entry.participant && participantNames.get(entry.participant)) || 'Unknown'
parts.push(`${speaker}: ${text}`)
}
return parts.join('\n')
}
/**
* Builds the conference records list `filter` from the connector's scoping config.
* Only the documented `start_time` filter is emitted, and only when a lookback window
* is configured (full sync otherwise).
*/
function buildRecordsFilter(sourceConfig: Record<string, unknown>): string | undefined {
const lookbackDays = sourceConfig.lookbackDays ? Number(sourceConfig.lookbackDays) : 0
if (!Number.isFinite(lookbackDays) || lookbackDays <= 0) return undefined
const since = new Date(Date.now() - lookbackDays * MS_PER_DAY).toISOString()
return `start_time >= "${since}"`
}
export const googleMeetConnector: ConnectorConfig = {
...googleMeetConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const maxMeetings = sourceConfig.maxMeetings ? Number(sourceConfig.maxMeetings) : 0
const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
const pageSize =
maxMeetings > 0
? Math.min(RECORDS_PAGE_SIZE, Math.max(1, maxMeetings - prevFetched))
: RECORDS_PAGE_SIZE
const params = new URLSearchParams({ pageSize: String(pageSize) })
if (cursor) params.set('pageToken', cursor)
const filter = buildRecordsFilter(sourceConfig)
if (filter) params.set('filter', filter)
logger.info('Listing Google Meet conference records', {
hasCursor: Boolean(cursor),
hasFilter: Boolean(filter),
})
const response = await fetchWithRetry(
`${MEET_API_BASE}/conferenceRecords?${params.toString()}`,
{ method: 'GET', headers: meetHeaders(accessToken) }
)
if (!response.ok) {
const errorText = await response.text().catch(() => '')
logger.error('Failed to list Google Meet conference records', {
status: response.status,
error: errorText.slice(0, 500),
})
throw new Error(`Failed to list Google Meet conference records: ${response.status}`)
}
const data = (await response.json()) as ConferenceRecordsListResponse
const records = data.conferenceRecords ?? []
const nextPageToken = data.nextPageToken?.trim() || undefined
const allDocuments = records
.filter((record) => Boolean(record.name))
.map((record) => recordToStub(record))
let documents = allDocuments
if (maxMeetings > 0) {
const remaining = Math.max(0, maxMeetings - prevFetched)
if (allDocuments.length > remaining) documents = allDocuments.slice(0, remaining)
}
const totalFetched = prevFetched + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const reachedCap = maxMeetings > 0 && totalFetched >= maxMeetings
// Only flag the listing as capped when the cap actually truncated a larger source —
// either more pages remain, or records were dropped from this page. If the source
// was fully listed and merely happens to equal the cap, leave it unflagged so the
// sync engine still reconciles deletions of meetings that disappear upstream.
const truncated =
reachedCap && (Boolean(nextPageToken) || allDocuments.length > documents.length)
if (truncated && syncContext) syncContext.listingCapped = true
const hasMore = !reachedCap && Boolean(nextPageToken)
return {
documents,
nextCursor: hasMore ? nextPageToken : undefined,
hasMore,
}
},
getDocument: async (
accessToken: string,
_sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
try {
if (!externalId) return null
const recordName = conferenceResourceName(externalId)
const record = await fetchConferenceRecord(accessToken, recordName)
if (!record) return null
const transcripts = await fetchTranscripts(accessToken, recordName)
if (transcripts.length === 0) return null
// Only index once every transcript is fully generated. Before then the entry set
// is still being populated, and because the content hash is keyed on the (now
// fixed) conference endTime, a partial transcript stored here would never be
// refreshed on later syncs. Waiting for FILE_GENERATED keeps indexed content final.
if (transcripts.some((transcript) => transcript.state !== 'FILE_GENERATED')) {
logger.info('Google Meet transcript not finalized yet', { externalId })
return null
}
const entryGroups = await Promise.all(
transcripts.map((transcript) => fetchTranscriptEntries(accessToken, transcript.name))
)
// The API guarantees chronological order only within a single transcript, so sort
// the merged entries by start time to keep speaker lines in sequence when a
// conference has more than one transcript.
const entries = entryGroups.flat().sort((a, b) => entryStartMs(a) - entryStartMs(b))
const hasText = entries.some((entry) => entry.text?.trim())
if (!hasText) {
logger.info('Transcript not yet available for Google Meet conference', { externalId })
return null
}
const participantNames = await resolveParticipantNames(
accessToken,
Array.from(
new Set(
entries
.map((entry) => entry.participant)
.filter((name): name is string => Boolean(name))
)
)
)
const content = formatTranscriptContent(record, entries, participantNames)
const sourceUrl = transcripts.find((t) => t.docsDestination?.exportUri)?.docsDestination
?.exportUri
const speakers = Array.from(new Set(Array.from(participantNames.values())))
return {
externalId: record.name,
title: recordTitle(record),
content,
contentDeferred: false,
mimeType: 'text/plain',
sourceUrl: sourceUrl || undefined,
contentHash: buildContentHash(record),
metadata: {
meetingDate: record.startTime,
duration: recordDurationMinutes(record),
participants: speakers,
},
}
} catch (error) {
logger.warn('Failed to get Google Meet transcript', {
externalId,
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const maxMeetings = sourceConfig.maxMeetings as string | undefined
if (maxMeetings && (Number.isNaN(Number(maxMeetings)) || Number(maxMeetings) < 0)) {
return { valid: false, error: 'Max meetings must be a non-negative number' }
}
const lookbackDays = sourceConfig.lookbackDays as string | undefined
if (lookbackDays && (Number.isNaN(Number(lookbackDays)) || Number(lookbackDays) < 0)) {
return { valid: false, error: 'Lookback window must be a non-negative number of days' }
}
try {
const response = await fetchWithRetry(
`${MEET_API_BASE}/conferenceRecords?pageSize=1`,
{ method: 'GET', headers: meetHeaders(accessToken) },
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
const errorText = await response.text().catch(() => '')
return {
valid: false,
error: `Google Meet access failed: ${response.status}${errorText ? `${errorText.slice(0, 200)}` : ''}`,
}
}
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
const participants = joinTagArray(metadata.participants)
if (participants) result.participants = participants
if (metadata.duration != null) {
const num = Number(metadata.duration)
if (!Number.isNaN(num)) result.duration = num
}
const meetingDate = parseTagDate(metadata.meetingDate)
if (meetingDate) result.meetingDate = meetingDate
return result
},
}
+1
View File
@@ -0,0 +1 @@
export { googleMeetConnector } from '@/connectors/google-meet/google-meet'
+42
View File
@@ -0,0 +1,42 @@
import { GoogleMeetIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const googleMeetConnectorMeta: ConnectorMeta = {
id: 'google_meet',
name: 'Google Meet',
description: 'Sync meeting transcripts from Google Meet into your knowledge base',
version: '1.0.0',
icon: GoogleMeetIcon,
auth: {
mode: 'oauth',
provider: 'google-meet',
requiredScopes: ['https://www.googleapis.com/auth/meetings.space.readonly'],
},
configFields: [
{
id: 'maxMeetings',
title: 'Max Meetings',
type: 'short-input',
required: false,
placeholder: 'e.g. 200 (default: unlimited)',
description: 'Cap the total number of meetings synced. Leave blank to sync all.',
},
{
id: 'lookbackDays',
title: 'Lookback Window (days)',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. 90 (default: all time)',
description: 'Only sync meetings from the last N days. Leave blank to sync any age.',
},
],
tagDefinitions: [
{ id: 'participants', displayName: 'Participants', fieldType: 'text' },
{ id: 'duration', displayName: 'Duration (minutes)', fieldType: 'number' },
{ id: 'meetingDate', displayName: 'Meeting Date', fieldType: 'date' },
],
}
@@ -0,0 +1,379 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { googleSheetsConnectorMeta } from '@/connectors/google-sheets/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { parseTagDate } from '@/connectors/utils'
const logger = createLogger('GoogleSheetsConnector')
const SHEETS_API_BASE = 'https://sheets.googleapis.com/v4/spreadsheets'
const DRIVE_API_BASE = 'https://www.googleapis.com/drive/v3/files'
const MAX_ROWS = 10000
const CONCURRENCY = 3
interface SheetProperties {
sheetId: number
title: string
index: number
gridProperties?: {
rowCount?: number
columnCount?: number
}
}
interface SpreadsheetMetadata {
spreadsheetId: string
properties: {
title: string
locale?: string
}
sheets: { properties: SheetProperties }[]
}
/**
* Formats sheet data into an LLM-friendly text representation.
* Each row is labeled with its index and columns are identified by header names.
*/
function formatSheetContent(headers: string[], rows: string[][]): string {
if (headers.length === 0) return ''
const lines: string[] = []
for (let i = 0; i < rows.length; i++) {
const row = rows[i]
lines.push(`Row ${i + 1}:`)
for (let j = 0; j < headers.length; j++) {
const value = j < row.length ? row[j] : ''
lines.push(` ${headers[j]}: ${value}`)
}
lines.push('')
}
return lines.join('\n').trim()
}
/**
* Fetches all values from a single sheet tab.
*/
async function fetchSheetValues(
accessToken: string,
spreadsheetId: string,
sheetTitle: string
): Promise<string[][]> {
const range = `'${sheetTitle.replace(/'/g, "''")}'!A1:ZZ${MAX_ROWS}`
const url = `${SHEETS_API_BASE}/${encodeURIComponent(spreadsheetId)}/values/${encodeURIComponent(range)}?majorDimension=ROWS&valueRenderOption=FORMATTED_VALUE`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) {
throw new Error(`Failed to fetch sheet values for "${sheetTitle}": ${response.status}`)
}
const data = await response.json()
return (data.values || []) as string[][]
}
/**
* Fetches spreadsheet metadata (title, sheet names, grid properties).
*/
async function fetchSpreadsheetMetadata(
accessToken: string,
spreadsheetId: string
): Promise<SpreadsheetMetadata> {
const url = `${SHEETS_API_BASE}/${encodeURIComponent(spreadsheetId)}?fields=spreadsheetId,properties.title,properties.locale,sheets.properties`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) {
throw new Error(`Failed to fetch spreadsheet metadata: ${response.status}`)
}
return (await response.json()) as SpreadsheetMetadata
}
/**
* Fetches the spreadsheet's modifiedTime from the Drive API.
*/
async function fetchSpreadsheetModifiedTime(
accessToken: string,
spreadsheetId: string
): Promise<string | undefined> {
try {
const url = `${DRIVE_API_BASE}/${encodeURIComponent(spreadsheetId)}?fields=modifiedTime&supportsAllDrives=true`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) {
logger.warn('Failed to fetch modifiedTime from Drive API', { status: response.status })
return undefined
}
const data = (await response.json()) as { modifiedTime?: string }
return data.modifiedTime
} catch (error) {
logger.warn('Error fetching modifiedTime from Drive API', {
error: toError(error).message,
})
return undefined
}
}
/**
* Converts a single sheet tab into an ExternalDocument.
*/
async function sheetToDocument(
accessToken: string,
spreadsheetId: string,
spreadsheetTitle: string,
sheet: SheetProperties,
modifiedTime?: string
): Promise<ExternalDocument | null> {
try {
const values = await fetchSheetValues(accessToken, spreadsheetId, sheet.title)
if (values.length === 0) {
logger.info(`Skipping empty sheet: ${sheet.title}`)
return null
}
const headers = values[0].map((h, idx) =>
typeof h === 'string' && h.trim() ? h.trim() : `Column ${idx + 1}`
)
const dataRows = values.slice(1)
if (dataRows.length === 0) {
logger.info(`Skipping header-only sheet: ${sheet.title}`)
return null
}
const content = formatSheetContent(headers, dataRows)
if (!content.trim()) {
return null
}
const rowCount = dataRows.length
return {
externalId: `${spreadsheetId}__sheet__${sheet.sheetId}`,
title: `${spreadsheetTitle} - ${sheet.title}`,
content,
mimeType: 'text/plain',
sourceUrl: `https://docs.google.com/spreadsheets/d/${spreadsheetId}/edit#gid=${sheet.sheetId}`,
contentHash: `gsheets:${spreadsheetId}:${sheet.sheetId}:${modifiedTime ?? ''}`,
metadata: {
spreadsheetId,
spreadsheetTitle,
sheetTitle: sheet.title,
sheetId: sheet.sheetId,
rowCount,
columnCount: headers.length,
...(modifiedTime ? { modifiedTime } : {}),
},
}
} catch (error) {
logger.warn(`Failed to extract content from sheet: ${sheet.title}`, {
error: toError(error).message,
})
return null
}
}
export const googleSheetsConnector: ConnectorConfig = {
...googleSheetsConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
_cursor?: string,
_syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const spreadsheetId = (sourceConfig.spreadsheetId as string)?.trim()
if (!spreadsheetId) {
throw new Error('Spreadsheet ID is required')
}
logger.info('Fetching spreadsheet metadata', { spreadsheetId })
const [metadata, modifiedTime] = await Promise.all([
fetchSpreadsheetMetadata(accessToken, spreadsheetId),
fetchSpreadsheetModifiedTime(accessToken, spreadsheetId),
])
const sheetFilter = (sourceConfig.sheetFilter as string) || 'all'
let sheets = metadata.sheets.map((s) => s.properties)
if (sheetFilter === 'first' && sheets.length > 0) {
sheets = [sheets[0]]
}
logger.info('Processing sheets', {
spreadsheetTitle: metadata.properties.title,
sheetCount: sheets.length,
})
const documents: ExternalDocument[] = sheets.map((sheet) => ({
externalId: `${spreadsheetId}__sheet__${sheet.sheetId}`,
title: `${metadata.properties.title} - ${sheet.title}`,
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: `https://docs.google.com/spreadsheets/d/${spreadsheetId}/edit#gid=${sheet.sheetId}`,
contentHash: `gsheets:${spreadsheetId}:${sheet.sheetId}:${modifiedTime ?? ''}`,
metadata: {
spreadsheetId,
spreadsheetTitle: metadata.properties.title,
sheetTitle: sheet.title,
sheetId: sheet.sheetId,
rowCount: sheet.gridProperties?.rowCount,
columnCount: sheet.gridProperties?.columnCount,
...(modifiedTime ? { modifiedTime } : {}),
},
}))
return {
documents,
hasMore: false,
}
},
getDocument: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
const parts = externalId.split('__sheet__')
if (parts.length !== 2) {
logger.warn('Invalid external ID format', { externalId })
return null
}
const spreadsheetId = parts[0]
const sheetId = Number(parts[1])
if (Number.isNaN(sheetId)) {
logger.warn('Invalid sheet ID in external ID', { externalId })
return null
}
let metadata: SpreadsheetMetadata
let modifiedTime: string | undefined
try {
;[metadata, modifiedTime] = await Promise.all([
fetchSpreadsheetMetadata(accessToken, spreadsheetId),
fetchSpreadsheetModifiedTime(accessToken, spreadsheetId),
])
} catch (error) {
const message = toError(error).message
if (message.includes('404')) {
logger.info('Spreadsheet not found (possibly deleted)', { spreadsheetId })
return null
}
throw error
}
const sheetEntry = metadata.sheets.find((s) => s.properties.sheetId === sheetId)
if (!sheetEntry) {
logger.info('Sheet not found in spreadsheet', { spreadsheetId, sheetId })
return null
}
const doc = await sheetToDocument(
accessToken,
spreadsheetId,
metadata.properties.title,
sheetEntry.properties,
modifiedTime
)
if (!doc) return null
return { ...doc, contentDeferred: false }
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const spreadsheetId = (sourceConfig.spreadsheetId as string)?.trim()
if (!spreadsheetId) {
return { valid: false, error: 'Spreadsheet ID is required' }
}
try {
const url = `${SHEETS_API_BASE}/${encodeURIComponent(spreadsheetId)}?fields=spreadsheetId,properties.title`
const response = await fetchWithRetry(
url,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
if (response.status === 404) {
return {
valid: false,
error: 'Spreadsheet not found. Check the ID and ensure it is shared with your account.',
}
}
if (response.status === 403) {
return {
valid: false,
error: 'Access denied. Ensure the spreadsheet is shared with your Google account.',
}
}
return { valid: false, error: `Failed to access spreadsheet: ${response.status}` }
}
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.sheetTitle === 'string') {
result.sheetTitle = metadata.sheetTitle
}
if (typeof metadata.rowCount === 'number') {
result.rowCount = metadata.rowCount
}
if (typeof metadata.columnCount === 'number') {
result.columnCount = metadata.columnCount
}
const lastModified = parseTagDate(metadata.modifiedTime)
if (lastModified) {
result.lastModified = lastModified
}
return result
},
}
@@ -0,0 +1 @@
export { googleSheetsConnector } from '@/connectors/google-sheets/google-sheets'
+57
View File
@@ -0,0 +1,57 @@
import { GoogleSheetsIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const googleSheetsConnectorMeta: ConnectorMeta = {
id: 'google_sheets',
name: 'Google Sheets',
description: 'Sync spreadsheet data from Google Sheets',
version: '1.0.0',
icon: GoogleSheetsIcon,
auth: {
mode: 'oauth',
provider: 'google-sheets',
requiredScopes: ['https://www.googleapis.com/auth/drive'],
},
configFields: [
{
id: 'spreadsheetSelector',
title: 'Spreadsheet',
type: 'selector',
selectorKey: 'google.drive',
mimeType: 'application/vnd.google-apps.spreadsheet',
canonicalParamId: 'spreadsheetId',
mode: 'basic',
placeholder: 'Select a spreadsheet',
required: true,
},
{
id: 'spreadsheetId',
title: 'Spreadsheet ID',
type: 'short-input',
canonicalParamId: 'spreadsheetId',
mode: 'advanced',
placeholder: 'e.g. 1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgVE2upms',
required: true,
description: 'The ID from the spreadsheet URL: docs.google.com/spreadsheets/d/{ID}/edit',
},
{
id: 'sheetFilter',
title: 'Sheets to Sync',
type: 'dropdown',
required: false,
options: [
{ label: 'All sheets', id: 'all' },
{ label: 'First sheet only', id: 'first' },
],
},
],
tagDefinitions: [
{ id: 'sheetTitle', displayName: 'Sheet Name', fieldType: 'text' },
{ id: 'rowCount', displayName: 'Row Count', fieldType: 'number' },
{ id: 'columnCount', displayName: 'Column Count', fieldType: 'number' },
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
],
}
+504
View File
@@ -0,0 +1,504 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { grainConnectorMeta } from '@/connectors/grain/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { joinTagArray, parseTagDate } from '@/connectors/utils'
const logger = createLogger('GrainConnector')
const GRAIN_API_BASE = 'https://api.grain.com/_/public-api/v2'
/**
* Grain's Public API requires a pinned date-based version header on every request.
* Matches the version used by the in-repo Grain tools.
*/
const GRAIN_API_VERSION = '2025-10-31'
/**
* A participant on a Grain recording. The list endpoint only populates this when
* `include.participants` is requested in the body.
*/
interface GrainParticipant {
id: string
name: string
email: string | null
}
/**
* A team a Grain recording belongs to. Always present on the recording object (may be
* an empty array).
*/
interface GrainTeam {
id: string
name: string
}
/**
* The meeting type classification of a Grain recording. Always present on the recording
* object but nullable.
*/
interface GrainMeetingType {
id: string
name: string
scope?: 'internal' | 'external'
}
/**
* A Grain recording as returned by the v2 recordings endpoints. Only the fields the
* connector reads are modeled; the API returns additional optional fields.
*
* The v2 Public API returns the recording identifier as `id` (confirmed against the
* Grain Public API reference and the in-repo Grain tools).
*
* `source`, `tags`, `teams`, and `meeting_type` are returned by default on the
* recording object. `participants` is populated only when requested via the
* `include.participants` flag — the connector requests it (see {@link RECORDING_INCLUDE})
* so participant names are available for tag mapping.
*/
interface GrainRecording {
id?: string
title?: string
start_datetime?: string
end_datetime?: string
duration_ms?: number
url?: string
source?: string
tags?: string[]
teams?: GrainTeam[]
meeting_type?: GrainMeetingType | null
participants?: GrainParticipant[]
}
interface GrainRecordingsListResponse {
recordings?: GrainRecording[]
cursor?: string | null
}
/**
* A single speaker-attributed segment of a Grain transcript. The transcript endpoint
* returns a bare JSON array of these.
*/
interface GrainTranscriptSegment {
participant_id: string | null
speaker?: string
start?: number
end?: number
text?: string
}
/**
* The `include` flags requested on every recordings call. Grain returns `teams` and
* `meeting_type` by default, but gates `participants` behind an include flag. Participant
* names feed connector tag mapping, so the flag is always requested. Only documented
* include flags are sent to avoid the API rejecting unknown keys.
*/
const RECORDING_INCLUDE = { participants: true } as const
/** Number of milliseconds in a day, used to convert the lookback window to a timestamp. */
const MS_PER_DAY = 24 * 60 * 60 * 1000
/**
* Valid values for the recordings list `participant_scope` filter (verified against the
* Grain Public API recordings list request body, which the in-repo Grain tools also use).
*/
const PARTICIPANT_SCOPES = ['internal', 'external'] as const
type ParticipantScope = (typeof PARTICIPANT_SCOPES)[number]
function isParticipantScope(value: unknown): value is ParticipantScope {
return typeof value === 'string' && PARTICIPANT_SCOPES.includes(value as ParticipantScope)
}
/**
* Builds the recordings list `filter` object from the connector's scoping config. Only
* documented Grain filter keys are emitted, and only when configured, so an empty config
* produces no `filter` (full sync). Returns undefined when no scoping is configured.
*
* Supported keys (verified against the in-repo Grain list_recordings tool / Public API):
* - `after_datetime` — derived from `lookbackDays`; recordings on/after the window start
* - `participant_scope` — `internal` or `external`
* - `title_search` — substring match against recording titles
* - `team` — recordings belonging to the given team UUID
* - `meeting_type` — recordings classified as the given meeting type UUID
*/
function buildRecordingFilter(
sourceConfig: Record<string, unknown>
): Record<string, unknown> | undefined {
const filter: Record<string, unknown> = {}
const lookbackDays = sourceConfig.lookbackDays ? Number(sourceConfig.lookbackDays) : 0
if (Number.isFinite(lookbackDays) && lookbackDays > 0) {
filter.after_datetime = new Date(Date.now() - lookbackDays * MS_PER_DAY).toISOString()
}
if (isParticipantScope(sourceConfig.participantScope)) {
filter.participant_scope = sourceConfig.participantScope
}
const titleSearch =
typeof sourceConfig.titleSearch === 'string' ? sourceConfig.titleSearch.trim() : ''
if (titleSearch) filter.title_search = titleSearch
const team = typeof sourceConfig.teamId === 'string' ? sourceConfig.teamId.trim() : ''
if (team) filter.team = team
const meetingType =
typeof sourceConfig.meetingTypeId === 'string' ? sourceConfig.meetingTypeId.trim() : ''
if (meetingType) filter.meeting_type = meetingType
return Object.keys(filter).length > 0 ? filter : undefined
}
/**
* Builds the auth + version headers shared by every Grain API request.
*/
function grainHeaders(accessToken: string): Record<string, string> {
return {
'Content-Type': 'application/json',
Authorization: `Bearer ${accessToken}`,
'Public-Api-Version': GRAIN_API_VERSION,
}
}
/**
* Resolves the recording's unique identifier. The v2 Public API returns the recording
* id as the `id` field. Returns an empty string when it is absent.
*/
function recordingId(recording: GrainRecording): string {
return (recording.id ?? '').trim()
}
/**
* Derives the document title for a recording, falling back to a stable placeholder.
*/
function recordingTitle(recording: GrainRecording): string {
return recording.title?.trim() || 'Untitled Grain Recording'
}
/**
* Extracts participant display names from a recording, dropping blanks.
*/
function participantNames(recording: GrainRecording): string[] {
return (recording.participants ?? [])
.map((p) => p.name?.trim())
.filter((name): name is string => Boolean(name))
}
/**
* Extracts team names from a recording, dropping blanks.
*/
function teamNames(recording: GrainRecording): string[] {
return (recording.teams ?? [])
.map((t) => t.name?.trim())
.filter((name): name is string => Boolean(name))
}
/**
* Extracts user-applied tag labels from a recording, dropping blanks.
*/
function recordingLabels(recording: GrainRecording): string[] {
return (recording.tags ?? [])
.map((tag) => tag?.trim())
.filter((tag): tag is string => Boolean(tag))
}
/**
* Computes the metadata-based change-detection hash for a recording.
*
* Grain exposes no `updated_at`/`modified` field, so the hash combines the stable
* recording id with `end_datetime` and `duration_ms` — the values that change when a
* recording is re-processed or re-cut. The identical formula is used for both the
* listing stub and the fully-fetched document so unchanged recordings are skipped.
*/
function buildContentHash(recording: GrainRecording): string {
return `grain:${recordingId(recording)}:${recording.end_datetime ?? ''}:${recording.duration_ms ?? ''}`
}
/**
* Builds the metadata bag attached to both stubs and fetched documents. Keeping a
* single source ensures the stub and getDocument agree on tag inputs.
*/
function buildMetadata(recording: GrainRecording): Record<string, unknown> {
return {
title: recordingTitle(recording),
duration: recording.duration_ms,
meetingDate: recording.start_datetime,
participants: participantNames(recording),
source: recording.source,
labels: recordingLabels(recording),
teams: teamNames(recording),
meetingType: recording.meeting_type?.name,
}
}
/**
* Builds the deferred listing stub for a recording. Content is fetched lazily in
* getDocument; only metadata and the change hash are computed here.
*/
function recordingToStub(recording: GrainRecording): ExternalDocument {
return {
externalId: recordingId(recording),
title: recordingTitle(recording),
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: recording.url || undefined,
contentHash: buildContentHash(recording),
metadata: buildMetadata(recording),
}
}
/**
* Formats a recording header plus speaker-attributed transcript lines into plain text.
*/
function formatTranscriptContent(
recording: GrainRecording,
segments: GrainTranscriptSegment[]
): string {
const parts: string[] = []
parts.push(`Meeting: ${recordingTitle(recording)}`)
if (recording.start_datetime) parts.push(`Date: ${recording.start_datetime}`)
if (recording.duration_ms != null) {
const minutes = Math.round(recording.duration_ms / 60000)
parts.push(`Duration: ${minutes} minutes`)
}
const names = participantNames(recording)
if (names.length > 0) parts.push(`Participants: ${names.join(', ')}`)
parts.push('')
parts.push('--- Transcript ---')
for (const segment of segments) {
const text = segment.text?.trim()
if (!text) continue
const speaker = segment.speaker?.trim() || 'Unknown'
parts.push(`${speaker}: ${text}`)
}
return parts.join('\n')
}
/**
* Fetches a single recording's metadata from the v2 recordings endpoint.
* Returns null on 404 (recording deleted/inaccessible).
*/
async function fetchRecording(accessToken: string, id: string): Promise<GrainRecording | null> {
const response = await fetchWithRetry(`${GRAIN_API_BASE}/recordings/${id}`, {
method: 'POST',
headers: grainHeaders(accessToken),
body: JSON.stringify({ include: RECORDING_INCLUDE }),
})
if (!response.ok) {
if (response.status === 404) return null
throw new Error(`Failed to fetch Grain recording: ${response.status}`)
}
return (await response.json()) as GrainRecording
}
/**
* Fetches the speaker-attributed transcript segments for a recording.
* Returns null on 404, or an empty array when the recording has no transcript yet.
*/
async function fetchTranscript(
accessToken: string,
id: string
): Promise<GrainTranscriptSegment[] | null> {
const response = await fetchWithRetry(`${GRAIN_API_BASE}/recordings/${id}/transcript`, {
method: 'GET',
headers: grainHeaders(accessToken),
})
if (!response.ok) {
if (response.status === 404) return null
throw new Error(`Failed to fetch Grain transcript: ${response.status}`)
}
const data = await response.json()
return Array.isArray(data) ? (data as GrainTranscriptSegment[]) : []
}
export const grainConnector: ConnectorConfig = {
...grainConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const maxRecordings = sourceConfig.maxRecordings ? Number(sourceConfig.maxRecordings) : 0
const body: Record<string, unknown> = { include: RECORDING_INCLUDE }
if (cursor) body.cursor = cursor
const cachedFilter = syncContext?.grainFilter as Record<string, unknown> | undefined | null
const filter = cachedFilter !== undefined ? cachedFilter : buildRecordingFilter(sourceConfig)
if (syncContext && cachedFilter === undefined) syncContext.grainFilter = filter ?? null
if (filter) body.filter = filter
logger.info('Listing Grain recordings', {
hasCursor: Boolean(cursor),
hasFilter: Boolean(filter),
})
const response = await fetchWithRetry(`${GRAIN_API_BASE}/recordings`, {
method: 'POST',
headers: grainHeaders(accessToken),
body: JSON.stringify(body),
})
if (!response.ok) {
const errorText = await response.text().catch(() => '')
logger.error('Failed to list Grain recordings', {
status: response.status,
error: errorText.slice(0, 500),
})
throw new Error(`Failed to list Grain recordings: ${response.status}`)
}
const data = (await response.json()) as GrainRecordingsListResponse
const recordings = data.recordings ?? []
const nextCursor = data.cursor?.trim() || undefined
const allDocuments: ExternalDocument[] = []
for (const recording of recordings) {
if (!recordingId(recording)) continue
allDocuments.push(recordingToStub(recording))
}
const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
let documents = allDocuments
if (maxRecordings > 0) {
const remaining = Math.max(0, maxRecordings - prevFetched)
if (allDocuments.length > remaining) {
documents = allDocuments.slice(0, remaining)
}
}
const totalFetched = prevFetched + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = maxRecordings > 0 && totalFetched >= maxRecordings
if (hitLimit && syncContext) syncContext.listingCapped = true
const hasMore = !hitLimit && Boolean(nextCursor)
return {
documents,
nextCursor: hasMore ? nextCursor : undefined,
hasMore,
}
},
getDocument: async (
accessToken: string,
_sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
try {
if (!externalId) return null
const [recording, segments] = await Promise.all([
fetchRecording(accessToken, externalId),
fetchTranscript(accessToken, externalId),
])
if (!recording) return null
if (!segments) return null
const hasTranscript = segments.some((segment) => segment.text?.trim())
if (!hasTranscript) {
logger.info('Transcript not yet available for Grain recording', { externalId })
return null
}
const content = formatTranscriptContent(recording, segments)
return {
externalId,
title: recordingTitle(recording),
content,
contentDeferred: false,
mimeType: 'text/plain',
sourceUrl: recording.url || undefined,
contentHash: buildContentHash(recording),
metadata: buildMetadata(recording),
}
} catch (error) {
logger.warn('Failed to get Grain recording', {
externalId,
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const maxRecordings = sourceConfig.maxRecordings as string | undefined
if (maxRecordings && (Number.isNaN(Number(maxRecordings)) || Number(maxRecordings) < 0)) {
return { valid: false, error: 'Max recordings must be a non-negative number' }
}
try {
const response = await fetchWithRetry(
`${GRAIN_API_BASE}/recordings`,
{
method: 'POST',
headers: grainHeaders(accessToken),
body: JSON.stringify({}),
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
const errorText = await response.text().catch(() => '')
return {
valid: false,
error: `Grain access failed: ${response.status}${errorText ? `${errorText.slice(0, 200)}` : ''}`,
}
}
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.title === 'string' && metadata.title.trim()) {
result.title = metadata.title
}
const participants = joinTagArray(metadata.participants)
if (participants) result.participants = participants
if (typeof metadata.source === 'string' && metadata.source.trim()) {
result.source = metadata.source.trim()
}
const labels = joinTagArray(metadata.labels)
if (labels) result.labels = labels
const teams = joinTagArray(metadata.teams)
if (teams) result.teams = teams
if (typeof metadata.meetingType === 'string' && metadata.meetingType.trim()) {
result.meetingType = metadata.meetingType.trim()
}
if (metadata.duration != null) {
const num = Number(metadata.duration)
if (!Number.isNaN(num)) result.duration = num
}
const meetingDate = parseTagDate(metadata.meetingDate)
if (meetingDate) result.meetingDate = meetingDate
return result
},
}
+1
View File
@@ -0,0 +1 @@
export { grainConnector } from '@/connectors/grain/grain'
+90
View File
@@ -0,0 +1,90 @@
import { GrainIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const grainConnectorMeta: ConnectorMeta = {
id: 'grain',
name: 'Grain',
description: 'Sync meeting recording transcripts from Grain',
version: '1.0.0',
icon: GrainIcon,
auth: {
mode: 'apiKey',
label: 'API Key',
placeholder: 'Enter your Grain API key',
},
configFields: [
{
id: 'maxRecordings',
title: 'Max Recordings',
type: 'short-input',
required: false,
placeholder: 'e.g. 200 (default: unlimited)',
description: 'Cap the total number of recordings synced. Leave blank to sync all.',
},
{
id: 'lookbackDays',
title: 'Lookback Window (days)',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. 90 (default: all time)',
description: 'Only sync recordings from the last N days. Leave blank to sync any age.',
},
{
id: 'participantScope',
title: 'Participant Scope',
type: 'dropdown',
required: false,
mode: 'advanced',
description:
'Limit to internal-only meetings or meetings that include an external participant. Leave as Any to sync both.',
options: [
{ label: 'Any', id: '' },
{ label: 'Internal only', id: 'internal' },
{ label: 'External (has external participant)', id: 'external' },
],
},
{
id: 'titleSearch',
title: 'Title Search',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. weekly standup',
description: 'Only sync recordings whose title matches this text. Leave blank to sync all.',
},
{
id: 'teamId',
title: 'Team ID',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. a1b2c3d4-e5f6-7890-abcd-ef1234567890',
description:
'Only sync recordings belonging to this team (Grain team UUID). Leave blank to sync all teams.',
},
{
id: 'meetingTypeId',
title: 'Meeting Type ID',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. a1b2c3d4-e5f6-7890-abcd-ef1234567890',
description:
'Only sync recordings of this meeting type (Grain meeting type UUID). Leave blank to sync all types.',
},
],
tagDefinitions: [
{ id: 'title', displayName: 'Title', fieldType: 'text' },
{ id: 'participants', displayName: 'Participants', fieldType: 'text' },
{ id: 'source', displayName: 'Source', fieldType: 'text' },
{ id: 'labels', displayName: 'Labels', fieldType: 'text' },
{ id: 'teams', displayName: 'Teams', fieldType: 'text' },
{ id: 'meetingType', displayName: 'Meeting Type', fieldType: 'text' },
{ id: 'duration', displayName: 'Duration (ms)', fieldType: 'number' },
{ id: 'meetingDate', displayName: 'Meeting Date', fieldType: 'date' },
],
}
+465
View File
@@ -0,0 +1,465 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { granolaConnectorMeta } from '@/connectors/granola/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { htmlToPlainText, joinTagArray, parseTagDate } from '@/connectors/utils'
const logger = createLogger('GranolaConnector')
const GRANOLA_API_BASE = 'https://public-api.granola.ai/v1'
/** Granola caps page_size at 30; request the maximum to minimize round trips. */
const PAGE_SIZE = 30
/** Granola folder identifiers match `fol_` followed by 14 alphanumeric chars. */
const FOLDER_ID_PATTERN = /^fol_[a-zA-Z0-9]{14}$/
/**
* A note owner or attendee as returned by the Granola API.
*/
interface GranolaUser {
name: string | null
email: string
}
/**
* The lightweight note shape returned by the list endpoint. It contains only
* metadata — no summary or transcript content — so content must be fetched per
* note via the get endpoint (the deferred-content pattern).
*/
interface GranolaNoteSummary {
id: string
object?: string
title: string | null
owner?: GranolaUser
created_at: string
updated_at: string
}
/**
* A folder the note belongs to, as returned by the get endpoint.
*/
interface GranolaFolderMembership {
id: string
name: string
parent_folder_id?: string | null
}
/**
* Calendar event details attached to a note, when available.
* Field names match the Granola API's CalendarEvent schema.
*/
interface GranolaCalendarEvent {
event_title?: string | null
organiser?: string | null
calendar_event_id?: string | null
scheduled_start_time?: string | null
scheduled_end_time?: string | null
invitees?: { email: string }[]
}
/**
* The full note shape returned by the get endpoint, including summary content.
*/
interface GranolaNoteDetail extends GranolaNoteSummary {
web_url?: string | null
calendar_event?: GranolaCalendarEvent | null
attendees?: GranolaUser[]
folder_membership?: GranolaFolderMembership[]
summary_text?: string | null
summary_markdown?: string | null
}
/**
* The list endpoint response envelope.
*/
interface GranolaListNotesResponse {
notes?: GranolaNoteSummary[]
hasMore?: boolean
cursor?: string | null
}
/**
* Builds the authorization headers for a Granola API request.
*/
function granolaHeaders(accessToken: string): Record<string, string> {
return {
Authorization: `Bearer ${accessToken}`,
'Content-Type': 'application/json',
}
}
/**
* Produces the change-detection hash for a note from its stable identifiers.
* Granola exposes `updated_at`, which advances whenever the note (or its summary)
* changes, so a metadata-only hash is sufficient and stays identical between the
* list stub and the fetched document — letting the sync engine skip re-fetching
* unchanged notes.
*/
function buildContentHash(id: string, updatedAt: string): string {
return `granola:${id}:${updatedAt}`
}
/**
* Parses the optional `maxNotes` cap from source config.
* Returns 0 (unlimited) when unset or invalid.
*/
function parseMaxNotes(sourceConfig: Record<string, unknown>): number {
const raw = sourceConfig.maxNotes
if (raw == null || raw === '') return 0
const num = Number(raw)
return Number.isFinite(num) && num > 0 ? Math.floor(num) : 0
}
/**
* Parses the optional `folderId` scope from source config. Returns a trimmed
* folder id only when it matches Granola's `fol_…` identifier shape; otherwise
* returns undefined so the request is not scoped to an invalid folder.
*/
function parseFolderId(sourceConfig: Record<string, unknown>): string | undefined {
const raw = sourceConfig.folderId
if (typeof raw !== 'string') return undefined
const trimmed = raw.trim()
if (!trimmed) return undefined
return FOLDER_ID_PATTERN.test(trimmed) ? trimmed : undefined
}
/**
* Parses an optional ISO 8601 date filter from a named source-config field.
* Returns a normalized ISO 8601 string when the value is a valid date; otherwise
* returns undefined so the request is not scoped to an invalid date.
*/
function parseDateFilter(sourceConfig: Record<string, unknown>, key: string): string | undefined {
const raw = sourceConfig[key]
if (typeof raw !== 'string') return undefined
const trimmed = raw.trim()
if (!trimmed) return undefined
const parsed = new Date(trimmed)
return Number.isNaN(parsed.getTime()) ? undefined : parsed.toISOString()
}
/**
* Detects whether a string contains HTML markup. Granola returns markdown for
* `summary_markdown`, but this guard lets us defensively strip tags if the API
* ever emits HTML, without mangling legitimate markdown.
*/
function looksLikeHtml(value: string): boolean {
return /<\/?[a-z][\s\S]*?>/i.test(value)
}
/**
* Assembles the document content from a note's title and summary. Prefers the
* markdown summary, falling back to plain-text summary. HTML is stripped only
* when detected so markdown formatting is preserved.
*/
function buildContent(note: GranolaNoteDetail): string {
const parts: string[] = []
const title = note.title?.trim()
if (title) parts.push(`# ${title}`)
const rawSummary = note.summary_markdown?.trim() || note.summary_text?.trim() || ''
if (rawSummary) {
parts.push('')
parts.push(looksLikeHtml(rawSummary) ? htmlToPlainText(rawSummary) : rawSummary)
}
return parts.join('\n').trim()
}
/**
* Resolves an owner's display name, falling back to email, for tag mapping.
*/
function ownerDisplay(owner?: GranolaUser): string | undefined {
if (!owner) return undefined
return owner.name?.trim() || owner.email?.trim() || undefined
}
/**
* Collects attendee display names (falling back to email) for tag mapping.
*/
function collectAttendees(note: GranolaNoteDetail): string[] {
if (!Array.isArray(note.attendees)) return []
return note.attendees
.map((a) => a.name?.trim() || a.email?.trim() || '')
.filter((name): name is string => name.length > 0)
}
/**
* Collects folder names for tag mapping.
*/
function collectFolders(note: GranolaNoteDetail): string[] {
if (!Array.isArray(note.folder_membership)) return []
return note.folder_membership
.map((f) => f.name?.trim() || '')
.filter((name): name is string => name.length > 0)
}
/**
* Builds the deferred stub for a note from list metadata. Content is empty and
* fetched later via `getDocument` only for new/changed notes.
*/
function noteSummaryToStub(note: GranolaNoteSummary): ExternalDocument {
return {
externalId: note.id,
title: note.title?.trim() || 'Untitled Note',
content: '',
contentDeferred: true,
mimeType: 'text/markdown',
contentHash: buildContentHash(note.id, note.updated_at),
metadata: {
title: note.title?.trim() || undefined,
owner: ownerDisplay(note.owner),
ownerName: note.owner?.name ?? undefined,
ownerEmail: note.owner?.email ?? undefined,
noteDate: note.created_at,
updatedAt: note.updated_at,
},
}
}
export const granolaConnector: ConnectorConfig = {
...granolaConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>,
lastSyncAt?: Date
): Promise<ExternalDocumentList> => {
const maxNotes = parseMaxNotes(sourceConfig)
const folderId = parseFolderId(sourceConfig)
const createdAfter = parseDateFilter(sourceConfig, 'createdAfter')
const createdBefore = parseDateFilter(sourceConfig, 'createdBefore')
const url = new URL(`${GRANOLA_API_BASE}/notes`)
url.searchParams.set('page_size', String(PAGE_SIZE))
if (cursor) url.searchParams.set('cursor', cursor)
if (lastSyncAt) url.searchParams.set('updated_after', lastSyncAt.toISOString())
if (folderId) url.searchParams.set('folder_id', folderId)
if (createdAfter) url.searchParams.set('created_after', createdAfter)
if (createdBefore) url.searchParams.set('created_before', createdBefore)
logger.info('Listing Granola notes', {
hasCursor: Boolean(cursor),
incremental: Boolean(lastSyncAt),
scopedToFolder: Boolean(folderId),
scopedByCreatedAfter: Boolean(createdAfter),
scopedByCreatedBefore: Boolean(createdBefore),
})
const response = await fetchWithRetry(url.toString(), {
method: 'GET',
headers: granolaHeaders(accessToken),
})
if (!response.ok) {
const errorText = await response.text().catch(() => '')
logger.error('Failed to list Granola notes', {
status: response.status,
error: errorText.slice(0, 500),
})
throw new Error(`Failed to list Granola notes: ${response.status}`)
}
const data = (await response.json()) as GranolaListNotesResponse
const notes = Array.isArray(data.notes) ? data.notes : []
const nextCursor = data.cursor?.trim() || undefined
const allStubs = notes.filter((note) => Boolean(note.id)).map((note) => noteSummaryToStub(note))
const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
let documents = allStubs
if (maxNotes > 0) {
const remaining = Math.max(0, maxNotes - prevFetched)
if (allStubs.length > remaining) {
documents = allStubs.slice(0, remaining)
}
}
const totalFetched = prevFetched + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = maxNotes > 0 && totalFetched >= maxNotes
if (hitLimit && syncContext) syncContext.listingCapped = true
const hasMore = !hitLimit && Boolean(data.hasMore) && Boolean(nextCursor)
return {
documents,
nextCursor: hasMore ? nextCursor : undefined,
hasMore,
}
},
getDocument: async (
accessToken: string,
_sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
try {
if (!externalId) return null
const url = `${GRANOLA_API_BASE}/notes/${encodeURIComponent(externalId)}`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: granolaHeaders(accessToken),
})
if (!response.ok) {
if (response.status === 404 || response.status === 410) return null
throw new Error(`Failed to fetch Granola note: ${response.status}`)
}
const note = (await response.json()) as GranolaNoteDetail
if (!note?.id) return null
const content = buildContent(note)
if (!content) {
logger.info('Granola note has no content', { externalId })
return null
}
const attendees = collectAttendees(note)
const folders = collectFolders(note)
const meeting = note.calendar_event?.event_title?.trim() || undefined
const meetingDate = note.calendar_event?.scheduled_start_time?.trim() || undefined
return {
externalId: note.id,
title: note.title?.trim() || 'Untitled Note',
content,
contentDeferred: false,
mimeType: 'text/markdown',
sourceUrl: note.web_url?.trim() || undefined,
contentHash: buildContentHash(note.id, note.updated_at),
metadata: {
title: note.title?.trim() || undefined,
owner: ownerDisplay(note.owner),
ownerName: note.owner?.name ?? undefined,
ownerEmail: note.owner?.email ?? undefined,
noteDate: note.created_at,
updatedAt: note.updated_at,
attendees,
folders,
meeting,
meetingDate,
},
}
} catch (error) {
logger.warn('Failed to get Granola note', {
externalId,
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const maxNotes = sourceConfig.maxNotes as string | undefined
if (maxNotes && (Number.isNaN(Number(maxNotes)) || Number(maxNotes) < 0)) {
return { valid: false, error: 'Max notes must be a non-negative number' }
}
const folderId = sourceConfig.folderId
if (
typeof folderId === 'string' &&
folderId.trim() &&
!FOLDER_ID_PATTERN.test(folderId.trim())
) {
return {
valid: false,
error:
'Folder ID must look like fol_ followed by 14 alphanumeric characters (e.g. fol_4y6LduVdwSKC27)',
}
}
const createdAfter = sourceConfig.createdAfter
if (
typeof createdAfter === 'string' &&
createdAfter.trim() &&
Number.isNaN(new Date(createdAfter.trim()).getTime())
) {
return {
valid: false,
error:
'Created After must be a valid date (ISO 8601, e.g. 2025-01-01 or 2025-01-01T00:00:00Z)',
}
}
const createdBefore = sourceConfig.createdBefore
if (
typeof createdBefore === 'string' &&
createdBefore.trim() &&
Number.isNaN(new Date(createdBefore.trim()).getTime())
) {
return {
valid: false,
error:
'Created Before must be a valid date (ISO 8601, e.g. 2025-12-31 or 2025-12-31T23:59:59Z)',
}
}
try {
const url = new URL(`${GRANOLA_API_BASE}/notes`)
url.searchParams.set('page_size', '1')
const response = await fetchWithRetry(
url.toString(),
{
method: 'GET',
headers: granolaHeaders(accessToken),
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
const errorText = await response.text().catch(() => '')
return {
valid: false,
error: `Granola access failed: ${response.status}${errorText ? `${errorText.slice(0, 200)}` : ''}`,
}
}
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.title === 'string' && metadata.title.trim()) {
result.title = metadata.title.trim()
}
if (typeof metadata.owner === 'string' && metadata.owner.trim()) {
result.owner = metadata.owner.trim()
}
const attendees = joinTagArray(metadata.attendees)
if (attendees) result.attendees = attendees
const folders = joinTagArray(metadata.folders)
if (folders) result.folders = folders
if (typeof metadata.meeting === 'string' && metadata.meeting.trim()) {
result.meeting = metadata.meeting.trim()
}
const noteDate = parseTagDate(metadata.noteDate)
if (noteDate) result.noteDate = noteDate
const meetingDate = parseTagDate(metadata.meetingDate)
if (meetingDate) result.meetingDate = meetingDate
return result
},
}
+1
View File
@@ -0,0 +1 @@
export { granolaConnector } from '@/connectors/granola/granola'
+69
View File
@@ -0,0 +1,69 @@
import { GranolaIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const granolaConnectorMeta: ConnectorMeta = {
id: 'granola',
name: 'Granola',
description: 'Sync AI meeting notes and summaries from Granola',
version: '1.0.0',
icon: GranolaIcon,
auth: {
mode: 'apiKey',
label: 'API Key',
placeholder: 'Enter your Granola API key',
},
supportsIncrementalSync: true,
configFields: [
{
id: 'maxNotes',
title: 'Max Notes',
type: 'short-input',
required: false,
placeholder: 'e.g. 200 (default: unlimited)',
description: 'Cap the number of notes synced. Leave blank to sync all notes.',
},
{
id: 'folderId',
title: 'Folder ID',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. fol_4y6LduVdwSKC27',
description:
'Scope the sync to a single folder and its child folders. Leave blank to sync notes from all folders.',
},
{
id: 'createdAfter',
title: 'Created After',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. 2025-01-01 or 2025-01-01T00:00:00Z',
description:
'Only sync notes created on or after this date (ISO 8601). Leave blank to sync notes regardless of creation date.',
},
{
id: 'createdBefore',
title: 'Created Before',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. 2025-12-31 or 2025-12-31T23:59:59Z',
description:
'Only sync notes created on or before this date (ISO 8601). Leave blank to sync notes regardless of creation date.',
},
],
tagDefinitions: [
{ id: 'title', displayName: 'Title', fieldType: 'text' },
{ id: 'owner', displayName: 'Owner', fieldType: 'text' },
{ id: 'attendees', displayName: 'Attendees', fieldType: 'text' },
{ id: 'folders', displayName: 'Folders', fieldType: 'text' },
{ id: 'meeting', displayName: 'Meeting', fieldType: 'text' },
{ id: 'noteDate', displayName: 'Note Date', fieldType: 'date' },
{ id: 'meetingDate', displayName: 'Meeting Date', fieldType: 'date' },
],
}
@@ -0,0 +1,671 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { greenhouseConnectorMeta } from '@/connectors/greenhouse/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { htmlToPlainText, parseTagDate } from '@/connectors/utils'
const logger = createLogger('GreenhouseConnector')
const GREENHOUSE_API_BASE = 'https://harvest.greenhouse.io/v1'
/**
* Upper bound on per-application scorecard requests during a single getDocument call.
* A candidate can have many applications (e.g. internal-transfer tracking); without a
* cap, one document fetch could fan out into dozens of sequential requests. Mirrors the
* Ashby connector's MAX_APPLICATIONS_FOR_FEEDBACK bound.
*/
const MAX_APPLICATIONS_FOR_SCORECARDS = 10
/**
* Greenhouse Harvest allows up to 500 candidates per page. We page through the
* full list using the `page` query parameter and stop when the `Link` response
* header no longer advertises a `rel="next"` relationship.
*/
const CANDIDATES_PER_PAGE = 500
/**
* Minutes of overlap subtracted from `lastSyncAt` when computing the incremental
* `updated_after` window. Catches candidates whose `updated_at` lands fractionally
* before the recorded sync boundary (clock skew, late-committed writes) at the
* cost of re-listing a small number of recently-touched candidates.
*/
const INCREMENTAL_OVERLAP_MINUTES = 5
const MS_PER_MINUTE = 60 * 1000
interface GreenhouseUser {
id?: number
first_name?: string | null
last_name?: string | null
name?: string | null
employee_id?: string | null
}
interface GreenhouseEmailAddress {
value?: string
type?: string
}
interface GreenhouseSource {
id?: number
public_name?: string | null
}
interface GreenhouseApplication {
id?: number
status?: string | null
applied_at?: string | null
last_activity_at?: string | null
source?: GreenhouseSource | null
recruiter?: GreenhouseUser | null
coordinator?: GreenhouseUser | null
}
interface GreenhouseCandidate {
id: number
first_name?: string | null
last_name?: string | null
company?: string | null
title?: string | null
created_at?: string | null
updated_at?: string | null
last_activity?: string | null
email_addresses?: GreenhouseEmailAddress[]
tags?: string[]
application_ids?: number[]
applications?: GreenhouseApplication[]
recruiter?: GreenhouseUser | null
coordinator?: GreenhouseUser | null
}
interface GreenhouseActivityNote {
id?: number
created_at?: string | null
body?: string | null
user?: GreenhouseUser | null
visibility?: string | null
}
interface GreenhouseActivityEmail {
id?: number
created_at?: string | null
subject?: string | null
body?: string | null
to?: string | null
from?: string | null
user?: GreenhouseUser | null
}
interface GreenhouseActivityEvent {
id?: number
created_at?: string | null
subject?: string | null
body?: string | null
user?: GreenhouseUser | null
}
interface GreenhouseActivityFeed {
notes?: GreenhouseActivityNote[]
emails?: GreenhouseActivityEmail[]
activities?: GreenhouseActivityEvent[]
}
interface GreenhouseScorecardAttribute {
name?: string | null
type?: string | null
note?: string | null
rating?: string | null
}
interface GreenhouseScorecardQuestion {
question?: string | null
answer?: string | null
}
interface GreenhouseScorecard {
id?: number
interview?: string | null
interviewer?: GreenhouseUser | null
submitted_by?: GreenhouseUser | null
submitted_at?: string | null
interviewed_at?: string | null
overall_recommendation?: string | null
attributes?: GreenhouseScorecardAttribute[]
questions?: GreenhouseScorecardQuestion[]
}
/**
* Builds the HTTP Basic authorization header for Greenhouse Harvest. The API key
* is used as the username with an empty password, base64-encoded as `apiKey:`.
*/
function buildAuthHeader(accessToken: string): string {
return `Basic ${Buffer.from(`${accessToken}:`).toString('base64')}`
}
/**
* Parses the RFC 5988 `Link` response header and returns true when a
* `rel="next"` relationship is present, indicating another page exists.
*/
function hasNextPage(linkHeader: string | null): boolean {
if (!linkHeader) return false
return /;\s*rel\s*=\s*"?next"?/i.test(linkHeader)
}
/**
* Builds a display name from a candidate's first and last name, falling back to
* the candidate ID when both are missing.
*/
function candidateDisplayName(candidate: GreenhouseCandidate): string {
const parts = [candidate.first_name, candidate.last_name]
.map((part) => part?.trim())
.filter((part): part is string => Boolean(part))
return parts.length > 0 ? parts.join(' ') : `Candidate ${candidate.id}`
}
/**
* Computes the metadata-based content hash for a candidate. Both the listing stub
* and `getDocument` use the same formula so the sync engine can detect changes
* without downloading the deferred content. Greenhouse advances a candidate's
* `updated_at` when the candidate record changes; profile-affecting activity
* (notes, emails, stage changes, scorecard submissions) typically also touches it,
* which is why `updated_after` listing and this hash track the same field.
*/
function buildContentHash(id: number, updatedAt?: string | null): string {
return `greenhouse:${id}:${updatedAt ?? ''}`
}
/**
* Resolves the source URL for a candidate in the Greenhouse recruiting UI.
*/
function buildSourceUrl(id: number): string {
return `https://app.greenhouse.io/people/${id}`
}
/**
* Resolves the recruiter, coordinator, and source for a candidate. Greenhouse
* exposes these both at the candidate level and per-application; the candidate
* level is preferred and the most-recent application is used as a fallback so
* the tags are populated even when the candidate-level fields are empty.
*/
function resolveOwnership(candidate: GreenhouseCandidate): {
recruiter?: string
coordinator?: string
source?: string
} {
const applications = Array.isArray(candidate.applications) ? candidate.applications : []
const latest = applications.reduce<GreenhouseApplication | undefined>((acc, app) => {
if (!acc) return app
const accTime = acc.applied_at ? Date.parse(acc.applied_at) : 0
const appTime = app.applied_at ? Date.parse(app.applied_at) : 0
return appTime >= accTime ? app : acc
}, undefined)
const recruiterName = userName(candidate.recruiter ?? latest?.recruiter)
const coordinatorName = userName(candidate.coordinator ?? latest?.coordinator)
const sourceName = latest?.source?.public_name?.trim()
return {
recruiter: recruiterName !== 'Unknown' ? recruiterName : undefined,
coordinator: coordinatorName !== 'Unknown' ? coordinatorName : undefined,
source: sourceName || undefined,
}
}
/**
* Builds the shared metadata block for a candidate, used by both the listing
* stub and the fully-hydrated document.
*/
function buildMetadata(candidate: GreenhouseCandidate): Record<string, unknown> {
const emails = (candidate.email_addresses ?? [])
.map((e) => e.value?.trim())
.filter((value): value is string => Boolean(value))
const { recruiter, coordinator, source } = resolveOwnership(candidate)
const applicationCount = Array.isArray(candidate.application_ids)
? candidate.application_ids.length
: Array.isArray(candidate.applications)
? candidate.applications.length
: 0
return {
candidateName: candidateDisplayName(candidate),
company: candidate.company ?? undefined,
title: candidate.title ?? undefined,
emails,
tags: Array.isArray(candidate.tags) ? candidate.tags : [],
createdAt: candidate.created_at ?? undefined,
updatedAt: candidate.updated_at ?? undefined,
lastActivity: candidate.last_activity ?? undefined,
recruiter,
coordinator,
source,
applicationCount,
}
}
/**
* Creates a lightweight, content-deferred document stub from a candidate listing
* entry. The real content is fetched lazily via `getDocument` for new or changed
* candidates only.
*/
function candidateToStub(candidate: GreenhouseCandidate): ExternalDocument {
return {
externalId: String(candidate.id),
title: candidateDisplayName(candidate),
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: buildSourceUrl(candidate.id),
contentHash: buildContentHash(candidate.id, candidate.updated_at),
metadata: buildMetadata(candidate),
}
}
/**
* Formats a user's display name for inclusion in content lines.
*/
function userName(user?: GreenhouseUser | null): string {
if (!user) return 'Unknown'
if (user.name?.trim()) return user.name.trim()
const parts = [user.first_name, user.last_name]
.map((part) => part?.trim())
.filter((part): part is string => Boolean(part))
return parts.length > 0 ? parts.join(' ') : 'Unknown'
}
/**
* Renders a candidate's activity feed (notes, emails, and events) into plain text.
*/
function formatActivityFeed(feed: GreenhouseActivityFeed): string {
const lines: string[] = []
const notes = feed.notes ?? []
if (notes.length > 0) {
lines.push('--- Notes ---')
for (const note of notes) {
const body = htmlToPlainText(note.body ?? '').trim()
if (!body) continue
const when = note.created_at ? ` (${note.created_at})` : ''
lines.push(`[${userName(note.user)}${when}] ${body}`)
}
lines.push('')
}
const activities = feed.activities ?? []
if (activities.length > 0) {
lines.push('--- Activities ---')
for (const activity of activities) {
const body = htmlToPlainText(activity.body ?? '').trim()
const subject = activity.subject?.trim()
const text = [subject, body].filter(Boolean).join(': ')
if (!text) continue
const when = activity.created_at ? ` (${activity.created_at})` : ''
lines.push(`[${userName(activity.user)}${when}] ${text}`)
}
lines.push('')
}
const emails = feed.emails ?? []
if (emails.length > 0) {
lines.push('--- Emails ---')
for (const email of emails) {
const body = htmlToPlainText(email.body ?? '').trim()
const subject = email.subject?.trim()
if (!subject && !body) continue
const when = email.created_at ? ` (${email.created_at})` : ''
if (subject) lines.push(`[${userName(email.user)}${when}] Subject: ${subject}`)
if (body) lines.push(body)
}
lines.push('')
}
return lines.join('\n').trim()
}
/**
* Renders interview scorecards (recommendations, attribute ratings, and free-text
* question feedback) into plain text.
*/
function formatScorecards(scorecards: GreenhouseScorecard[]): string {
if (scorecards.length === 0) return ''
const lines: string[] = ['--- Scorecards ---']
for (const scorecard of scorecards) {
const interview = scorecard.interview?.trim() || 'Interview'
const reviewer = userName(scorecard.submitted_by ?? scorecard.interviewer)
const when = scorecard.submitted_at ? ` (${scorecard.submitted_at})` : ''
lines.push(`# ${interview}${reviewer}${when}`)
if (scorecard.overall_recommendation?.trim()) {
lines.push(`Overall recommendation: ${scorecard.overall_recommendation.trim()}`)
}
for (const attribute of scorecard.attributes ?? []) {
const name = attribute.name?.trim()
if (!name) continue
const rating = attribute.rating?.trim()
const note = htmlToPlainText(attribute.note ?? '').trim()
const detail = [rating ? `rating: ${rating}` : '', note].filter(Boolean).join(' — ')
lines.push(detail ? `- ${name}: ${detail}` : `- ${name}`)
}
for (const question of scorecard.questions ?? []) {
const q = htmlToPlainText(question.question ?? '').trim()
const a = htmlToPlainText(question.answer ?? '').trim()
if (!q && !a) continue
lines.push(q ? `Q: ${q}` : 'Q:')
if (a) lines.push(`A: ${a}`)
}
lines.push('')
}
return lines.join('\n').trim()
}
/**
* Assembles the full document content for a candidate from their profile header,
* activity feed, and interview scorecards.
*/
function formatContent(
candidate: GreenhouseCandidate,
feed: GreenhouseActivityFeed,
scorecards: GreenhouseScorecard[]
): string {
const sections: string[] = []
const header: string[] = [`Candidate: ${candidateDisplayName(candidate)}`]
if (candidate.title?.trim()) header.push(`Title: ${candidate.title.trim()}`)
if (candidate.company?.trim()) header.push(`Company: ${candidate.company.trim()}`)
const emails = (candidate.email_addresses ?? [])
.map((e) => e.value?.trim())
.filter((value): value is string => Boolean(value))
if (emails.length > 0) header.push(`Email: ${emails.join(', ')}`)
if (Array.isArray(candidate.tags) && candidate.tags.length > 0) {
header.push(`Tags: ${candidate.tags.join(', ')}`)
}
sections.push(header.join('\n'))
const activity = formatActivityFeed(feed)
if (activity) sections.push(activity)
const scorecardText = formatScorecards(scorecards)
if (scorecardText) sections.push(scorecardText)
return sections.join('\n\n').trim()
}
/**
* Fetches a single candidate by ID. Returns null on 404.
*/
async function fetchCandidate(
accessToken: string,
id: string
): Promise<GreenhouseCandidate | null> {
const response = await fetchWithRetry(`${GREENHOUSE_API_BASE}/candidates/${id}`, {
method: 'GET',
headers: { Authorization: buildAuthHeader(accessToken), Accept: 'application/json' },
})
if (!response.ok) {
if (response.status === 404) return null
throw new Error(`Failed to fetch Greenhouse candidate: ${response.status}`)
}
return (await response.json()) as GreenhouseCandidate
}
/**
* Fetches a candidate's activity feed. Returns an empty feed when the endpoint
* 404s so the candidate's profile header still produces content.
*/
async function fetchActivityFeed(accessToken: string, id: string): Promise<GreenhouseActivityFeed> {
const response = await fetchWithRetry(`${GREENHOUSE_API_BASE}/candidates/${id}/activity_feed`, {
method: 'GET',
headers: { Authorization: buildAuthHeader(accessToken), Accept: 'application/json' },
})
if (!response.ok) {
if (response.status === 404) return {}
throw new Error(`Failed to fetch Greenhouse activity feed: ${response.status}`)
}
return (await response.json()) as GreenhouseActivityFeed
}
/**
* Fetches all scorecards across a candidate's applications. Individual
* application failures are tolerated so partial feedback is still indexed.
*/
async function fetchScorecards(
accessToken: string,
applicationIds: number[]
): Promise<GreenhouseScorecard[]> {
const all: GreenhouseScorecard[] = []
for (const applicationId of applicationIds.slice(0, MAX_APPLICATIONS_FOR_SCORECARDS)) {
try {
const response = await fetchWithRetry(
`${GREENHOUSE_API_BASE}/applications/${applicationId}/scorecards`,
{
method: 'GET',
headers: { Authorization: buildAuthHeader(accessToken), Accept: 'application/json' },
}
)
if (!response.ok) {
if (response.status === 404) continue
throw new Error(`Failed to fetch Greenhouse scorecards: ${response.status}`)
}
const data = (await response.json()) as GreenhouseScorecard[]
if (Array.isArray(data)) all.push(...data)
} catch (error) {
logger.warn('Failed to fetch scorecards for application', {
applicationId,
error: toError(error).message,
})
}
}
return all
}
/**
* Computes the `updated_after` value for an incremental sync, subtracting a small
* overlap from the last sync timestamp. Returns undefined for full syncs.
*/
function computeUpdatedAfter(lastSyncAt: Date | undefined): string | undefined {
if (!lastSyncAt) return undefined
const since = new Date(lastSyncAt.getTime() - INCREMENTAL_OVERLAP_MINUTES * MS_PER_MINUTE)
return since.toISOString()
}
export const greenhouseConnector: ConnectorConfig = {
...greenhouseConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>,
lastSyncAt?: Date
): Promise<ExternalDocumentList> => {
const maxCandidates = sourceConfig.maxCandidates ? Number(sourceConfig.maxCandidates) : 0
const parsedPage = cursor ? Number(cursor) : 1
const page = Number.isFinite(parsedPage) && parsedPage > 0 ? Math.floor(parsedPage) : 1
const updatedAfter = computeUpdatedAfter(lastSyncAt)
const jobId = typeof sourceConfig.jobId === 'string' ? sourceConfig.jobId.trim() : ''
const createdAfter =
typeof sourceConfig.createdAfter === 'string' ? sourceConfig.createdAfter.trim() : ''
const createdBefore =
typeof sourceConfig.createdBefore === 'string' ? sourceConfig.createdBefore.trim() : ''
const queryParams = new URLSearchParams({
per_page: String(CANDIDATES_PER_PAGE),
page: String(page),
})
if (updatedAfter) queryParams.set('updated_after', updatedAfter)
if (jobId) queryParams.set('job_id', jobId)
if (createdAfter) queryParams.set('created_after', createdAfter)
if (createdBefore) queryParams.set('created_before', createdBefore)
const url = `${GREENHOUSE_API_BASE}/candidates?${queryParams.toString()}`
logger.info('Listing Greenhouse candidates', {
page,
perPage: CANDIDATES_PER_PAGE,
incremental: Boolean(updatedAfter),
})
const response = await fetchWithRetry(url, {
method: 'GET',
headers: { Authorization: buildAuthHeader(accessToken), Accept: 'application/json' },
})
if (!response.ok) {
const errorText = await response.text().catch(() => '')
logger.error('Failed to list Greenhouse candidates', {
status: response.status,
error: errorText.slice(0, 500),
})
throw new Error(`Failed to list Greenhouse candidates: ${response.status}`)
}
const data = (await response.json()) as GreenhouseCandidate[]
const candidates = Array.isArray(data) ? data : []
const linkHasNext = hasNextPage(response.headers.get('link'))
const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
let pageCandidates = candidates
if (maxCandidates > 0) {
const remaining = Math.max(0, maxCandidates - prevFetched)
if (pageCandidates.length > remaining) {
pageCandidates = pageCandidates.slice(0, remaining)
}
}
const documents = pageCandidates.map(candidateToStub)
const totalFetched = prevFetched + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = maxCandidates > 0 && totalFetched >= maxCandidates
if (hitLimit && syncContext) syncContext.listingCapped = true
const hasMore = !hitLimit && linkHasNext
return {
documents,
nextCursor: hasMore ? String(page + 1) : undefined,
hasMore,
}
},
getDocument: async (
accessToken: string,
_sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
try {
if (!externalId) return null
const candidate = await fetchCandidate(accessToken, externalId)
if (!candidate) return null
const applicationIds = Array.isArray(candidate.application_ids)
? candidate.application_ids
: []
const [feed, scorecards] = await Promise.all([
fetchActivityFeed(accessToken, externalId),
fetchScorecards(accessToken, applicationIds),
])
const content = formatContent(candidate, feed, scorecards)
if (!content.trim()) return null
return {
externalId: String(candidate.id),
title: candidateDisplayName(candidate),
content,
contentDeferred: false,
mimeType: 'text/plain',
sourceUrl: buildSourceUrl(candidate.id),
contentHash: buildContentHash(candidate.id, candidate.updated_at),
metadata: buildMetadata(candidate),
}
} catch (error) {
logger.warn('Failed to get Greenhouse candidate', {
externalId,
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const maxCandidates = sourceConfig.maxCandidates as string | undefined
if (maxCandidates && (Number.isNaN(Number(maxCandidates)) || Number(maxCandidates) < 0)) {
return { valid: false, error: 'Max candidates must be a non-negative number' }
}
try {
const response = await fetchWithRetry(
`${GREENHOUSE_API_BASE}/candidates?per_page=1`,
{
method: 'GET',
headers: { Authorization: buildAuthHeader(accessToken), Accept: 'application/json' },
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
const errorText = await response.text().catch(() => '')
return {
valid: false,
error: `Greenhouse access failed: ${response.status}${errorText ? `${errorText.slice(0, 200)}` : ''}`,
}
}
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
const textFields = [
'candidateName',
'company',
'title',
'recruiter',
'coordinator',
'source',
] as const
for (const field of textFields) {
const value = metadata[field]
if (typeof value === 'string' && value.trim()) {
result[field] = value.trim()
}
}
if (typeof metadata.applicationCount === 'number' && metadata.applicationCount >= 0) {
result.applicationCount = metadata.applicationCount
}
const dateFields = ['updatedAt', 'lastActivity'] as const
for (const field of dateFields) {
const parsed = parseTagDate(metadata[field])
if (parsed) result[field] = parsed
}
return result
},
}
+1
View File
@@ -0,0 +1 @@
export { greenhouseConnector } from '@/connectors/greenhouse/greenhouse'
+72
View File
@@ -0,0 +1,72 @@
import { GreenhouseIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const greenhouseConnectorMeta: ConnectorMeta = {
id: 'greenhouse',
name: 'Greenhouse',
description: 'Sync candidate activity and interview scorecards from Greenhouse',
version: '1.0.0',
icon: GreenhouseIcon,
auth: {
mode: 'apiKey',
label: 'API Key',
placeholder: 'Enter your Greenhouse Harvest API key',
},
supportsIncrementalSync: true,
configFields: [
{
id: 'maxCandidates',
title: 'Max Candidates',
type: 'short-input',
required: false,
placeholder: 'e.g. 500 (default: unlimited)',
description:
'Cap the number of candidates synced. Leave empty to sync ALL candidates in the organization.',
},
{
id: 'jobId',
title: 'Job ID',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. 123456',
description:
'Sync only candidates who applied to this Greenhouse job. Leave empty to sync candidates across all jobs.',
},
{
id: 'createdAfter',
title: 'Created After',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. 2024-01-01T00:00:00Z',
description:
'Sync only candidates created at or after this ISO 8601 timestamp. Leave empty to sync candidates regardless of creation date.',
},
{
id: 'createdBefore',
title: 'Created Before',
type: 'short-input',
required: false,
mode: 'advanced',
placeholder: 'e.g. 2024-12-31T23:59:59Z',
description:
'Sync only candidates created before this ISO 8601 timestamp. Combine with Created After to backfill a bounded date range.',
},
],
tagDefinitions: [
{ id: 'candidateName', displayName: 'Candidate Name', fieldType: 'text' },
{ id: 'company', displayName: 'Company', fieldType: 'text' },
{ id: 'title', displayName: 'Title', fieldType: 'text' },
{ id: 'recruiter', displayName: 'Recruiter', fieldType: 'text' },
{ id: 'coordinator', displayName: 'Coordinator', fieldType: 'text' },
{ id: 'source', displayName: 'Source', fieldType: 'text' },
{ id: 'applicationCount', displayName: 'Application Count', fieldType: 'number' },
{ id: 'updatedAt', displayName: 'Last Updated', fieldType: 'date' },
{ id: 'lastActivity', displayName: 'Last Activity', fieldType: 'date' },
],
}
+351
View File
@@ -0,0 +1,351 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { hubspotConnectorMeta } from '@/connectors/hubspot/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { parseTagDate } from '@/connectors/utils'
const logger = createLogger('HubSpotConnector')
const BASE_URL = 'https://api.hubapi.com'
const PAGE_SIZE = 100
/** Properties to fetch per object type. */
const OBJECT_PROPERTIES: Record<string, string[]> = {
contacts: [
'firstname',
'lastname',
'email',
'phone',
'company',
'jobtitle',
'lifecyclestage',
'hs_lead_status',
'hubspot_owner_id',
'createdate',
'lastmodifieddate',
],
companies: [
'name',
'domain',
'industry',
'description',
'phone',
'city',
'state',
'country',
'numberofemployees',
'annualrevenue',
'hubspot_owner_id',
'createdate',
'hs_lastmodifieddate',
],
deals: [
'dealname',
'amount',
'dealstage',
'pipeline',
'closedate',
'hubspot_owner_id',
'createdate',
'hs_lastmodifieddate',
],
tickets: [
'subject',
'content',
'hs_pipeline',
'hs_pipeline_stage',
'hs_ticket_priority',
'hubspot_owner_id',
'createdate',
'hs_lastmodifieddate',
],
} as const
/**
* Fetches the HubSpot portal ID for the authenticated account.
* Caches the result in syncContext to avoid repeated calls.
*/
async function getPortalId(
accessToken: string,
syncContext?: Record<string, unknown>
): Promise<string> {
if (syncContext?.portalId) {
return syncContext.portalId as string
}
const response = await fetchWithRetry(`${BASE_URL}/account-info/v3/details`, {
method: 'GET',
headers: {
Accept: 'application/json',
Authorization: `Bearer ${accessToken}`,
},
})
if (!response.ok) {
const errorText = await response.text()
throw new Error(`Failed to fetch HubSpot portal ID: ${response.status} - ${errorText}`)
}
const data = await response.json()
const portalId = String(data.portalId)
if (syncContext) {
syncContext.portalId = portalId
}
return portalId
}
/**
* Builds the document title for a CRM record based on its object type.
*/
function buildRecordTitle(objectType: string, properties: Record<string, string | null>): string {
switch (objectType) {
case 'contacts': {
const first = properties.firstname || ''
const last = properties.lastname || ''
const name = `${first} ${last}`.trim()
return name || properties.email || 'Unnamed Contact'
}
case 'companies':
return properties.name || properties.domain || 'Unnamed Company'
case 'deals':
return properties.dealname || 'Unnamed Deal'
case 'tickets':
return properties.subject || 'Unnamed Ticket'
default:
return `Record ${properties.hs_object_id || 'Unknown'}`
}
}
/**
* Builds a plain-text representation of a CRM record's properties for indexing.
*/
function buildRecordContent(objectType: string, properties: Record<string, string | null>): string {
const parts: string[] = []
const title = buildRecordTitle(objectType, properties)
parts.push(title)
for (const [key, value] of Object.entries(properties)) {
if (value && key !== 'hs_object_id') {
const label = key.replace(/_/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase())
parts.push(`${label}: ${value}`)
}
}
return parts.join('\n').trim()
}
/**
* Converts a HubSpot CRM record to an ExternalDocument.
*/
function recordToDocument(
record: Record<string, unknown>,
objectType: string,
portalId: string
): ExternalDocument {
const id = record.id as string
const properties = (record.properties || {}) as Record<string, string | null>
const content = buildRecordContent(objectType, properties)
const title = buildRecordTitle(objectType, properties)
const lastModified =
properties.lastmodifieddate || properties.hs_lastmodifieddate || properties.createdate
return {
externalId: id,
title,
content,
mimeType: 'text/plain',
sourceUrl: `https://app.hubspot.com/contacts/${portalId}/record/${objectType}/${id}`,
contentHash: `hubspot:${id}:${lastModified ?? ''}`,
metadata: {
objectType,
owner: properties.hubspot_owner_id || undefined,
lastModified: lastModified || undefined,
pipeline: properties.pipeline || properties.hs_pipeline || undefined,
},
}
}
export const hubspotConnector: ConnectorConfig = {
...hubspotConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const objectType = sourceConfig.objectType as string
const maxRecords = sourceConfig.maxRecords ? Number(sourceConfig.maxRecords) : 0
const properties = OBJECT_PROPERTIES[objectType] || []
const portalId = await getPortalId(accessToken, syncContext)
const sortProperty = objectType === 'contacts' ? 'lastmodifieddate' : 'hs_lastmodifieddate'
const searchBody: Record<string, unknown> = {
properties,
sorts: [{ propertyName: sortProperty, direction: 'DESCENDING' }],
limit: PAGE_SIZE,
}
if (cursor) {
searchBody.after = cursor
}
logger.info(`Listing HubSpot ${objectType}`, { cursor })
const response = await fetchWithRetry(`${BASE_URL}/crm/v3/objects/${objectType}/search`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Accept: 'application/json',
Authorization: `Bearer ${accessToken}`,
},
body: JSON.stringify(searchBody),
})
if (!response.ok) {
const errorText = await response.text()
logger.error(`Failed to list HubSpot ${objectType}`, {
status: response.status,
error: errorText,
})
throw new Error(`Failed to list HubSpot ${objectType}: ${response.status}`)
}
const data = await response.json()
const results = (data.results || []) as Record<string, unknown>[]
const paging = data.paging as { next?: { after?: string } } | undefined
const nextCursor = paging?.next?.after
const documents: ExternalDocument[] = results.map((record) =>
recordToDocument(record, objectType, portalId)
)
const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
if (maxRecords > 0) {
const remaining = maxRecords - previouslyFetched
if (documents.length > remaining) {
documents.splice(remaining)
}
}
const totalFetched = previouslyFetched + documents.length
if (syncContext) {
syncContext.totalDocsFetched = totalFetched
}
const hasMore = Boolean(nextCursor) && (maxRecords <= 0 || totalFetched < maxRecords)
return {
documents,
nextCursor: hasMore ? nextCursor : undefined,
hasMore,
}
},
getDocument: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
externalId: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocument | null> => {
const objectType = sourceConfig.objectType as string
const properties = OBJECT_PROPERTIES[objectType] || []
let portalId = syncContext?.portalId as string | undefined
if (!portalId) {
portalId = await getPortalId(accessToken)
if (syncContext) syncContext.portalId = portalId
}
const params = new URLSearchParams()
for (const prop of properties) {
params.append('properties', prop)
}
const url = `${BASE_URL}/crm/v3/objects/${objectType}/${externalId}?${params.toString()}`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/json',
Authorization: `Bearer ${accessToken}`,
},
})
if (!response.ok) {
if (response.status === 404) return null
throw new Error(`Failed to get HubSpot ${objectType} record: ${response.status}`)
}
const record = await response.json()
return recordToDocument(record as Record<string, unknown>, objectType, portalId)
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const objectType = sourceConfig.objectType as string
if (!objectType) {
return { valid: false, error: 'Object type is required' }
}
if (!OBJECT_PROPERTIES[objectType]) {
return { valid: false, error: `Unsupported object type: ${objectType}` }
}
const maxRecords = sourceConfig.maxRecords as string | undefined
if (maxRecords && (Number.isNaN(Number(maxRecords)) || Number(maxRecords) <= 0)) {
return { valid: false, error: 'Max records must be a positive number' }
}
try {
const response = await fetchWithRetry(
`${BASE_URL}/crm/v3/objects/${objectType}?limit=1`,
{
method: 'GET',
headers: {
Accept: 'application/json',
Authorization: `Bearer ${accessToken}`,
},
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
const errorText = await response.text()
return {
valid: false,
error: `Failed to access HubSpot ${objectType}: ${response.status} - ${errorText}`,
}
}
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.objectType === 'string') result.objectType = metadata.objectType
if (typeof metadata.owner === 'string') result.owner = metadata.owner
const lastModified = parseTagDate(metadata.lastModified)
if (lastModified) result.lastModified = lastModified
if (typeof metadata.pipeline === 'string') result.pipeline = metadata.pipeline
return result
},
}
+1
View File
@@ -0,0 +1 @@
export { hubspotConnector } from '@/connectors/hubspot/hubspot'
+50
View File
@@ -0,0 +1,50 @@
import { HubspotIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const hubspotConnectorMeta: ConnectorMeta = {
id: 'hubspot',
name: 'HubSpot',
description: 'Sync CRM records from HubSpot',
version: '1.0.0',
icon: HubspotIcon,
auth: {
mode: 'oauth',
provider: 'hubspot',
requiredScopes: [
'crm.objects.contacts.read',
'crm.objects.companies.read',
'crm.objects.deals.read',
'tickets',
],
},
configFields: [
{
id: 'objectType',
title: 'Object Type',
type: 'dropdown',
required: true,
options: [
{ label: 'Contacts', id: 'contacts' },
{ label: 'Companies', id: 'companies' },
{ label: 'Deals', id: 'deals' },
{ label: 'Tickets', id: 'tickets' },
],
},
{
id: 'maxRecords',
title: 'Max Records',
type: 'short-input',
required: false,
placeholder: 'e.g. 500 (default: unlimited)',
},
],
tagDefinitions: [
{ id: 'objectType', displayName: 'Object Type', fieldType: 'text' },
{ id: 'owner', displayName: 'Owner', fieldType: 'text' },
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
{ id: 'pipeline', displayName: 'Pipeline', fieldType: 'text' },
],
}
@@ -0,0 +1,553 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { incidentioConnectorMeta } from '@/connectors/incidentio/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { htmlToPlainText, parseTagDate } from '@/connectors/utils'
const logger = createLogger('IncidentioConnector')
const INCIDENTIO_API_BASE = 'https://api.incident.io'
const PAGE_SIZE = 100
/** Cap incident updates fetched per document so a noisy incident can't blow the time budget. */
const MAX_UPDATES_PER_INCIDENT = 100
interface IncidentioNamedRef {
id?: string
name?: string
/** Present on incident_status: the stable status category enum (triage/live/closed/...). */
category?: string
rank?: number
}
interface IncidentioCustomFieldValue {
value_text?: string
value_link?: string
value_numeric?: string
value_option?: { value?: string }
value_catalog_entry?: { name?: string }
}
interface IncidentioCustomFieldEntry {
custom_field?: { id?: string; name?: string; field_type?: string }
values?: IncidentioCustomFieldValue[]
}
interface IncidentioRoleAssignment {
role?: { id?: string; name?: string; role_type?: string }
assignee?: { id?: string; name?: string; email?: string }
}
interface IncidentioTimestampValue {
incident_timestamp?: { id?: string; name?: string; rank?: number }
/**
* The v2 API nests the timestamp string under `value.value` (an object), not a
* flat string. A flat string is tolerated defensively for older shapes.
*/
value?: { value?: string } | string
}
interface IncidentioIncident {
id: string
reference?: string
name?: string
summary?: string
mode?: string
visibility?: string
permalink?: string
call_url?: string
created_at?: string
updated_at?: string
slack_channel_id?: string
slack_channel_name?: string
severity?: IncidentioNamedRef
incident_status?: IncidentioNamedRef
incident_type?: IncidentioNamedRef
custom_field_entries?: IncidentioCustomFieldEntry[]
incident_role_assignments?: IncidentioRoleAssignment[]
incident_timestamp_values?: IncidentioTimestampValue[]
}
interface IncidentioPaginationMeta {
after?: string
page_size?: number
total_record_count?: number
}
interface IncidentioIncidentsListResponse {
incidents?: IncidentioIncident[]
pagination_meta?: IncidentioPaginationMeta
}
interface IncidentioIncidentShowResponse {
incident?: IncidentioIncident
}
/**
* incident.io ActorV2: the entity behind an action, used for incident update
* authors. The v2 API documents this as an ActorV2 with exactly one of the
* nested `user`/`api_key`/`workflow`/`alert` variants populated. A flat
* `{ name, email }` shape is also tolerated defensively; we read whichever is
* present so the author resolves regardless of which form the API returns.
*/
interface IncidentioActor {
user?: { id?: string; name?: string; email?: string }
api_key?: { id?: string; name?: string }
alert?: { id?: string; title?: string }
workflow?: { id?: string; name?: string }
name?: string
email?: string
}
interface IncidentioUpdate {
id?: string
message?: string
new_severity?: IncidentioNamedRef
/**
* The status change on an update. The v2 API documents `new_incident_status`,
* but a flat `new_status` shape also appears in the wild; both are read.
*/
new_incident_status?: IncidentioNamedRef
new_status?: IncidentioNamedRef
updater?: IncidentioActor
/** Some responses use `author` instead of `updater`; both are read. */
author?: IncidentioActor
created_at?: string
}
/**
* Resolves a human-readable display name for an update author, covering the
* ActorV2 variants and the flat `{ name, email }` shape.
*/
function actorName(actor: IncidentioActor | undefined): string | undefined {
if (!actor) return undefined
return (
actor.user?.name ||
actor.user?.email ||
actor.api_key?.name ||
actor.workflow?.name ||
actor.alert?.title ||
actor.name ||
actor.email ||
undefined
)
}
interface IncidentioUpdatesListResponse {
incident_updates?: IncidentioUpdate[]
pagination_meta?: IncidentioPaginationMeta
}
/**
* Builds the metadata-based content hash for an incident.
*
* Uses only the incident `updated_at` timestamp, which incident.io bumps whenever any
* field, role assignment, custom field, or status changes. This guarantees the hash is
* identical whether produced from the list stub or the fully-hydrated getDocument result,
* so the sync engine can detect changes without downloading content.
*/
function buildContentHash(incident: IncidentioIncident): string {
return `incidentio:${incident.id}:${incident.updated_at ?? ''}`
}
/**
* Builds the public URL for an incident, preferring the API-provided permalink.
*/
function buildSourceUrl(incident: IncidentioIncident): string | undefined {
return incident.permalink || undefined
}
/**
* Derives the document title from the incident reference and name.
*/
function buildTitle(incident: IncidentioIncident): string {
const name = incident.name?.trim()
const reference = incident.reference?.trim()
if (reference && name) return `${reference}: ${name}`
return name || reference || 'Untitled Incident'
}
/**
* Collects the source-specific metadata fed to mapTags. Shared between the list stub
* and getDocument so tag values stay consistent regardless of which path produced the doc.
*/
function buildMetadata(incident: IncidentioIncident): Record<string, unknown> {
const lead = incident.incident_role_assignments?.find(
(assignment) => assignment.role?.role_type === 'lead'
)
const reporter = incident.incident_role_assignments?.find(
(assignment) => assignment.role?.role_type === 'reporter'
)
const reportedBy = (reporter ?? lead)?.assignee?.name
return {
reference: incident.reference,
status: incident.incident_status?.name,
statusCategory: incident.incident_status?.category,
severity: incident.severity?.name,
incidentType: incident.incident_type?.name,
mode: incident.mode,
visibility: incident.visibility,
incidentDate: incident.created_at,
reportedBy,
}
}
/**
* Renders a single custom field value to a human-readable string, covering each of
* incident.io's value variants (text, link, numeric, select option, catalog entry).
*/
function renderCustomFieldValue(value: IncidentioCustomFieldValue): string | undefined {
if (value.value_text) return htmlToPlainText(value.value_text)
if (value.value_link) return value.value_link
if (value.value_numeric) return value.value_numeric
if (value.value_option?.value) return value.value_option.value
if (value.value_catalog_entry?.name) return value.value_catalog_entry.name
return undefined
}
/**
* Formats the incident, its custom fields, role assignments, timeline, and status
* updates into a single plain-text document. HTML in summary / custom field text is
* stripped via htmlToPlainText.
*/
function formatIncidentContent(incident: IncidentioIncident, updates: IncidentioUpdate[]): string {
const parts: string[] = []
parts.push(`Incident: ${buildTitle(incident)}`)
if (incident.incident_status?.name) parts.push(`Status: ${incident.incident_status.name}`)
if (incident.severity?.name) parts.push(`Severity: ${incident.severity.name}`)
if (incident.incident_type?.name) parts.push(`Type: ${incident.incident_type.name}`)
if (incident.created_at) parts.push(`Reported: ${incident.created_at}`)
if (incident.summary?.trim()) {
parts.push('')
parts.push('--- Summary ---')
parts.push(htmlToPlainText(incident.summary))
}
const roleLines = (incident.incident_role_assignments ?? [])
.map((assignment) => {
const role = assignment.role?.name
const assignee = assignment.assignee?.name || assignment.assignee?.email
if (!role || !assignee) return undefined
return `${role}: ${assignee}`
})
.filter((line): line is string => Boolean(line))
if (roleLines.length > 0) {
parts.push('')
parts.push('--- Roles ---')
parts.push(...roleLines)
}
const customFieldLines = (incident.custom_field_entries ?? [])
.map((entry) => {
const name = entry.custom_field?.name
if (!name) return undefined
const rendered = (entry.values ?? [])
.map(renderCustomFieldValue)
.filter((v): v is string => Boolean(v))
if (rendered.length === 0) return undefined
return `${name}: ${rendered.join(', ')}`
})
.filter((line): line is string => Boolean(line))
if (customFieldLines.length > 0) {
parts.push('')
parts.push('--- Custom Fields ---')
parts.push(...customFieldLines)
}
const timestampLines = (incident.incident_timestamp_values ?? [])
.map((entry) => {
const name = entry.incident_timestamp?.name
const value = typeof entry.value === 'string' ? entry.value : entry.value?.value
if (!name || !value) return undefined
return `${name}: ${value}`
})
.filter((line): line is string => Boolean(line))
if (timestampLines.length > 0) {
parts.push('')
parts.push('--- Timeline ---')
parts.push(...timestampLines)
}
if (updates.length > 0) {
parts.push('')
parts.push('--- Updates ---')
for (const update of updates) {
const segments: string[] = []
if (update.created_at) segments.push(`[${update.created_at}]`)
const author = actorName(update.updater ?? update.author)
if (author) segments.push(author)
const changes: string[] = []
const newStatusName = update.new_incident_status?.name ?? update.new_status?.name
if (newStatusName) {
changes.push(`status → ${newStatusName}`)
}
if (update.new_severity?.name) changes.push(`severity → ${update.new_severity.name}`)
const message = update.message ? htmlToPlainText(update.message) : ''
const tail = [changes.join(', '), message].filter(Boolean).join(' — ')
const line = [segments.join(' '), tail].filter(Boolean).join(': ')
if (line.trim()) parts.push(line)
}
}
return parts.join('\n').trim()
}
/**
* Creates a lightweight document stub from a list entry. No API calls — content is
* deferred to getDocument and only fetched for new or changed incidents.
*/
function incidentToStub(incident: IncidentioIncident): ExternalDocument {
return {
externalId: incident.id,
title: buildTitle(incident),
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: buildSourceUrl(incident),
contentHash: buildContentHash(incident),
metadata: buildMetadata(incident),
}
}
/**
* Fetches all status updates for an incident, following the `after` cursor and capping
* the total to keep getDocument bounded for very long-running incidents.
*/
async function fetchIncidentUpdates(
accessToken: string,
incidentId: string
): Promise<IncidentioUpdate[]> {
const updates: IncidentioUpdate[] = []
let after: string | undefined
while (updates.length < MAX_UPDATES_PER_INCIDENT) {
const url = new URL(`${INCIDENTIO_API_BASE}/v2/incident_updates`)
url.searchParams.set('incident_id', incidentId)
url.searchParams.set('page_size', String(PAGE_SIZE))
if (after) url.searchParams.set('after', after)
const response = await fetchWithRetry(url.toString(), {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) {
logger.warn('Failed to fetch incident updates', { incidentId, status: response.status })
break
}
const data = (await response.json()) as IncidentioUpdatesListResponse
const page = data.incident_updates ?? []
updates.push(...page)
after = data.pagination_meta?.after?.trim() || undefined
if (!after || page.length === 0) break
}
return updates.slice(0, MAX_UPDATES_PER_INCIDENT)
}
export const incidentioConnector: ConnectorConfig = {
...incidentioConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>,
lastSyncAt?: Date
): Promise<ExternalDocumentList> => {
const maxIncidents = sourceConfig.maxIncidents ? Number(sourceConfig.maxIncidents) : 0
const statusCategory =
typeof sourceConfig.statusCategory === 'string' ? sourceConfig.statusCategory.trim() : ''
const mode = typeof sourceConfig.mode === 'string' ? sourceConfig.mode.trim() : ''
const url = new URL(`${INCIDENTIO_API_BASE}/v2/incidents`)
url.searchParams.set('page_size', String(PAGE_SIZE))
url.searchParams.set('sort_by', 'created_at_oldest_first')
if (cursor) url.searchParams.set('after', cursor)
if (lastSyncAt) url.searchParams.set('updated_at[gte]', lastSyncAt.toISOString())
if (statusCategory) url.searchParams.set('status_category[one_of]', statusCategory)
if (mode) url.searchParams.set('mode[one_of]', mode)
logger.info('Listing incident.io incidents', {
cursor: cursor ?? 'initial',
incremental: Boolean(lastSyncAt),
maxIncidents,
})
const response = await fetchWithRetry(url.toString(), {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) {
const errorText = await response.text().catch(() => '')
logger.error('Failed to list incident.io incidents', {
status: response.status,
error: errorText.slice(0, 500),
})
throw new Error(`Failed to list incident.io incidents: ${response.status}`)
}
const data = (await response.json()) as IncidentioIncidentsListResponse
const incidents = (data.incidents ?? []).filter((incident) => Boolean(incident.id))
const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
let documents = incidents.map(incidentToStub)
if (maxIncidents > 0) {
const remaining = Math.max(0, maxIncidents - prevFetched)
if (documents.length > remaining) {
documents = documents.slice(0, remaining)
}
}
const totalFetched = prevFetched + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = maxIncidents > 0 && totalFetched >= maxIncidents
if (hitLimit && syncContext) syncContext.listingCapped = true
const after = data.pagination_meta?.after?.trim() || undefined
const hasMore = !hitLimit && Boolean(after)
return {
documents,
nextCursor: hasMore ? after : undefined,
hasMore,
}
},
getDocument: async (
accessToken: string,
_sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
try {
if (!externalId) return null
const url = `${INCIDENTIO_API_BASE}/v2/incidents/${encodeURIComponent(externalId)}`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
})
if (!response.ok) {
if (response.status === 404 || response.status === 410) return null
throw new Error(`Failed to fetch incident.io incident: ${response.status}`)
}
const data = (await response.json()) as IncidentioIncidentShowResponse
const incident = data.incident
if (!incident?.id) return null
const updates = await fetchIncidentUpdates(accessToken, incident.id)
const content = formatIncidentContent(incident, updates)
if (!content.trim()) return null
return {
externalId: incident.id,
title: buildTitle(incident),
content,
contentDeferred: false,
mimeType: 'text/plain',
sourceUrl: buildSourceUrl(incident),
contentHash: buildContentHash(incident),
metadata: buildMetadata(incident),
}
} catch (error) {
logger.warn('Failed to get incident.io incident', {
externalId,
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const maxIncidents = sourceConfig.maxIncidents as string | undefined
if (maxIncidents && (Number.isNaN(Number(maxIncidents)) || Number(maxIncidents) < 0)) {
return { valid: false, error: 'Max incidents must be a non-negative number' }
}
try {
const response = await fetchWithRetry(
`${INCIDENTIO_API_BASE}/v2/incidents?page_size=1`,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
const errorText = await response.text().catch(() => '')
return {
valid: false,
error: `incident.io access failed: ${response.status}${errorText ? `${errorText.slice(0, 200)}` : ''}`,
}
}
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.status === 'string' && metadata.status.trim()) {
result.status = metadata.status
}
if (typeof metadata.statusCategory === 'string' && metadata.statusCategory.trim()) {
result.statusCategory = metadata.statusCategory
}
if (typeof metadata.severity === 'string' && metadata.severity.trim()) {
result.severity = metadata.severity
}
if (typeof metadata.incidentType === 'string' && metadata.incidentType.trim()) {
result.incidentType = metadata.incidentType
}
if (typeof metadata.mode === 'string' && metadata.mode.trim()) {
result.mode = metadata.mode
}
if (typeof metadata.visibility === 'string' && metadata.visibility.trim()) {
result.visibility = metadata.visibility
}
const incidentDate = parseTagDate(metadata.incidentDate)
if (incidentDate) result.incidentDate = incidentDate
if (typeof metadata.reportedBy === 'string' && metadata.reportedBy.trim()) {
result.reportedBy = metadata.reportedBy
}
return result
},
}
+1
View File
@@ -0,0 +1 @@
export { incidentioConnector } from '@/connectors/incidentio/incidentio'
+76
View File
@@ -0,0 +1,76 @@
import { IncidentioIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const incidentioConnectorMeta: ConnectorMeta = {
id: 'incidentio',
name: 'incident.io',
description: 'Sync incidents and postmortems from incident.io into your knowledge base',
version: '1.0.0',
icon: IncidentioIcon,
auth: {
mode: 'apiKey',
label: 'API Key',
placeholder: 'Enter your incident.io API key',
},
supportsIncrementalSync: true,
configFields: [
{
id: 'statusCategory',
title: 'Status Category',
type: 'dropdown',
required: false,
mode: 'advanced',
options: [
{ label: 'All', id: '' },
{ label: 'Live (active)', id: 'live' },
{ label: 'Paused', id: 'paused' },
{ label: 'Closed', id: 'closed' },
{ label: 'Triage', id: 'triage' },
{ label: 'Learning (post-incident)', id: 'learning' },
{ label: 'Declined', id: 'declined' },
{ label: 'Merged', id: 'merged' },
{ label: 'Canceled', id: 'canceled' },
],
description:
'Only sync incidents in this status category. Leave as All to sync every category.',
},
{
id: 'mode',
title: 'Mode',
type: 'dropdown',
required: false,
mode: 'advanced',
options: [
{ label: 'All', id: '' },
{ label: 'Standard (real incidents)', id: 'standard' },
{ label: 'Retrospective', id: 'retrospective' },
{ label: 'Test', id: 'test' },
{ label: 'Tutorial', id: 'tutorial' },
],
description:
'Only sync incidents of this mode. Use Standard to exclude test/tutorial incidents.',
},
{
id: 'maxIncidents',
title: 'Max Incidents',
type: 'short-input',
required: false,
placeholder: 'e.g. 200 (default: unlimited)',
description: 'Cap the number of incidents synced. Leave empty to sync all incidents.',
},
],
tagDefinitions: [
{ id: 'status', displayName: 'Status', fieldType: 'text' },
{ id: 'statusCategory', displayName: 'Status Category', fieldType: 'text' },
{ id: 'severity', displayName: 'Severity', fieldType: 'text' },
{ id: 'incidentType', displayName: 'Incident Type', fieldType: 'text' },
{ id: 'mode', displayName: 'Mode', fieldType: 'text' },
{ id: 'visibility', displayName: 'Visibility', fieldType: 'text' },
{ id: 'incidentDate', displayName: 'Incident Date', fieldType: 'date' },
{ id: 'reportedBy', displayName: 'Reported By', fieldType: 'text' },
],
}
+1
View File
@@ -0,0 +1 @@
export { intercomConnector } from '@/connectors/intercom/intercom'
+491
View File
@@ -0,0 +1,491 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { z } from 'zod'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { DEFAULT_MAX_ITEMS, intercomConnectorMeta } from '@/connectors/intercom/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { htmlToPlainText, parseTagDate } from '@/connectors/utils'
const logger = createLogger('IntercomConnector')
const INTERCOM_API_BASE = 'https://api.intercom.io'
const ARTICLES_PER_PAGE = 50
const CONVERSATIONS_PER_PAGE = 50
const IntercomAuthorSchema = z
.object({
type: z.string(),
id: z.string(),
name: z.string().optional(),
})
.passthrough()
const IntercomArticleSchema = z
.object({
type: z.string().optional(),
id: z.string(),
title: z.string().optional(),
description: z.string().nullable().optional(),
body: z.string().nullable().optional(),
author_id: z.union([z.number(), z.string()]).optional(),
state: z.string(),
created_at: z.number(),
updated_at: z.number(),
url: z.string().optional(),
parent_id: z.number().nullable().optional(),
parent_type: z.string().nullable().optional(),
})
.passthrough()
type IntercomArticle = z.infer<typeof IntercomArticleSchema>
const IntercomConversationPartSchema = z
.object({
type: z.string().optional(),
id: z.string(),
part_type: z.string().optional(),
body: z.string().nullable().optional(),
created_at: z.number(),
author: IntercomAuthorSchema.optional(),
})
.passthrough()
const IntercomConversationSchema = z
.object({
type: z.string().optional(),
id: z.string(),
created_at: z.number(),
updated_at: z.number(),
title: z.string().nullable().optional(),
state: z.string(),
open: z.boolean().optional(),
source: z
.object({
type: z.string(),
id: z.string(),
subject: z.string().optional(),
body: z.string().nullable().optional(),
author: IntercomAuthorSchema,
delivered_as: z.string().optional(),
})
.passthrough()
.optional(),
tags: z
.object({
type: z.string().optional(),
tags: z
.array(
z
.object({
id: z.string(),
name: z.string(),
})
.passthrough()
)
.default([]),
})
.passthrough()
.optional(),
conversation_parts: z
.object({
type: z.string(),
conversation_parts: z.array(IntercomConversationPartSchema),
total_count: z.number(),
})
.passthrough()
.optional(),
})
.passthrough()
type IntercomConversation = z.infer<typeof IntercomConversationSchema>
/**
* Makes a GET request to the Intercom API with Bearer token auth.
*/
async function intercomApiGet(
path: string,
accessToken: string,
params?: Record<string, string>,
retryOptions?: Parameters<typeof fetchWithRetry>[2]
): Promise<Record<string, unknown>> {
const url = new URL(`${INTERCOM_API_BASE}${path}`)
if (params) {
for (const [key, value] of Object.entries(params)) {
url.searchParams.set(key, value)
}
}
const response = await fetchWithRetry(
url.toString(),
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
'Intercom-Version': '2.11',
},
},
retryOptions
)
if (!response.ok) {
const errorBody = await response.text().catch(() => '')
throw new Error(`Intercom API HTTP error ${response.status}: ${errorBody}`)
}
return (await response.json()) as Record<string, unknown>
}
/**
* Fetches all articles from Intercom, respecting page-based pagination and max items cap.
*/
async function fetchArticles(
accessToken: string,
maxItems: number,
stateFilter: string
): Promise<IntercomArticle[]> {
const allArticles: IntercomArticle[] = []
let page = 1
while (allArticles.length < maxItems) {
const data = await intercomApiGet('/articles', accessToken, {
page: String(page),
per_page: String(ARTICLES_PER_PAGE),
})
const articles = z.array(IntercomArticleSchema).parse(data.data ?? [])
if (articles.length === 0) break
for (const article of articles) {
if (stateFilter !== 'all' && article.state !== stateFilter) continue
allArticles.push(article)
if (allArticles.length >= maxItems) break
}
const pages = data.pages as { total_pages?: number } | null
if (!pages?.total_pages || page >= pages.total_pages) break
page++
}
return allArticles
}
/**
* Fetches conversations from Intercom using cursor-based pagination.
*/
async function fetchConversations(
accessToken: string,
maxItems: number,
stateFilter: string
): Promise<IntercomConversation[]> {
const allConversations: IntercomConversation[] = []
let startingAfter: string | undefined
while (allConversations.length < maxItems) {
const params: Record<string, string> = {
per_page: String(Math.min(CONVERSATIONS_PER_PAGE, 150)),
}
if (startingAfter) {
params.starting_after = startingAfter
}
const data = await intercomApiGet('/conversations', accessToken, params)
const conversations = z.array(IntercomConversationSchema).parse(data.conversations ?? [])
if (conversations.length === 0) break
for (const conversation of conversations) {
if (stateFilter !== 'all' && conversation.state !== stateFilter) continue
allConversations.push(conversation)
if (allConversations.length >= maxItems) break
}
const pages = data.pages as { next?: { starting_after?: string } } | null
const nextCursor = pages?.next?.starting_after
if (!nextCursor) break
startingAfter = nextCursor
}
return allConversations
}
/**
* Fetches the full conversation with conversation_parts included.
*/
async function fetchConversationDetail(
accessToken: string,
conversationId: string
): Promise<IntercomConversation> {
const data = await intercomApiGet(`/conversations/${conversationId}`, accessToken)
return IntercomConversationSchema.parse(data)
}
/**
* Converts a conversation (with parts) into a plain text document.
*/
function formatConversation(conversation: IntercomConversation): string {
const lines: string[] = []
if (conversation.title) {
lines.push(`Subject: ${conversation.title}`)
}
const source = conversation.source
const sourceBody = source?.body
if (sourceBody) {
const authorName = source.author?.name || source.author?.type || 'unknown'
const timestamp = new Date(conversation.created_at * 1000).toISOString()
lines.push(`[${timestamp}] ${authorName}: ${htmlToPlainText(sourceBody)}`)
}
const parts = conversation.conversation_parts?.conversation_parts || []
for (const part of parts) {
if (!part.body) continue
const authorName = part.author?.name || part.author?.type || 'unknown'
const timestamp = new Date(part.created_at * 1000).toISOString()
lines.push(`[${timestamp}] ${authorName}: ${htmlToPlainText(part.body)}`)
}
return lines.join('\n')
}
/**
* Converts an article to a plain text document.
*/
function formatArticle(article: IntercomArticle): string {
const parts: string[] = []
if (article.title) {
parts.push(article.title)
}
if (article.description) {
parts.push(article.description)
}
if (article.body) {
parts.push(htmlToPlainText(article.body))
}
return parts.join('\n\n')
}
export const intercomConnector: ConnectorConfig = {
...intercomConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
_cursor?: string,
_syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const contentType = (sourceConfig.contentType as string) || 'articles'
const articleState = (sourceConfig.articleState as string) || 'published'
const conversationState = (sourceConfig.conversationState as string) || 'all'
const maxItems = sourceConfig.maxItems ? Number(sourceConfig.maxItems) : DEFAULT_MAX_ITEMS
const documents: ExternalDocument[] = []
if (contentType === 'articles' || contentType === 'both') {
logger.info('Fetching Intercom articles', { articleState, maxItems })
const articles = await fetchArticles(accessToken, maxItems, articleState)
for (const article of articles) {
const content = formatArticle(article)
if (!content.trim()) continue
const updatedAt = new Date(article.updated_at * 1000).toISOString()
documents.push({
externalId: `article-${article.id}`,
title: article.title || `Article ${article.id}`,
content,
mimeType: 'text/plain',
sourceUrl: `https://app.intercom.com/a/apps/_/articles/articles/${article.id}/show`,
contentHash: `intercom:article-${article.id}:${article.updated_at}`,
metadata: {
type: 'article',
state: article.state,
authorId: String(article.author_id),
updatedAt,
createdAt: new Date(article.created_at * 1000).toISOString(),
},
})
}
logger.info('Fetched Intercom articles', { count: articles.length })
}
if (contentType === 'conversations' || contentType === 'both') {
logger.info('Fetching Intercom conversations', { conversationState, maxItems })
const conversations = await fetchConversations(accessToken, maxItems, conversationState)
for (const conversation of conversations) {
const updatedAt = new Date(conversation.updated_at * 1000).toISOString()
const tags = conversation.tags?.tags?.map((t) => t.name) || []
documents.push({
externalId: `conversation-${conversation.id}`,
title: conversation.title || `Conversation #${conversation.id}`,
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: `https://app.intercom.com/a/apps/_/inbox/inbox/all/conversations/${conversation.id}`,
contentHash: `intercom:conversation-${conversation.id}:${conversation.updated_at}`,
metadata: {
type: 'conversation',
state: conversation.state,
tags: tags.join(', '),
updatedAt,
createdAt: new Date(conversation.created_at * 1000).toISOString(),
},
})
}
logger.info('Fetched Intercom conversations', { count: conversations.length })
}
return { documents, hasMore: false }
},
getDocument: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
try {
if (externalId.startsWith('article-')) {
const articleId = externalId.replace('article-', '')
const data = await intercomApiGet(`/articles/${articleId}`, accessToken)
const article = IntercomArticleSchema.parse(data)
const content = formatArticle(article)
if (!content.trim()) return null
const updatedAt = new Date(article.updated_at * 1000).toISOString()
return {
externalId,
title: article.title || `Article ${article.id}`,
content,
mimeType: 'text/plain',
sourceUrl: `https://app.intercom.com/a/apps/_/articles/articles/${article.id}/show`,
contentHash: `intercom:article-${article.id}:${article.updated_at}`,
metadata: {
type: 'article',
state: article.state,
authorId: String(article.author_id),
updatedAt,
createdAt: new Date(article.created_at * 1000).toISOString(),
},
}
}
if (externalId.startsWith('conversation-')) {
const conversationId = externalId.replace('conversation-', '')
const detail = await fetchConversationDetail(accessToken, conversationId)
const content = formatConversation(detail)
if (!content.trim()) return null
const updatedAt = new Date(detail.updated_at * 1000).toISOString()
const tags = detail.tags?.tags?.map((t) => t.name) || []
return {
externalId,
title: detail.title || `Conversation #${detail.id}`,
content,
contentDeferred: false,
mimeType: 'text/plain',
sourceUrl: `https://app.intercom.com/a/apps/_/inbox/inbox/all/conversations/${detail.id}`,
contentHash: `intercom:conversation-${detail.id}:${detail.updated_at}`,
metadata: {
type: 'conversation',
state: detail.state,
tags: tags.join(', '),
updatedAt,
createdAt: new Date(detail.created_at * 1000).toISOString(),
messageCount: (detail.conversation_parts?.total_count ?? 0) + 1,
},
}
}
logger.warn('Unknown external ID format', { externalId })
return null
} catch (error) {
logger.warn('Failed to get Intercom document', {
externalId,
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const contentType = sourceConfig.contentType as string | undefined
const maxItems = sourceConfig.maxItems as string | undefined
if (!contentType) {
return { valid: false, error: 'Content type is required' }
}
if (maxItems && (Number.isNaN(Number(maxItems)) || Number(maxItems) <= 0)) {
return { valid: false, error: 'Max items must be a positive number' }
}
try {
// Verify API access by fetching the first page of articles or conversations
if (contentType === 'articles' || contentType === 'both') {
await intercomApiGet(
'/articles',
accessToken,
{ page: '1', per_page: '1' },
VALIDATE_RETRY_OPTIONS
)
}
if (contentType === 'conversations' || contentType === 'both') {
await intercomApiGet(
'/conversations',
accessToken,
{ per_page: '1' },
VALIDATE_RETRY_OPTIONS
)
}
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.type === 'string') {
result.type = metadata.type
}
if (typeof metadata.state === 'string') {
result.state = metadata.state
}
if (typeof metadata.tags === 'string' && metadata.tags) {
result.tags = metadata.tags
}
if (typeof metadata.authorId === 'string') {
result.authorId = metadata.authorId
}
if (typeof metadata.messageCount === 'number') {
result.messageCount = metadata.messageCount
}
const updatedAt = parseTagDate(metadata.updatedAt)
if (updatedAt) {
result.updatedAt = updatedAt
}
return result
},
}
+74
View File
@@ -0,0 +1,74 @@
import { IntercomIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const DEFAULT_MAX_ITEMS = 500
export const intercomConnectorMeta: ConnectorMeta = {
id: 'intercom',
name: 'Intercom',
description: 'Sync Help Center articles and conversations from Intercom',
version: '1.0.0',
icon: IntercomIcon,
auth: {
mode: 'apiKey',
label: 'Access Token',
placeholder: 'Enter your Intercom access token',
},
configFields: [
{
id: 'contentType',
title: 'Content Type',
type: 'dropdown',
required: true,
description: 'Choose what to sync from Intercom',
options: [
{ label: 'Articles Only', id: 'articles' },
{ label: 'Conversations Only', id: 'conversations' },
{ label: 'Articles & Conversations', id: 'both' },
],
},
{
id: 'articleState',
title: 'Article State',
type: 'dropdown',
required: false,
description: 'Filter articles by state (default: published)',
options: [
{ label: 'Published', id: 'published' },
{ label: 'Draft', id: 'draft' },
{ label: 'All', id: 'all' },
],
},
{
id: 'conversationState',
title: 'Conversation State',
type: 'dropdown',
required: false,
description: 'Filter conversations by state (default: all)',
options: [
{ label: 'Open', id: 'open' },
{ label: 'Closed', id: 'closed' },
{ label: 'All', id: 'all' },
],
},
{
id: 'maxItems',
title: 'Max Items',
type: 'short-input',
required: false,
placeholder: `e.g. 200 (default: ${DEFAULT_MAX_ITEMS})`,
description: 'Maximum number of articles or conversations to sync',
},
],
tagDefinitions: [
{ id: 'type', displayName: 'Content Type', fieldType: 'text' },
{ id: 'state', displayName: 'State', fieldType: 'text' },
{ id: 'tags', displayName: 'Tags', fieldType: 'text' },
{ id: 'authorId', displayName: 'Author ID', fieldType: 'text' },
{ id: 'messageCount', displayName: 'Message Count', fieldType: 'number' },
{ id: 'updatedAt', displayName: 'Last Updated', fieldType: 'date' },
],
}
+1
View File
@@ -0,0 +1 @@
export { jiraConnector } from '@/connectors/jira/jira'
+357
View File
@@ -0,0 +1,357 @@
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { jiraConnectorMeta } from '@/connectors/jira/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils'
import { extractAdfText, getJiraCloudId } from '@/tools/jira/utils'
const logger = createLogger('JiraConnector')
const PAGE_SIZE = 50
/**
* Builds a JQL clause restricting issues to the given project keys.
* Single key uses `project = "X"`; multiple keys use `project in ("X","Y")`.
* Each key is escaped for inclusion in a JQL double-quoted string.
*/
function buildProjectClause(projectKeys: string[]): string {
const escapeKey = (key: string) => key.replace(/\\/g, '\\\\').replace(/"/g, '\\"')
if (projectKeys.length === 1) {
return `project = "${escapeKey(projectKeys[0])}"`
}
const list = projectKeys.map((k) => `"${escapeKey(k)}"`).join(',')
return `project in (${list})`
}
/**
* Builds a plain-text representation of a Jira issue for knowledge base indexing.
*/
function buildIssueContent(fields: Record<string, unknown>): string {
const parts: string[] = []
const summary = fields.summary as string | undefined
if (summary) parts.push(summary)
const description = extractAdfText(fields.description)
if (description) parts.push(description)
const comments = fields.comment as { comments?: Array<{ body?: unknown }> } | undefined
if (comments?.comments) {
for (const comment of comments.comments) {
const text = extractAdfText(comment.body)
if (text) parts.push(text)
}
}
return parts.join('\n\n').trim()
}
/**
* Extracts common metadata fields from a Jira issue into an ExternalDocument
* stub with deferred content. The contentHash is metadata-based so it is
* identical whether produced during listing or full fetch.
*/
function issueToStub(issue: Record<string, unknown>, domain: string): ExternalDocument {
const fields = (issue.fields || {}) as Record<string, unknown>
const key = issue.key as string
const issueType = fields.issuetype as Record<string, unknown> | undefined
const status = fields.status as Record<string, unknown> | undefined
const priority = fields.priority as Record<string, unknown> | undefined
const assignee = fields.assignee as Record<string, unknown> | undefined
const reporter = fields.reporter as Record<string, unknown> | undefined
const project = fields.project as Record<string, unknown> | undefined
const labels = Array.isArray(fields.labels) ? (fields.labels as string[]) : []
const updated = (fields.updated as string) ?? ''
return {
externalId: String(issue.id),
title: `${key}: ${(fields.summary as string) || 'Untitled'}`,
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: `https://${domain}/browse/${key}`,
contentHash: `jira:${issue.id}:${updated}`,
metadata: {
key,
issueType: issueType?.name,
status: status?.name,
priority: priority?.name,
assignee: assignee?.displayName,
reporter: reporter?.displayName,
project: project?.key,
labels,
created: fields.created,
updated: fields.updated,
},
}
}
/**
* Converts a fully-fetched Jira issue (with description and comments) into an
* ExternalDocument with resolved content.
*/
function issueToFullDocument(issue: Record<string, unknown>, domain: string): ExternalDocument {
const stub = issueToStub(issue, domain)
const fields = (issue.fields || {}) as Record<string, unknown>
const content = buildIssueContent(fields)
return {
...stub,
content,
contentDeferred: false,
}
}
export const jiraConnector: ConnectorConfig = {
...jiraConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const domain = sourceConfig.domain as string
const projectKeys = parseMultiValue(sourceConfig.projectKey)
const jqlFilter = (sourceConfig.jql as string) || ''
const maxIssues = sourceConfig.maxIssues ? Number(sourceConfig.maxIssues) : 0
if (projectKeys.length === 0) {
throw new Error('At least one project key is required')
}
let cloudId = syncContext?.cloudId as string | undefined
if (!cloudId) {
cloudId = await getJiraCloudId(domain, accessToken)
if (syncContext) syncContext.cloudId = cloudId
}
const projectClause = buildProjectClause(projectKeys)
let jql = `${projectClause} ORDER BY updated DESC`
if (jqlFilter.trim()) {
jql = `${projectClause} AND (${jqlFilter.trim()}) ORDER BY updated DESC`
}
/**
* Collected-count is encoded in the cursor as `${pageToken}|${count}` so
* the maxIssues cap works correctly even when the caller doesn't pass
* syncContext. Falls back to syncContext.collectedCount for backwards
* compatibility with cursors emitted before this format existed.
*/
let pageToken: string | undefined
let collectedSoFar = (syncContext?.collectedCount as number | undefined) ?? 0
if (cursor) {
const sep = cursor.lastIndexOf('|')
if (sep > 0) {
pageToken = cursor.slice(0, sep)
const parsed = Number(cursor.slice(sep + 1))
if (Number.isFinite(parsed) && parsed >= 0) collectedSoFar = parsed
} else {
pageToken = cursor
}
}
const remaining = maxIssues > 0 ? Math.max(0, maxIssues - collectedSoFar) : PAGE_SIZE
if (maxIssues > 0 && remaining === 0) {
return { documents: [], hasMore: false }
}
const params = new URLSearchParams()
params.append('jql', jql)
params.append('maxResults', String(Math.min(PAGE_SIZE, remaining)))
params.append(
'fields',
'summary,issuetype,status,priority,assignee,reporter,project,labels,created,updated'
)
if (pageToken) params.append('nextPageToken', pageToken)
const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/search/jql?${params.toString()}`
logger.info(`Listing Jira issues for ${projectKeys.length} project(s)`, {
projectKeys,
hasCursor: Boolean(cursor),
})
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/json',
Authorization: `Bearer ${accessToken}`,
},
})
if (!response.ok) {
const errorText = await response.text()
logger.error('Failed to search Jira issues', {
status: response.status,
error: errorText,
})
throw new Error(`Failed to search Jira issues: ${response.status}`)
}
const data = await response.json()
let issues = (data.issues || []) as Record<string, unknown>[]
/**
* `/rest/api/3/search/jql` signals end-of-results purely by the absence
* of `nextPageToken`. `data.isLast` is unreliable on this endpoint and
* has been observed returning `true` alongside a valid token
* (JRACLOUD-95477), so we ignore it.
*/
const nextPageToken = data.nextPageToken as string | undefined
const isLast = !nextPageToken
if (maxIssues > 0 && issues.length > remaining) {
issues = issues.slice(0, remaining)
}
const documents: ExternalDocument[] = issues.map((issue) => issueToStub(issue, domain))
const newCollected = collectedSoFar + issues.length
if (syncContext) syncContext.collectedCount = newCollected
const reachedCap = maxIssues > 0 && newCollected >= maxIssues
const hasMore = !isLast && !reachedCap
return {
documents,
nextCursor: hasMore && nextPageToken ? `${nextPageToken}|${newCollected}` : undefined,
hasMore,
}
},
getDocument: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
externalId: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocument | null> => {
const domain = sourceConfig.domain as string
let cloudId = syncContext?.cloudId as string | undefined
if (!cloudId) {
cloudId = await getJiraCloudId(domain, accessToken)
if (syncContext) syncContext.cloudId = cloudId
}
const params = new URLSearchParams()
params.append(
'fields',
'summary,description,comment,issuetype,status,priority,assignee,reporter,project,labels,created,updated'
)
const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${externalId}?${params.toString()}`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: {
Accept: 'application/json',
Authorization: `Bearer ${accessToken}`,
},
})
if (!response.ok) {
if (response.status === 404) return null
throw new Error(`Failed to get Jira issue: ${response.status}`)
}
const issue = await response.json()
return issueToFullDocument(issue, domain)
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const domain = sourceConfig.domain as string
const projectKeys = parseMultiValue(sourceConfig.projectKey)
if (!domain || projectKeys.length === 0) {
return { valid: false, error: 'Domain and at least one project key are required' }
}
const maxIssues = sourceConfig.maxIssues as string | undefined
if (maxIssues && (Number.isNaN(Number(maxIssues)) || Number(maxIssues) <= 0)) {
return { valid: false, error: 'Max issues must be a positive number' }
}
const jqlFilter = (sourceConfig.jql as string | undefined)?.trim() || ''
try {
const cloudId = await getJiraCloudId(domain, accessToken, VALIDATE_RETRY_OPTIONS)
const projectClause = buildProjectClause(projectKeys)
const params = new URLSearchParams()
params.append('jql', projectClause)
params.append('maxResults', '1')
const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/search/jql?${params.toString()}`
const response = await fetchWithRetry(
url,
{
method: 'GET',
headers: {
Accept: 'application/json',
Authorization: `Bearer ${accessToken}`,
},
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
const errorText = await response.text()
if (response.status === 400) {
return {
valid: false,
error: `One or more projects not found (${projectKeys.join(', ')}) or JQL is invalid`,
}
}
return { valid: false, error: `Failed to validate: ${response.status} - ${errorText}` }
}
if (jqlFilter) {
const filterParams = new URLSearchParams()
filterParams.append('jql', `${projectClause} AND (${jqlFilter})`)
filterParams.append('maxResults', '1')
const filterUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/search/jql?${filterParams.toString()}`
const filterResponse = await fetchWithRetry(
filterUrl,
{
method: 'GET',
headers: {
Accept: 'application/json',
Authorization: `Bearer ${accessToken}`,
},
},
VALIDATE_RETRY_OPTIONS
)
if (!filterResponse.ok) {
return { valid: false, error: 'Invalid JQL filter. Check syntax and field names.' }
}
}
return { valid: true }
} catch (error) {
return { valid: false, error: toError(error).message || 'Failed to validate configuration' }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.issueType === 'string') result.issueType = metadata.issueType
if (typeof metadata.status === 'string') result.status = metadata.status
if (typeof metadata.priority === 'string') result.priority = metadata.priority
const labels = joinTagArray(metadata.labels)
if (labels) result.labels = labels
if (typeof metadata.assignee === 'string') result.assignee = metadata.assignee
const updated = parseTagDate(metadata.updated)
if (updated) result.updated = updated
return result
},
}
+67
View File
@@ -0,0 +1,67 @@
import { JiraIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const jiraConnectorMeta: ConnectorMeta = {
id: 'jira',
name: 'Jira',
description: 'Sync issues from a Jira project',
version: '1.0.0',
icon: JiraIcon,
auth: { mode: 'oauth', provider: 'jira', requiredScopes: ['read:jira-work', 'offline_access'] },
configFields: [
{
id: 'domain',
title: 'Jira Domain',
type: 'short-input',
placeholder: 'yoursite.atlassian.net',
required: true,
},
{
id: 'projectSelector',
title: 'Projects',
type: 'selector',
selectorKey: 'jira.projects',
canonicalParamId: 'projectKey',
mode: 'basic',
multi: true,
dependsOn: ['domain'],
placeholder: 'Select one or more projects',
required: true,
},
{
id: 'projectKey',
title: 'Project Keys',
type: 'short-input',
canonicalParamId: 'projectKey',
mode: 'advanced',
multi: true,
placeholder: 'e.g. ENG, PROJ (comma-separated for multiple)',
required: true,
},
{
id: 'jql',
title: 'JQL Filter',
type: 'short-input',
required: false,
placeholder: 'e.g. status = "Done" AND type = Bug',
},
{
id: 'maxIssues',
title: 'Max Issues',
type: 'short-input',
required: false,
placeholder: 'e.g. 500 (default: unlimited)',
},
],
tagDefinitions: [
{ id: 'issueType', displayName: 'Issue Type', fieldType: 'text' },
{ id: 'status', displayName: 'Status', fieldType: 'text' },
{ id: 'priority', displayName: 'Priority', fieldType: 'text' },
{ id: 'labels', displayName: 'Labels', fieldType: 'text' },
{ id: 'assignee', displayName: 'Assignee', fieldType: 'text' },
{ id: 'updated', displayName: 'Last Updated', fieldType: 'date' },
],
}
+1
View File
@@ -0,0 +1 @@
export { jsmConnector } from '@/connectors/jsm/jsm'
+556
View File
@@ -0,0 +1,556 @@
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { jsmConnectorMeta } from '@/connectors/jsm/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { parseTagDate } from '@/connectors/utils'
import { extractAdfText, getJiraCloudId } from '@/tools/jira/utils'
import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils'
const logger = createLogger('JsmConnector')
const PAGE_SIZE = 50
/**
* Allowed `requestStatus` filter values for `GET /rest/servicedeskapi/request`.
* When omitted, the JSM API defaults to `ALL_REQUESTS`.
*/
const VALID_REQUEST_STATUS = ['OPEN_REQUESTS', 'CLOSED_REQUESTS', 'ALL_REQUESTS'] as const
type JsmRequestStatus = (typeof VALID_REQUEST_STATUS)[number]
/**
* Allowed `requestOwnership` filter values for `GET /rest/servicedeskapi/request`.
*
* This param scopes results to the OAuth user's relationship to each request. When
* omitted, the JSM API defaults to `OWNED_REQUESTS` — i.e. only requests the
* authenticated user reported. For a knowledge-base sync the user almost always
* wants every request in the service desk, so the connector defaults this to
* `ALL_REQUESTS` (which the JSM API treats as "owned + participated") rather than
* relying on the API's narrower default.
*/
const VALID_REQUEST_OWNERSHIP = ['OWNED_REQUESTS', 'PARTICIPATED_REQUESTS', 'ALL_REQUESTS'] as const
type JsmRequestOwnership = (typeof VALID_REQUEST_OWNERSHIP)[number]
/**
* Which comments to include in synced documents.
*/
const VALID_COMMENT_SCOPE = ['none', 'public', 'all'] as const
type JsmCommentScope = (typeof VALID_COMMENT_SCOPE)[number]
/**
* A JSM date object as returned by the Service Desk REST API. The same shape is
* used for `createdDate`, `currentStatus.statusDate`, and comment `created`.
*/
interface JsmDate {
iso8601?: string
friendly?: string
epochMillis?: number
}
/**
* Subset of a JSM customer request returned by `GET /request` and
* `GET /request/{issueIdOrKey}`. Only the fields the connector reads are typed.
*/
interface JsmRequest {
issueId?: string
issueKey?: string
requestTypeId?: string
serviceDeskId?: string
createdDate?: JsmDate
currentStatus?: {
status?: string
statusCategory?: string
statusDate?: JsmDate
}
reporter?: {
displayName?: string
emailAddress?: string
}
requestFieldValues?: Array<{
fieldId?: string
label?: string
value?: unknown
renderedValue?: unknown
}>
_links?: {
web?: string
}
}
/**
* A single comment on a JSM request. The JSM API returns the comment `body` as a
* plain string containing Jira wiki markup (not an ADF document), so no rich-text
* extraction is required.
*/
interface JsmComment {
id?: string
body?: string
public?: boolean
author?: {
displayName?: string
}
created?: JsmDate
}
/**
* Paginated envelope shared by every JSM Service Desk list endpoint.
*/
interface JsmPage<T> {
values?: T[]
size?: number
isLastPage?: boolean
}
/**
* Reads the resolved sync options off the raw `sourceConfig`, normalizing
* enum-like fields to their valid set and clamping the numeric cap. Centralized
* so `listDocuments`, `getDocument`, and `validateConfig` agree on defaults.
*/
function resolveOptions(sourceConfig: Record<string, unknown>): {
requestStatus: JsmRequestStatus
requestOwnership: JsmRequestOwnership
requestTypeId: string
searchTerm: string
commentScope: JsmCommentScope
maxRequests: number
} {
const requestStatus = VALID_REQUEST_STATUS.includes(
sourceConfig.requestStatus as JsmRequestStatus
)
? (sourceConfig.requestStatus as JsmRequestStatus)
: 'ALL_REQUESTS'
const requestOwnership = VALID_REQUEST_OWNERSHIP.includes(
sourceConfig.requestOwnership as JsmRequestOwnership
)
? (sourceConfig.requestOwnership as JsmRequestOwnership)
: 'ALL_REQUESTS'
const commentScope = VALID_COMMENT_SCOPE.includes(sourceConfig.comments as JsmCommentScope)
? (sourceConfig.comments as JsmCommentScope)
: 'public'
const requestTypeId =
typeof sourceConfig.requestTypeId === 'string' ? sourceConfig.requestTypeId.trim() : ''
const searchTerm =
typeof sourceConfig.searchTerm === 'string' ? sourceConfig.searchTerm.trim() : ''
const parsedMax = sourceConfig.maxRequests ? Number(sourceConfig.maxRequests) : 0
const maxRequests = Number.isFinite(parsedMax) && parsedMax > 0 ? Math.floor(parsedMax) : 0
return { requestStatus, requestOwnership, requestTypeId, searchTerm, commentScope, maxRequests }
}
/**
* Extracts a plain-text value for a given request field id (e.g. `summary`,
* `description`) from a request's `requestFieldValues`. The JSM API returns
* `value` either as a plain string (wiki markup) or, for some rich-text fields,
* as an ADF document — both are handled.
*/
function getFieldText(request: JsmRequest, fieldId: string): string {
const field = request.requestFieldValues?.find((f) => f.fieldId === fieldId)
if (!field) return ''
const { value } = field
if (typeof value === 'string') return value
if (value && typeof value === 'object') {
const adf = extractAdfText(value)
if (adf) return adf
}
return ''
}
/**
* Resolves the best available "change indicator" timestamp for a request.
*
* The JSM list endpoint does NOT return an updated/last-modified field — only
* `createdDate` and `currentStatus.statusDate` are present. We use
* `statusDate` (the time the request last changed status) when available, and
* fall back to `createdDate`. This is the change signal encoded into the
* contentHash. Note: edits that do not change status (e.g. a new comment) are
* not reflected here, so such changes may not trigger a re-sync.
*/
function getChangeIndicator(request: JsmRequest): string {
const statusDate = request.currentStatus?.statusDate
if (statusDate?.epochMillis != null) return String(statusDate.epochMillis)
if (statusDate?.iso8601) return statusDate.iso8601
const created = request.createdDate
if (created?.epochMillis != null) return String(created.epochMillis)
if (created?.iso8601) return created.iso8601
return ''
}
/**
* Builds a stub ExternalDocument from a request returned by the list endpoint.
* Content is deferred — description and comments require a per-request API call
* fetched lazily in `getDocument`. The contentHash is metadata-only so it is
* identical whether produced here or in `getDocument`.
*/
function requestToStub(request: JsmRequest, domain: string): ExternalDocument {
const issueId = String(request.issueId ?? '')
const issueKey = request.issueKey ?? issueId
const summary = getFieldText(request, 'summary') || 'Untitled'
const status = request.currentStatus?.status
const bareDomain = domain
.trim()
.replace(/^https?:\/\//i, '')
.replace(/\/+$/, '')
return {
externalId: issueId,
title: `${issueKey}: ${summary}`,
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: request._links?.web || `https://${bareDomain}/browse/${issueKey}`,
contentHash: `jsm:${issueId}:${getChangeIndicator(request)}`,
metadata: {
issueKey,
requestTypeId: request.requestTypeId,
serviceDeskId: request.serviceDeskId,
status,
reporter: request.reporter?.displayName,
created: request.createdDate?.iso8601,
/**
* The list endpoint has no true "last updated" field; `statusDate` is the
* closest available signal (time of last status change). Mapped to the
* `updated` tag and documented as such.
*/
statusDate: request.currentStatus?.statusDate?.iso8601,
},
}
}
/**
* Renders a readable plain-text document from a fully-fetched request and its
* comments. Includes summary, description, reporter, status, and comment thread.
*/
function buildContent(request: JsmRequest, comments: JsmComment[]): string {
const parts: string[] = []
const summary = getFieldText(request, 'summary')
if (summary) parts.push(summary)
const description = getFieldText(request, 'description')
if (description) parts.push(description)
const status = request.currentStatus?.status
if (status) parts.push(`Status: ${status}`)
const reporter = request.reporter?.displayName
if (reporter) parts.push(`Reporter: ${reporter}`)
if (comments.length > 0) {
parts.push('Comments:')
for (const comment of comments) {
const body = (comment.body ?? '').trim()
if (!body) continue
const author = comment.author?.displayName
parts.push(author ? `${author}: ${body}` : body)
}
}
return parts.join('\n\n').trim()
}
/**
* Resolves and caches the Jira cloud ID for a domain across a sync run.
*/
async function resolveCloudId(
domain: string,
accessToken: string,
syncContext?: Record<string, unknown>
): Promise<string> {
const cached = syncContext?.cloudId as string | undefined
if (cached) return cached
const cloudId = await getJiraCloudId(domain, accessToken)
if (syncContext) syncContext.cloudId = cloudId
return cloudId
}
/**
* Fetches comments for a request, following offset pagination until the API
* signals `isLastPage`. When `publicOnly` is true the `public=true` filter is
* applied so internal/agent-only comments are excluded.
*/
async function fetchComments(
baseUrl: string,
accessToken: string,
issueIdOrKey: string,
publicOnly: boolean
): Promise<JsmComment[]> {
const comments: JsmComment[] = []
let start = 0
while (true) {
const params = new URLSearchParams({
start: String(start),
limit: String(PAGE_SIZE),
})
/**
* The JSM comment endpoint exposes `public` and `internal` as independent
* inclusion filters that both default to `true`. Requesting public-only
* therefore requires explicitly disabling `internal` — passing `public=true`
* alone would still return agent-only/internal comments.
*/
if (publicOnly) {
params.append('public', 'true')
params.append('internal', 'false')
}
const url = `${baseUrl}/request/${encodeURIComponent(issueIdOrKey)}/comment?${params.toString()}`
const response = await fetchWithRetry(url, {
method: 'GET',
headers: getJsmHeaders(accessToken),
})
if (!response.ok) {
logger.warn('Failed to fetch JSM comments', {
issueIdOrKey,
status: response.status,
})
break
}
const data = (await response.json()) as JsmPage<JsmComment>
const values = data.values ?? []
comments.push(...values)
if (data.isLastPage || values.length === 0) break
start += values.length
}
return comments
}
export const jsmConnector: ConnectorConfig = {
...jsmConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const domain = sourceConfig.domain as string
const serviceDeskId = sourceConfig.serviceDeskId as string
if (!domain || !serviceDeskId) {
throw new Error('Domain and service desk ID are required')
}
const { requestStatus, requestOwnership, requestTypeId, searchTerm, maxRequests } =
resolveOptions(sourceConfig)
const cloudId = await resolveCloudId(domain, accessToken, syncContext)
const baseUrl = getJsmApiBaseUrl(cloudId)
/**
* `start|collected` is encoded in the cursor so the maxRequests cap holds
* across pages even if syncContext is not threaded through by the caller.
*/
let start = 0
let collectedSoFar = (syncContext?.collectedCount as number | undefined) ?? 0
if (cursor) {
const sep = cursor.indexOf('|')
if (sep > 0) {
const parsedStart = Number(cursor.slice(0, sep))
const parsedCount = Number(cursor.slice(sep + 1))
if (Number.isFinite(parsedStart) && parsedStart >= 0) start = parsedStart
if (Number.isFinite(parsedCount) && parsedCount >= 0) collectedSoFar = parsedCount
} else {
const parsedStart = Number(cursor)
if (Number.isFinite(parsedStart) && parsedStart >= 0) start = parsedStart
}
}
const remaining = maxRequests > 0 ? Math.max(0, maxRequests - collectedSoFar) : PAGE_SIZE
if (maxRequests > 0 && remaining === 0) {
return { documents: [], hasMore: false }
}
const params = new URLSearchParams({
serviceDeskId,
requestStatus,
start: String(start),
limit: String(Math.min(PAGE_SIZE, remaining)),
})
params.append('requestOwnership', requestOwnership)
if (requestTypeId) params.append('requestTypeId', requestTypeId)
if (searchTerm) params.append('searchTerm', searchTerm)
const url = `${baseUrl}/request?${params.toString()}`
logger.info('Listing JSM requests', {
serviceDeskId,
requestStatus,
requestOwnership,
hasCursor: Boolean(cursor),
})
const response = await fetchWithRetry(url, {
method: 'GET',
headers: getJsmHeaders(accessToken),
})
if (!response.ok) {
const errorText = await response.text()
logger.error('Failed to list JSM requests', { status: response.status, error: errorText })
throw new Error(`Failed to list JSM requests: ${response.status}`)
}
const data = (await response.json()) as JsmPage<JsmRequest>
let requests = data.values ?? []
let slicedSome = false
if (maxRequests > 0 && requests.length > remaining) {
slicedSome = true
requests = requests.slice(0, remaining)
}
const documents = requests.map((request) => requestToStub(request, domain))
const newCollected = collectedSoFar + requests.length
if (syncContext) syncContext.collectedCount = newCollected
const reachedCap = maxRequests > 0 && newCollected >= maxRequests
/**
* When `maxRequests` truncates the listing before the source is exhausted,
* flag the run as capped so the sync engine skips deletion reconciliation —
* otherwise unseen requests beyond the cap would be deleted on every sync.
* `slicedSome` covers truncation on the final page: requests dropped from
* this page still exist even when `isLastPage` is true. (The requested
* `limit` never exceeds the remaining budget, so a slice should be
* impossible — this is defense in depth against the API over-returning.)
*/
if (((reachedCap && !data.isLastPage) || slicedSome) && syncContext) {
syncContext.listingCapped = true
}
const hasMore = !data.isLastPage && requests.length > 0 && !reachedCap
const nextStart = start + requests.length
return {
documents,
nextCursor: hasMore ? `${nextStart}|${newCollected}` : undefined,
hasMore,
}
},
getDocument: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
externalId: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocument | null> => {
const domain = sourceConfig.domain as string
const { commentScope } = resolveOptions(sourceConfig)
const cloudId = await resolveCloudId(domain, accessToken, syncContext)
const baseUrl = getJsmApiBaseUrl(cloudId)
const requestUrl = `${baseUrl}/request/${encodeURIComponent(externalId)}?expand=status`
const response = await fetchWithRetry(requestUrl, {
method: 'GET',
headers: getJsmHeaders(accessToken),
})
if (!response.ok) {
if (response.status === 404) return null
if (response.status === 401 || response.status === 403) {
logger.warn('Access denied fetching JSM request', { externalId, status: response.status })
return null
}
throw new Error(`Failed to get JSM request: ${response.status}`)
}
const request = (await response.json()) as JsmRequest
const comments =
commentScope === 'none'
? []
: await fetchComments(baseUrl, accessToken, externalId, commentScope === 'public')
const stub = requestToStub(request, domain)
const content = buildContent(request, comments)
return {
...stub,
content,
contentDeferred: false,
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const domain = sourceConfig.domain as string
const serviceDeskId = sourceConfig.serviceDeskId as string
if (!domain || !serviceDeskId) {
return { valid: false, error: 'Domain and service desk ID are required' }
}
if (sourceConfig.maxRequests) {
const max = Number(sourceConfig.maxRequests)
if (Number.isNaN(max) || max <= 0) {
return { valid: false, error: 'Max requests must be a positive number' }
}
}
try {
const cloudId = await getJiraCloudId(domain, accessToken, VALIDATE_RETRY_OPTIONS)
const baseUrl = getJsmApiBaseUrl(cloudId)
const url = `${baseUrl}/servicedesk/${encodeURIComponent(serviceDeskId)}`
const response = await fetchWithRetry(
url,
{
method: 'GET',
headers: getJsmHeaders(accessToken),
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
if (response.status === 404) {
return { valid: false, error: `Service desk "${serviceDeskId}" not found` }
}
if (response.status === 401 || response.status === 403) {
return {
valid: false,
error: 'Access denied. Check the connected account has access to this service desk.',
}
}
const errorText = await response.text()
return { valid: false, error: `Failed to validate: ${response.status} - ${errorText}` }
}
return { valid: true }
} catch (error) {
return { valid: false, error: toError(error).message || 'Failed to validate configuration' }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.status === 'string') result.status = metadata.status
if (typeof metadata.requestTypeId === 'string') result.requestTypeId = metadata.requestTypeId
if (typeof metadata.reporter === 'string') result.reporter = metadata.reporter
const created = parseTagDate(metadata.created)
if (created) result.created = created
/**
* The list endpoint exposes no true last-updated field; `statusDate` (time
* of last status change) is the closest available signal and surfaces under
* the "Last Status Change" tag.
*/
const statusDate = parseTagDate(metadata.statusDate)
if (statusDate) result.updated = statusDate
return result
},
}
+143
View File
@@ -0,0 +1,143 @@
import { JiraServiceManagementIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const jsmConnectorMeta: ConnectorMeta = {
id: 'jsm',
name: 'Jira Service Management',
description: 'Sync service desk requests from Jira Service Management into your knowledge base',
version: '1.0.0',
icon: JiraServiceManagementIcon,
auth: {
mode: 'oauth',
provider: 'jira',
requiredScopes: [
/**
* Atlassian enforces granular scope sets all-or-nothing; the classic scope
* alone authorizes the request read endpoints, so require it to flag stale
* credentials that predate it in the provider scope list.
*/
'read:servicedesk-request',
'read:servicedesk:jira-service-management',
'read:request:jira-service-management',
'read:request.comment:jira-service-management',
'read:request.status:jira-service-management',
/**
* Requests embed a `reporter` user object whose `displayName` is surfaced
* in document content and the Reporter tag. Atlassian only populates
* embedded user data when the user-read scope is granted, so request it
* here. Present in the `jira` OAuth provider config as `read:jira-user`.
*/
'read:jira-user',
'offline_access',
],
},
configFields: [
{
id: 'domain',
title: 'Jira Domain',
type: 'short-input',
placeholder: 'yoursite.atlassian.net',
required: true,
},
{
id: 'serviceDeskSelector',
title: 'Service Desk',
type: 'selector',
selectorKey: 'jsm.serviceDesks',
canonicalParamId: 'serviceDeskId',
mode: 'basic',
dependsOn: ['domain'],
placeholder: 'Select a service desk',
required: true,
},
{
id: 'serviceDeskId',
title: 'Service Desk ID',
type: 'short-input',
canonicalParamId: 'serviceDeskId',
mode: 'advanced',
placeholder: 'e.g. 1, 2',
required: true,
},
{
id: 'requestTypeSelector',
title: 'Request Type',
type: 'selector',
selectorKey: 'jsm.requestTypes',
canonicalParamId: 'requestTypeId',
mode: 'basic',
dependsOn: ['domain', 'serviceDeskSelector'],
placeholder: 'All request types',
required: false,
},
{
id: 'requestTypeId',
title: 'Request Type ID',
type: 'short-input',
canonicalParamId: 'requestTypeId',
mode: 'advanced',
placeholder: 'e.g. 10 (leave blank for all)',
required: false,
},
{
id: 'requestStatus',
title: 'Request Status',
type: 'dropdown',
required: false,
options: [
{ label: 'All requests', id: 'ALL_REQUESTS' },
{ label: 'Open requests', id: 'OPEN_REQUESTS' },
{ label: 'Closed requests', id: 'CLOSED_REQUESTS' },
],
},
{
id: 'requestOwnership',
title: 'Request Ownership',
type: 'dropdown',
required: false,
description:
'Which requests the connected account can see. "Owned + participated" is the broadest scope a customer token can sync.',
options: [
{ label: 'Owned + participated', id: 'ALL_REQUESTS' },
{ label: 'Owned only', id: 'OWNED_REQUESTS' },
{ label: 'Participated only', id: 'PARTICIPATED_REQUESTS' },
],
},
{
id: 'comments',
title: 'Include Comments',
type: 'dropdown',
required: false,
description: 'Comments require an extra API call per request during sync.',
options: [
{ label: 'Public comments only', id: 'public' },
{ label: 'All comments (incl. internal)', id: 'all' },
{ label: 'No comments', id: 'none' },
],
},
{
id: 'searchTerm',
title: 'Search Filter',
type: 'short-input',
required: false,
placeholder: 'e.g. password reset (optional)',
},
{
id: 'maxRequests',
title: 'Max Requests',
type: 'short-input',
required: false,
placeholder: 'e.g. 500 (default: unlimited)',
},
],
tagDefinitions: [
{ id: 'status', displayName: 'Status', fieldType: 'text' },
{ id: 'requestTypeId', displayName: 'Request Type', fieldType: 'text' },
{ id: 'reporter', displayName: 'Reporter', fieldType: 'text' },
{ id: 'created', displayName: 'Created', fieldType: 'date' },
{ id: 'updated', displayName: 'Last Status Change', fieldType: 'date' },
],
}
+1
View File
@@ -0,0 +1 @@
export { linearConnector } from '@/connectors/linear/linear'
+377
View File
@@ -0,0 +1,377 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import type { RetryOptions } from '@/lib/knowledge/documents/utils'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { linearConnectorMeta } from '@/connectors/linear/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils'
const logger = createLogger('LinearConnector')
const LINEAR_API = 'https://api.linear.app/graphql'
/**
* Strips Markdown formatting to produce plain text.
*/
function markdownToPlainText(md: string): string {
let text = md
.replace(/!\[.*?\]\(.*?\)/g, '') // images
.replace(/\[([^\]]*)\]\(.*?\)/g, '$1') // links
.replace(/#{1,6}\s+/g, '') // headings
.replace(/(\*\*|__)(.*?)\1/g, '$2') // bold
.replace(/(\*|_)(.*?)\1/g, '$2') // italic
.replace(/~~(.*?)~~/g, '$1') // strikethrough
.replace(/`{3}[\s\S]*?`{3}/g, '') // code blocks
.replace(/`([^`]*)`/g, '$1') // inline code
.replace(/^\s*[-*+]\s+/gm, '') // list items
.replace(/^\s*\d+\.\s+/gm, '') // ordered list items
.replace(/^\s*>\s+/gm, '') // blockquotes
.replace(/---+/g, '') // horizontal rules
text = text.replace(/\s+/g, ' ').trim()
return text
}
/**
* Executes a GraphQL query against the Linear API.
*/
async function linearGraphQL(
accessToken: string,
query: string,
variables?: Record<string, unknown>,
retryOptions?: RetryOptions
): Promise<Record<string, unknown>> {
const response = await fetchWithRetry(
LINEAR_API,
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${accessToken}`,
},
body: JSON.stringify({ query, variables }),
},
retryOptions
)
if (!response.ok) {
const errorText = await response.text()
logger.error('Linear GraphQL request failed', { status: response.status, error: errorText })
throw new Error(`Linear API error: ${response.status}`)
}
const json = (await response.json()) as { data?: Record<string, unknown>; errors?: unknown[] }
if (json.errors) {
logger.error('Linear GraphQL errors', { errors: json.errors })
throw new Error(`Linear GraphQL error: ${JSON.stringify(json.errors)}`)
}
return json.data as Record<string, unknown>
}
/**
* Builds a formatted text document from a Linear issue.
*/
function buildIssueContent(issue: Record<string, unknown>): string {
const parts: string[] = []
const identifier = issue.identifier as string | undefined
const title = (issue.title as string) || 'Untitled'
parts.push(`${identifier ? `${identifier}: ` : ''}${title}`)
const state = issue.state as Record<string, unknown> | undefined
if (state?.name) parts.push(`Status: ${state.name}`)
const priority = issue.priorityLabel as string | undefined
if (priority) parts.push(`Priority: ${priority}`)
const assignee = issue.assignee as Record<string, unknown> | undefined
if (assignee?.name) parts.push(`Assignee: ${assignee.name}`)
const labelsConn = issue.labels as Record<string, unknown> | undefined
const labelNodes = (labelsConn?.nodes || []) as Record<string, unknown>[]
if (labelNodes.length > 0) {
parts.push(`Labels: ${labelNodes.map((l) => l.name as string).join(', ')}`)
}
const description = issue.description as string | undefined
if (description) {
parts.push('')
parts.push(markdownToPlainText(description))
}
return parts.join('\n')
}
const ISSUE_FIELDS = `
id
identifier
title
description
priority
priorityLabel
url
createdAt
updatedAt
state { name }
assignee { name }
labels { nodes { name } }
team { name key }
project { name }
`
const ISSUE_BY_ID_QUERY = `
query GetIssue($id: ID!) {
issue(id: $id) {
${ISSUE_FIELDS}
}
}
`
const TEAMS_QUERY = `
query { teams { nodes { id name key } } }
`
/**
* Dynamically builds a GraphQL issues query with only the filter clauses
* that have values, preventing null comparators from being sent to Linear.
*/
function buildIssuesQuery(
sourceConfig: Record<string, unknown>,
teamIds: string[],
projectIds: string[]
): {
query: string
variables: Record<string, unknown>
} {
const stateFilter = (sourceConfig.stateFilter as string) || ''
const varDefs: string[] = ['$first: Int!', '$after: String']
const filterClauses: string[] = []
const variables: Record<string, unknown> = {}
if (teamIds.length === 1) {
varDefs.push('$teamId: ID!')
filterClauses.push('team: { id: { eq: $teamId } }')
variables.teamId = teamIds[0]
} else if (teamIds.length > 1) {
varDefs.push('$teamIds: [ID!]!')
filterClauses.push('team: { id: { in: $teamIds } }')
variables.teamIds = teamIds
}
if (projectIds.length === 1) {
varDefs.push('$projectId: ID!')
filterClauses.push('project: { id: { eq: $projectId } }')
variables.projectId = projectIds[0]
} else if (projectIds.length > 1) {
varDefs.push('$projectIds: [ID!]!')
filterClauses.push('project: { id: { in: $projectIds } }')
variables.projectIds = projectIds
}
if (stateFilter) {
const states = stateFilter
.split(',')
.map((s) => s.trim())
.filter(Boolean)
if (states.length > 0) {
varDefs.push('$stateFilter: [String!]!')
filterClauses.push('state: { name: { in: $stateFilter } }')
variables.stateFilter = states
}
}
const filterArg = filterClauses.length > 0 ? `, filter: { ${filterClauses.join(', ')} }` : ''
const query = `
query ListIssues(${varDefs.join(', ')}) {
issues(first: $first, after: $after${filterArg}) {
nodes {
${ISSUE_FIELDS}
}
pageInfo {
hasNextPage
endCursor
}
}
}
`
return { query, variables }
}
export const linearConnector: ConnectorConfig = {
...linearConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const maxIssues = sourceConfig.maxIssues ? Number(sourceConfig.maxIssues) : 0
const pageSize = maxIssues > 0 ? Math.min(maxIssues, 50) : 50
const teamIds = parseMultiValue(sourceConfig.teamId)
const projectIds = parseMultiValue(sourceConfig.projectId)
const { query, variables } = buildIssuesQuery(sourceConfig, teamIds, projectIds)
const allVars = { ...variables, first: pageSize, after: cursor || undefined }
logger.info('Listing Linear issues', {
cursor,
pageSize,
teamFilterCount: teamIds.length,
projectFilterCount: projectIds.length,
})
const data = await linearGraphQL(accessToken, query, allVars)
const issuesConn = data.issues as Record<string, unknown>
const nodes = (issuesConn.nodes || []) as Record<string, unknown>[]
const pageInfo = issuesConn.pageInfo as Record<string, unknown>
const documents: ExternalDocument[] = nodes.map((issue) => {
const content = buildIssueContent(issue)
const contentHash = `linear:${issue.id}:${issue.updatedAt}`
const labelNodes = ((issue.labels as Record<string, unknown>)?.nodes || []) as Record<
string,
unknown
>[]
return {
externalId: issue.id as string,
title: `${(issue.identifier as string) || ''}: ${(issue.title as string) || 'Untitled'}`,
content,
mimeType: 'text/plain' as const,
sourceUrl: (issue.url as string) || undefined,
contentHash,
metadata: {
identifier: issue.identifier,
state: (issue.state as Record<string, unknown>)?.name,
priority: issue.priorityLabel,
assignee: (issue.assignee as Record<string, unknown>)?.name,
labels: labelNodes.map((l) => l.name as string),
team: (issue.team as Record<string, unknown>)?.name,
project: (issue.project as Record<string, unknown>)?.name,
lastModified: issue.updatedAt,
},
}
})
const hasNextPage = Boolean(pageInfo.hasNextPage)
const endCursor = (pageInfo.endCursor as string) || undefined
const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = maxIssues > 0 && totalFetched >= maxIssues
return {
documents,
nextCursor: hasNextPage && !hitLimit ? endCursor : undefined,
hasMore: hasNextPage && !hitLimit,
}
},
getDocument: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
try {
const data = await linearGraphQL(accessToken, ISSUE_BY_ID_QUERY, { id: externalId })
const issue = data.issue as Record<string, unknown> | null
if (!issue) return null
const content = buildIssueContent(issue)
const contentHash = `linear:${issue.id}:${issue.updatedAt}`
const labelNodes = ((issue.labels as Record<string, unknown>)?.nodes || []) as Record<
string,
unknown
>[]
return {
externalId: issue.id as string,
title: `${(issue.identifier as string) || ''}: ${(issue.title as string) || 'Untitled'}`,
content,
mimeType: 'text/plain' as const,
sourceUrl: (issue.url as string) || undefined,
contentHash,
metadata: {
identifier: issue.identifier,
state: (issue.state as Record<string, unknown>)?.name,
priority: issue.priorityLabel,
assignee: (issue.assignee as Record<string, unknown>)?.name,
labels: labelNodes.map((l) => l.name as string),
team: (issue.team as Record<string, unknown>)?.name,
project: (issue.project as Record<string, unknown>)?.name,
lastModified: issue.updatedAt,
},
}
} catch (error) {
logger.error('Failed to get Linear issue', {
externalId,
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const maxIssues = sourceConfig.maxIssues as string | undefined
if (maxIssues && (Number.isNaN(Number(maxIssues)) || Number(maxIssues) <= 0)) {
return { valid: false, error: 'Max issues must be a positive number' }
}
try {
const data = await linearGraphQL(accessToken, TEAMS_QUERY, undefined, VALIDATE_RETRY_OPTIONS)
const teamsConn = data.teams as Record<string, unknown>
const teams = (teamsConn.nodes || []) as Record<string, unknown>[]
if (teams.length === 0) {
return {
valid: false,
error: 'No teams found — check that the OAuth token has read access',
}
}
const requestedTeamIds = parseMultiValue(sourceConfig.teamId)
if (requestedTeamIds.length > 0) {
const availableIds = new Set(teams.map((t) => t.id as string))
const missing = requestedTeamIds.filter((id) => !availableIds.has(id))
if (missing.length > 0) {
return {
valid: false,
error: `Team ID(s) not found: ${missing.join(', ')}. Available teams: ${teams.map((t) => `${t.name} (${t.id})`).join(', ')}`,
}
}
}
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
const labels = joinTagArray(metadata.labels)
if (labels) result.labels = labels
if (typeof metadata.state === 'string') result.state = metadata.state
if (typeof metadata.priority === 'string') result.priority = metadata.priority
if (typeof metadata.assignee === 'string') result.assignee = metadata.assignee
const lastModified = parseTagDate(metadata.lastModified)
if (lastModified) result.lastModified = lastModified
return result
},
}
+80
View File
@@ -0,0 +1,80 @@
import { LinearIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const linearConnectorMeta: ConnectorMeta = {
id: 'linear',
name: 'Linear',
description: 'Sync issues from Linear',
version: '1.0.0',
icon: LinearIcon,
auth: { mode: 'oauth', provider: 'linear', requiredScopes: ['read'] },
configFields: [
{
id: 'teamSelector',
title: 'Teams',
type: 'selector',
selectorKey: 'linear.teams',
canonicalParamId: 'teamId',
mode: 'basic',
multi: true,
placeholder: 'Select one or more teams (optional)',
required: false,
},
{
id: 'teamId',
title: 'Team IDs',
type: 'short-input',
canonicalParamId: 'teamId',
mode: 'advanced',
multi: true,
placeholder: 'e.g. abc123, def456 (comma-separated for multiple)',
required: false,
},
{
id: 'projectSelector',
title: 'Projects',
type: 'selector',
selectorKey: 'linear.projects',
canonicalParamId: 'projectId',
mode: 'basic',
multi: true,
dependsOn: ['teamSelector'],
placeholder: 'Select one or more projects (optional)',
required: false,
},
{
id: 'projectId',
title: 'Project IDs',
type: 'short-input',
canonicalParamId: 'projectId',
mode: 'advanced',
multi: true,
placeholder: 'e.g. def456, ghi789 (comma-separated for multiple)',
required: false,
},
{
id: 'stateFilter',
title: 'State Filter',
type: 'short-input',
placeholder: 'e.g. In Progress, Todo',
required: false,
},
{
id: 'maxIssues',
title: 'Max Issues',
type: 'short-input',
required: false,
placeholder: 'e.g. 500 (default: unlimited)',
},
],
tagDefinitions: [
{ id: 'labels', displayName: 'Labels', fieldType: 'text' },
{ id: 'state', displayName: 'State', fieldType: 'text' },
{ id: 'priority', displayName: 'Priority', fieldType: 'text' },
{ id: 'assignee', displayName: 'Assignee', fieldType: 'text' },
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
],
}
@@ -0,0 +1 @@
export { microsoftTeamsConnector } from '@/connectors/microsoft-teams/microsoft-teams'
@@ -0,0 +1,77 @@
import { MicrosoftTeamsIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const DEFAULT_MAX_MESSAGES = 1000
export const microsoftTeamsConnectorMeta: ConnectorMeta = {
id: 'microsoft_teams',
name: 'Microsoft Teams',
description: 'Sync channel messages from Microsoft Teams',
version: '1.0.0',
icon: MicrosoftTeamsIcon,
auth: {
mode: 'oauth',
provider: 'microsoft-teams',
requiredScopes: ['ChannelMessage.Read.All', 'Channel.ReadBasic.All'],
},
configFields: [
{
id: 'teamSelector',
title: 'Team',
type: 'selector',
selectorKey: 'microsoft.teams',
canonicalParamId: 'teamId',
mode: 'basic',
placeholder: 'Select a team',
required: true,
},
{
id: 'teamId',
title: 'Team ID',
type: 'short-input',
canonicalParamId: 'teamId',
mode: 'advanced',
placeholder: 'e.g. fbe2bf47-16c8-47cf-b4a5-4b9b187c508b',
required: true,
description: 'The ID of the Microsoft Teams team',
},
{
id: 'channelSelector',
title: 'Channels',
type: 'selector',
selectorKey: 'microsoft.channels',
canonicalParamId: 'channel',
mode: 'basic',
multi: true,
dependsOn: ['teamSelector'],
placeholder: 'Select one or more channels',
required: true,
},
{
id: 'channel',
title: 'Channels',
type: 'short-input',
canonicalParamId: 'channel',
mode: 'advanced',
multi: true,
placeholder: 'e.g. General, Announcements (comma-separated for multiple)',
required: true,
description: 'Channel names or IDs to sync messages from',
},
{
id: 'maxMessages',
title: 'Max Messages',
type: 'short-input',
required: false,
placeholder: `e.g. 500 (default: ${DEFAULT_MAX_MESSAGES})`,
},
],
tagDefinitions: [
{ id: 'channelName', displayName: 'Channel Name', fieldType: 'text' },
{ id: 'messageCount', displayName: 'Message Count', fieldType: 'number' },
{ id: 'lastActivity', displayName: 'Last Activity', fieldType: 'date' },
],
}
@@ -0,0 +1,384 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import {
DEFAULT_MAX_MESSAGES,
microsoftTeamsConnectorMeta,
} from '@/connectors/microsoft-teams/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import {
computeContentHash,
htmlToPlainText,
parseMultiValue,
parseTagDate,
} from '@/connectors/utils'
const logger = createLogger('MicrosoftTeamsConnector')
const GRAPH_API_BASE = 'https://graph.microsoft.com/v1.0'
const MESSAGES_PER_PAGE = 50
interface TeamsMessage {
id: string
messageType: string
createdDateTime: string
lastModifiedDateTime?: string
deletedDateTime?: string | null
from?: {
user?: {
id: string
displayName: string
}
application?: {
id: string
displayName: string
}
}
body: {
contentType: string
content: string
}
subject?: string | null
}
interface TeamsChannel {
id: string
displayName: string
description?: string | null
}
interface TeamsMessagesResponse {
'@odata.nextLink'?: string
value: TeamsMessage[]
}
interface TeamsChannelsResponse {
'@odata.nextLink'?: string
value: TeamsChannel[]
}
/**
* Calls the Microsoft Graph API with the given path and access token.
*/
async function graphApiGet<T>(
path: string,
accessToken: string,
retryOptions?: Parameters<typeof fetchWithRetry>[2]
): Promise<T> {
const url = path.startsWith('https://') ? path : `${GRAPH_API_BASE}${path}`
const response = await fetchWithRetry(
url,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
Accept: 'application/json',
},
},
retryOptions
)
if (!response.ok) {
const errorBody = await response.text().catch(() => '')
throw new Error(`Microsoft Graph API error: ${response.status} ${errorBody}`)
}
return (await response.json()) as T
}
/**
* Fetches all messages from a channel, up to a maximum count, handling pagination.
*/
async function fetchChannelMessages(
accessToken: string,
teamId: string,
channelId: string,
maxMessages: number
): Promise<{ messages: TeamsMessage[]; lastActivityTs?: string }> {
const allMessages: TeamsMessage[] = []
let nextLink: string | undefined
let lastActivityTs: string | undefined
const initialPath = `/teams/${encodeURIComponent(teamId)}/channels/${encodeURIComponent(channelId)}/messages?$top=${Math.min(MESSAGES_PER_PAGE, maxMessages)}`
let currentUrl: string = initialPath
while (allMessages.length < maxMessages) {
const data = await graphApiGet<TeamsMessagesResponse>(currentUrl, accessToken)
const messages = data.value || []
if (messages.length === 0) break
// Filter to actual user messages (skip system/event messages)
const userMessages = messages.filter(
(msg) => msg.messageType === 'message' && !msg.deletedDateTime
)
// Messages are sorted by lastModifiedDateTime (per Graph docs), so the first
// user message on the first page reflects the most recent activity.
if (!lastActivityTs && userMessages.length > 0) {
const first = userMessages[0]
lastActivityTs = first.lastModifiedDateTime || first.createdDateTime
}
allMessages.push(...userMessages)
nextLink = data['@odata.nextLink']
if (!nextLink) break
currentUrl = nextLink
}
return { messages: allMessages.slice(0, maxMessages), lastActivityTs }
}
/**
* Converts fetched messages into a single document content string.
* Each line: "[ISO timestamp] username: message text"
*/
function formatMessages(messages: TeamsMessage[]): string {
const lines: string[] = []
// Process in reverse so oldest messages come first
const chronological = [...messages].reverse()
for (const msg of chronological) {
const bodyText =
msg.body.contentType === 'html' ? htmlToPlainText(msg.body.content) : msg.body.content
if (!bodyText.trim()) continue
const timestamp = msg.createdDateTime
const userName = msg.from?.user?.displayName || msg.from?.application?.displayName || 'unknown'
lines.push(`[${timestamp}] ${userName}: ${bodyText}`)
}
return lines.join('\n')
}
/**
* Resolves a channel name or ID to a channel object within the given team.
*/
async function resolveChannel(
accessToken: string,
teamId: string,
channelInput: string
): Promise<TeamsChannel | null> {
const trimmed = channelInput.trim()
// Fetch all channels for the team
let nextLink: string | undefined
// $select avoids the expensive `email` property per Graph perf guidance.
const initialPath = `/teams/${encodeURIComponent(teamId)}/channels?$select=id,displayName,description`
let currentUrl: string = initialPath
do {
const data = await graphApiGet<TeamsChannelsResponse>(currentUrl, accessToken)
const channels = data.value || []
// Try matching by ID first, then by display name (case-insensitive)
const match = channels.find(
(ch) => ch.id === trimmed || ch.displayName.toLowerCase() === trimmed.toLowerCase()
)
if (match) return match
nextLink = data['@odata.nextLink']
if (nextLink) {
currentUrl = nextLink
}
} while (nextLink)
return null
}
export const microsoftTeamsConnector: ConnectorConfig = {
...microsoftTeamsConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
_cursor?: string,
_syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const teamId = sourceConfig.teamId as string
const channelInputs = parseMultiValue(sourceConfig.channel)
if (!teamId?.trim()) {
throw new Error('Team ID is required')
}
if (channelInputs.length === 0) {
throw new Error('At least one channel is required')
}
const maxMessages = sourceConfig.maxMessages
? Number(sourceConfig.maxMessages)
: DEFAULT_MAX_MESSAGES
logger.info('Syncing Microsoft Teams channels', {
teamId,
channels: channelInputs,
maxMessages,
})
const documents: ExternalDocument[] = []
for (const channelInput of channelInputs) {
const channel = await resolveChannel(accessToken, teamId, channelInput)
if (!channel) {
throw new Error(`Channel not found: ${channelInput}`)
}
const { messages, lastActivityTs } = await fetchChannelMessages(
accessToken,
teamId,
channel.id,
maxMessages
)
const content = formatMessages(messages)
if (!content.trim()) {
logger.info(`No messages found in channel: ${channel.displayName}`)
continue
}
const contentHash = await computeContentHash(content)
const sourceUrl = `https://teams.microsoft.com/l/channel/${encodeURIComponent(channel.id)}/${encodeURIComponent(channel.displayName)}?groupId=${encodeURIComponent(teamId)}`
documents.push({
externalId: channel.id,
title: channel.displayName,
content,
mimeType: 'text/plain',
sourceUrl,
contentHash,
metadata: {
channelName: channel.displayName,
messageCount: messages.length,
lastActivity: lastActivityTs || undefined,
description: channel.description || undefined,
},
})
}
// All selected channels are emitted in a single page; no pagination needed
return {
documents,
hasMore: false,
}
},
getDocument: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
const teamId = sourceConfig.teamId as string
if (!teamId?.trim()) {
return null
}
const maxMessages = sourceConfig.maxMessages
? Number(sourceConfig.maxMessages)
: DEFAULT_MAX_MESSAGES
try {
// Fetch channel info
const channelPath = `/teams/${encodeURIComponent(teamId)}/channels/${encodeURIComponent(externalId)}`
const channel = await graphApiGet<TeamsChannel>(channelPath, accessToken)
const { messages, lastActivityTs } = await fetchChannelMessages(
accessToken,
teamId,
externalId,
maxMessages
)
const content = formatMessages(messages)
if (!content.trim()) return null
const contentHash = await computeContentHash(content)
const sourceUrl = `https://teams.microsoft.com/l/channel/${encodeURIComponent(channel.id)}/${encodeURIComponent(channel.displayName)}?groupId=${encodeURIComponent(teamId)}`
return {
externalId: channel.id,
title: channel.displayName,
content,
mimeType: 'text/plain',
sourceUrl,
contentHash,
metadata: {
channelName: channel.displayName,
messageCount: messages.length,
lastActivity: lastActivityTs || undefined,
description: channel.description || undefined,
},
}
} catch (error) {
logger.warn('Failed to get Microsoft Teams channel document', {
externalId,
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const teamId = sourceConfig.teamId as string | undefined
const channelInputs = parseMultiValue(sourceConfig.channel)
const maxMessages = sourceConfig.maxMessages as string | undefined
if (!teamId?.trim()) {
return { valid: false, error: 'Team ID is required' }
}
if (channelInputs.length === 0) {
return { valid: false, error: 'At least one channel is required' }
}
if (maxMessages && (Number.isNaN(Number(maxMessages)) || Number(maxMessages) <= 0)) {
return { valid: false, error: 'Max messages must be a positive number' }
}
try {
for (const channelInput of channelInputs) {
const channel = await resolveChannel(accessToken, teamId, channelInput)
if (!channel) {
return { valid: false, error: `Channel not found: ${channelInput}` }
}
// Verify we can read messages by fetching a single message
const messagesPath = `/teams/${encodeURIComponent(teamId)}/channels/${encodeURIComponent(channel.id)}/messages?$top=1`
await graphApiGet<TeamsMessagesResponse>(messagesPath, accessToken, VALIDATE_RETRY_OPTIONS)
}
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.channelName === 'string') {
result.channelName = metadata.channelName
}
if (typeof metadata.messageCount === 'number') {
result.messageCount = metadata.messageCount
}
const lastActivity = parseTagDate(metadata.lastActivity)
if (lastActivity) {
result.lastActivity = lastActivity
}
return result
},
}
+1
View File
@@ -0,0 +1 @@
export { mondayConnector } from '@/connectors/monday/monday'
+61
View File
@@ -0,0 +1,61 @@
import { MondayIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const mondayConnectorMeta: ConnectorMeta = {
id: 'monday',
name: 'Monday.com',
description: 'Sync board items and updates from Monday.com into your knowledge base',
version: '1.0.0',
icon: MondayIcon,
auth: {
mode: 'oauth',
provider: 'monday',
requiredScopes: ['boards:read', 'updates:read', 'me:read'],
},
configFields: [
{
id: 'boardSelector',
title: 'Boards',
type: 'selector',
selectorKey: 'monday.boards',
canonicalParamId: 'boardIds',
mode: 'basic',
multi: true,
required: false,
placeholder: 'Select boards (empty = all active boards)',
description:
'Boards to sync. Leave empty to sync items from every active board you can access.',
},
{
id: 'boardIds',
title: 'Board IDs',
type: 'short-input',
canonicalParamId: 'boardIds',
mode: 'advanced',
multi: true,
required: false,
placeholder: 'e.g. 1234567890, 9876543210 (empty = all active boards)',
description:
'Comma-separated board IDs to sync — find a board ID in its URL (.../boards/<id>). Leave empty to sync items from every active board you can access.',
},
{
id: 'maxItems',
title: 'Max Items',
type: 'short-input',
required: false,
placeholder: 'e.g. 500 (default: unlimited)',
},
],
tagDefinitions: [
{ id: 'boardName', displayName: 'Board', fieldType: 'text' },
{ id: 'groupTitle', displayName: 'Group', fieldType: 'text' },
{ id: 'itemName', displayName: 'Item', fieldType: 'text' },
{ id: 'state', displayName: 'State', fieldType: 'text' },
{ id: 'creatorName', displayName: 'Creator', fieldType: 'text' },
{ id: 'createdAt', displayName: 'Created', fieldType: 'date' },
{ id: 'updatedAt', displayName: 'Last Updated', fieldType: 'date' },
],
}
+514
View File
@@ -0,0 +1,514 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { mondayConnectorMeta } from '@/connectors/monday/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { parseMultiValue, parseTagDate } from '@/connectors/utils'
const logger = createLogger('MondayConnector')
/**
* monday.com GraphQL endpoint. All requests are POSTed here.
* @see https://developer.monday.com/api-reference/docs/basics
*/
const MONDAY_API_URL = 'https://api.monday.com/v2'
/**
* Stable monday.com API version pinned via the `API-Version` header. monday.com
* keeps at least three quarterly versions live; `2024-10` was deprecated on
* 2026-02-15, so this is pinned to the current stable release.
* @see https://developer.monday.com/api-reference/docs/api-versioning
*/
const MONDAY_API_VERSION = '2026-04'
/** Max items requested per `items_page` / `next_items_page` call (monday.com max is 500). */
const ITEMS_PAGE_SIZE = 100
/** Max boards requested per `boards` listing page (monday.com max is 500). */
const BOARDS_PAGE_SIZE = 100
/** Max updates fetched per item for content extraction. */
const UPDATES_LIMIT = 50
interface MondayColumnValue {
id: string
text: string | null
column: { id: string; title: string } | null
}
interface MondayUpdate {
id: string
text_body: string | null
created_at: string | null
creator: { name: string | null } | null
}
interface MondayItem {
id: string
name: string | null
state: string | null
created_at: string | null
updated_at: string | null
url: string | null
board: { id: string; name: string | null } | null
group: { id: string; title: string | null } | null
creator: { name: string | null } | null
column_values: MondayColumnValue[]
updates: MondayUpdate[]
}
interface MondayItemsPage {
cursor: string | null
items: MondayItem[]
}
interface MondayBoard {
id: string
name: string | null
items_page: MondayItemsPage | null
}
/**
* Pagination state encoded into `nextCursor`. Tracks which board in the
* configured/accessible list is being read (`boardIndex`) and the opaque
* `items_page` cursor within that board (`itemsCursor`).
*/
interface CursorState {
boardIndex: number
itemsCursor?: string
}
function encodeCursor(state: CursorState): string {
return Buffer.from(JSON.stringify(state), 'utf8').toString('base64url')
}
function decodeCursor(cursor?: string): CursorState {
if (!cursor) return { boardIndex: 0 }
try {
const json = Buffer.from(cursor, 'base64url').toString('utf8')
const parsed = JSON.parse(json) as Partial<CursorState>
return {
boardIndex: Number(parsed.boardIndex) || 0,
itemsCursor: typeof parsed.itemsCursor === 'string' ? parsed.itemsCursor : undefined,
}
} catch {
return { boardIndex: 0 }
}
}
/**
* monday.com uses the raw access token in the `Authorization` header — it is NOT
* prefixed with "Bearer". The `API-Version` header pins the schema version.
* @see https://developer.monday.com/api-reference/docs/authentication
*/
function mondayHeaders(accessToken: string): Record<string, string> {
return {
'Content-Type': 'application/json',
Authorization: accessToken,
'API-Version': MONDAY_API_VERSION,
}
}
/**
* Executes a GraphQL query against the monday.com API, surfacing GraphQL-level
* errors (which return HTTP 200 with an `errors` array) as thrown errors.
*/
async function mondayGraphQL<T>(
accessToken: string,
query: string,
variables: Record<string, unknown> = {},
retryOptions?: Parameters<typeof fetchWithRetry>[2]
): Promise<T> {
const response = await fetchWithRetry(
MONDAY_API_URL,
{
method: 'POST',
headers: mondayHeaders(accessToken),
body: JSON.stringify({ query, variables }),
},
retryOptions
)
if (!response.ok) {
const errorText = await response.text().catch(() => '')
throw new Error(
`monday.com API HTTP error: ${response.status}${errorText ? `${errorText.slice(0, 200)}` : ''}`
)
}
const data = (await response.json()) as {
data?: T
errors?: { message?: string }[]
error_message?: string
}
if (data.errors && data.errors.length > 0) {
const message = data.errors
.map((e) => e.message)
.filter(Boolean)
.join('; ')
throw new Error(`monday.com API error: ${message || 'Unknown GraphQL error'}`)
}
if (data.error_message) {
throw new Error(`monday.com API error: ${data.error_message}`)
}
return data.data as T
}
/**
* GraphQL selection set for an item, shared between listing and single-item
* fetches so the resolved fields stay in sync.
*/
const ITEM_FIELDS = `
id
name
state
created_at
updated_at
url
board { id name }
group { id title }
creator { name }
column_values {
id
text
column { id title }
}
updates(limit: ${UPDATES_LIMIT}) {
id
text_body
created_at
creator { name }
}
`
/**
* Builds the change-detection hash from item identity + last-modified time. Must
* be identical whether produced inline during listing or via a `getDocument`
* fetch, since monday's `updated_at` advances on any item or column change.
*/
function buildContentHash(itemId: string, updatedAt: string | null | undefined): string {
return `monday:${itemId}:${updatedAt ?? ''}`
}
/**
* Resolves a stable board name + id for an item, preferring the nested `board`
* field and falling back to the board the item was listed under.
*/
function resolveBoard(
item: MondayItem,
fallback?: { id: string; name: string | null }
): { id: string; name: string } {
const id = item.board?.id ?? fallback?.id ?? ''
const name = item.board?.name ?? fallback?.name ?? ''
return { id, name }
}
function buildSourceUrl(item: MondayItem): string | undefined {
return item.url ?? undefined
}
/**
* Builds the metadata object carried on every document, used both for tag mapping
* (`mapTags`) and for downstream display. Kept in one place so listing and
* single-item fetches emit identical metadata.
*/
function itemMetadata(
item: MondayItem,
board: { id: string; name: string }
): Record<string, unknown> {
return {
boardId: board.id,
boardName: board.name,
itemName: item.name ?? '',
groupTitle: item.group?.title ?? '',
state: item.state ?? '',
creatorName: item.creator?.name ?? '',
createdAt: item.created_at ?? undefined,
updatedAt: item.updated_at ?? undefined,
}
}
/**
* Produces a fully-hydrated document from a single `items_page` element. The list
* query already selects the complete item payload (`column_values` + `updates`),
* so content is built inline here — avoiding a redundant per-item `getDocument`
* round-trip. `contentHash` derives solely from id + `updated_at`, so it is
* identical whether produced here or via `getDocument`.
*/
function itemToDocument(
item: MondayItem,
fallbackBoard?: { id: string; name: string | null }
): ExternalDocument {
const board = resolveBoard(item, fallbackBoard)
return {
externalId: item.id,
title: item.name?.trim() || 'Untitled Item',
content: formatItemContent(item, board),
contentDeferred: false,
mimeType: 'text/plain',
sourceUrl: buildSourceUrl(item),
contentHash: buildContentHash(item.id, item.updated_at),
metadata: itemMetadata(item, board),
}
}
/**
* Formats an item's column values and updates into a plain-text document. The
* resolved board is passed in so listing (which has a board fallback) and
* single-item fetches produce identical content.
*/
function formatItemContent(item: MondayItem, board: { id: string; name: string }): string {
const parts: string[] = []
if (board.name) parts.push(`Board: ${board.name}`)
parts.push(`Item: ${item.name?.trim() || 'Untitled Item'}`)
if (item.group?.title) parts.push(`Group: ${item.group.title}`)
if (item.creator?.name) parts.push(`Created by: ${item.creator.name}`)
if (item.created_at) parts.push(`Created: ${item.created_at}`)
if (item.updated_at) parts.push(`Updated: ${item.updated_at}`)
const columns = item.column_values.filter((cv) => cv.text?.trim())
if (columns.length > 0) {
parts.push('')
parts.push('--- Fields ---')
for (const cv of columns) {
const title = cv.column?.title?.trim() || cv.id
parts.push(`${title}: ${cv.text}`)
}
}
const updates = item.updates.filter((u) => u.text_body?.trim())
if (updates.length > 0) {
parts.push('')
parts.push('--- Updates ---')
for (const update of updates) {
const author = update.creator?.name?.trim() || 'Unknown'
parts.push(`Update by ${author}: ${update.text_body}`)
}
}
return parts.join('\n')
}
/**
* Fetches the list of board ids the connector should sync. When `boardIds` is
* configured, those are used verbatim; otherwise all accessible active boards
* are enumerated.
*/
async function resolveBoardIds(
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ id: string; name: string | null }[]> {
const configured = parseMultiValue(sourceConfig.boardIds)
if (configured.length > 0) {
return configured.map((id) => ({ id, name: null }))
}
const boards: { id: string; name: string | null }[] = []
let page = 1
for (;;) {
const data = await mondayGraphQL<{ boards: { id: string; name: string | null }[] | null }>(
accessToken,
`query ($limit: Int!, $page: Int!) {
boards(limit: $limit, page: $page, state: active) {
id
name
}
}`,
{ limit: BOARDS_PAGE_SIZE, page }
)
const batch = data.boards ?? []
boards.push(...batch)
if (batch.length < BOARDS_PAGE_SIZE) break
page += 1
}
return boards
}
export const mondayConnector: ConnectorConfig = {
...mondayConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const maxItems = sourceConfig.maxItems ? Number(sourceConfig.maxItems) : 0
const state = decodeCursor(cursor)
const boards =
(syncContext?.boards as { id: string; name: string | null }[] | undefined) ??
(await resolveBoardIds(accessToken, sourceConfig))
if (syncContext) syncContext.boards = boards
if (state.boardIndex >= boards.length) {
return { documents: [], hasMore: false }
}
const board = boards[state.boardIndex]
const fallbackBoard = { id: board.id, name: board.name }
const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
const pageLimit =
maxItems > 0
? Math.min(ITEMS_PAGE_SIZE, Math.max(1, maxItems - prevFetched))
: ITEMS_PAGE_SIZE
let itemsPage: MondayItemsPage | null
if (state.itemsCursor) {
const data = await mondayGraphQL<{ next_items_page: MondayItemsPage | null }>(
accessToken,
`query ($cursor: String!, $limit: Int!) {
next_items_page(cursor: $cursor, limit: $limit) {
cursor
items { ${ITEM_FIELDS} }
}
}`,
{ cursor: state.itemsCursor, limit: pageLimit }
)
itemsPage = data.next_items_page
} else {
const data = await mondayGraphQL<{ boards: MondayBoard[] | null }>(
accessToken,
`query ($ids: [ID!], $limit: Int!) {
boards(ids: $ids) {
id
name
items_page(limit: $limit) {
cursor
items { ${ITEM_FIELDS} }
}
}
}`,
{ ids: [board.id], limit: pageLimit }
)
itemsPage = data.boards?.[0]?.items_page ?? null
}
const items = itemsPage?.items ?? []
const nextItemsCursor = itemsPage?.cursor?.trim() || undefined
logger.info('Listing Monday.com items', {
boardIndex: state.boardIndex,
boardTotal: boards.length,
boardId: board.id,
itemCount: items.length,
hasItemsCursor: Boolean(state.itemsCursor),
})
const allDocuments = items.map((item) => itemToDocument(item, fallbackBoard))
let documents = allDocuments
if (maxItems > 0) {
const remaining = Math.max(0, maxItems - prevFetched)
if (allDocuments.length > remaining) {
documents = allDocuments.slice(0, remaining)
}
}
const totalFetched = prevFetched + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = maxItems > 0 && totalFetched >= maxItems
if (hitLimit && syncContext) syncContext.listingCapped = true
let nextCursor: string | undefined
let hasMore = false
if (hitLimit) {
nextCursor = undefined
} else if (nextItemsCursor) {
nextCursor = encodeCursor({ boardIndex: state.boardIndex, itemsCursor: nextItemsCursor })
hasMore = true
} else if (state.boardIndex + 1 < boards.length) {
nextCursor = encodeCursor({ boardIndex: state.boardIndex + 1 })
hasMore = true
}
return { documents, nextCursor, hasMore }
},
getDocument: async (
accessToken: string,
_sourceConfig: Record<string, unknown>,
externalId: string
): Promise<ExternalDocument | null> => {
try {
if (!externalId) return null
const data = await mondayGraphQL<{ items: MondayItem[] | null }>(
accessToken,
`query ($ids: [ID!]) {
items(ids: $ids) { ${ITEM_FIELDS} }
}`,
{ ids: [externalId] }
)
const item = data.items?.[0]
if (!item) return null
const doc = itemToDocument(item)
if (!doc.content.trim()) return null
return doc
} catch (error) {
logger.warn('Failed to get Monday.com item', {
externalId,
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const maxItems = sourceConfig.maxItems as string | undefined
if (maxItems && (Number.isNaN(Number(maxItems)) || Number(maxItems) < 0)) {
return { valid: false, error: 'Max items must be a non-negative number' }
}
try {
await mondayGraphQL(accessToken, `query { me { id } }`, {}, VALIDATE_RETRY_OPTIONS)
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
if (typeof metadata.boardName === 'string' && metadata.boardName.trim()) {
result.boardName = metadata.boardName
}
if (typeof metadata.groupTitle === 'string' && metadata.groupTitle.trim()) {
result.groupTitle = metadata.groupTitle
}
if (typeof metadata.itemName === 'string' && metadata.itemName.trim()) {
result.itemName = metadata.itemName
}
if (typeof metadata.state === 'string' && metadata.state.trim()) {
result.state = metadata.state
}
if (typeof metadata.creatorName === 'string' && metadata.creatorName.trim()) {
result.creatorName = metadata.creatorName
}
const createdAt = parseTagDate(metadata.createdAt)
if (createdAt) result.createdAt = createdAt
const updatedAt = parseTagDate(metadata.updatedAt)
if (updatedAt) result.updatedAt = updatedAt
return result
},
}
+1
View File
@@ -0,0 +1 @@
export { notionConnector } from '@/connectors/notion/notion'
+86
View File
@@ -0,0 +1,86 @@
import { NotionIcon } from '@/components/icons'
import type { ConnectorMeta } from '@/connectors/types'
export const notionConnectorMeta: ConnectorMeta = {
id: 'notion',
name: 'Notion',
description: 'Sync pages from a Notion workspace',
version: '1.0.0',
icon: NotionIcon,
auth: { mode: 'oauth', provider: 'notion', requiredScopes: [] },
configFields: [
{
id: 'scope',
title: 'Sync Scope',
type: 'dropdown',
required: false,
options: [
{ label: 'Entire workspace', id: 'workspace' },
{ label: 'Specific database', id: 'database' },
{ label: 'Specific page (and children)', id: 'page' },
],
},
{
id: 'databaseSelector',
title: 'Databases',
type: 'selector',
selectorKey: 'notion.databases',
canonicalParamId: 'databaseId',
mode: 'basic',
multi: true,
placeholder: 'Select one or more databases',
required: false,
},
{
id: 'databaseId',
title: 'Database IDs',
type: 'short-input',
canonicalParamId: 'databaseId',
mode: 'advanced',
multi: true,
required: false,
placeholder: 'e.g. 8a3b5f6e-..., 9c4d6e7f-... (comma-separated for multiple)',
},
{
id: 'rootPageSelector',
title: 'Page',
type: 'selector',
selectorKey: 'notion.pages',
canonicalParamId: 'rootPageId',
mode: 'basic',
placeholder: 'Select a page',
required: false,
},
{
id: 'rootPageId',
title: 'Page ID',
type: 'short-input',
canonicalParamId: 'rootPageId',
mode: 'advanced',
required: false,
placeholder: 'e.g. 8a3b5f6e-1234-5678-abcd-ef0123456789',
},
{
id: 'searchQuery',
title: 'Search Filter',
type: 'short-input',
required: false,
placeholder: 'e.g. meeting notes, project plan',
},
{
id: 'maxPages',
title: 'Max Pages',
type: 'short-input',
required: false,
placeholder: 'e.g. 500 (default: unlimited)',
},
],
tagDefinitions: [
{ id: 'tags', displayName: 'Tags', fieldType: 'text' },
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
{ id: 'created', displayName: 'Created', fieldType: 'date' },
],
}
+634
View File
@@ -0,0 +1,634 @@
import { createLogger } from '@sim/logger'
import { getErrorMessage, toError } from '@sim/utils/errors'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { notionConnectorMeta } from '@/connectors/notion/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils'
const logger = createLogger('NotionConnector')
const NOTION_API_VERSION = '2022-06-28'
const NOTION_BASE_URL = 'https://api.notion.com/v1'
/**
* Extracts the title from a Notion page's properties.
*/
function extractTitle(properties: Record<string, unknown>): string {
for (const value of Object.values(properties)) {
const prop = value as Record<string, unknown>
if (prop.type === 'title' && Array.isArray(prop.title) && prop.title.length > 0) {
return prop.title.map((t: Record<string, unknown>) => (t.plain_text as string) || '').join('')
}
}
return 'Untitled'
}
/**
* Extracts plain text from a rich_text array.
*/
function richTextToPlain(richText: Record<string, unknown>[]): string {
return richText.map((t) => (t.plain_text as string) || '').join('')
}
/**
* Extracts plain text content from Notion blocks.
*/
function blocksToPlainText(blocks: Record<string, unknown>[]): string {
return blocks
.map((block) => {
const type = block.type as string
const blockData = block[type] as Record<string, unknown> | undefined
if (!blockData) return ''
if (type === 'code') {
const richText = blockData.rich_text as Record<string, unknown>[] | undefined
const language = (blockData.language as string) || ''
const code = richText ? richTextToPlain(richText) : ''
return language ? `\`\`\`${language}\n${code}\n\`\`\`` : `\`\`\`\n${code}\n\`\`\``
}
if (type === 'equation') {
const expression = (blockData.expression as string) || ''
return expression ? `$$${expression}$$` : ''
}
const richText = blockData.rich_text as Record<string, unknown>[] | undefined
if (!richText) return ''
const text = richTextToPlain(richText)
switch (type) {
case 'heading_1':
return `# ${text}`
case 'heading_2':
return `## ${text}`
case 'heading_3':
return `### ${text}`
case 'bulleted_list_item':
return `- ${text}`
case 'numbered_list_item':
return `1. ${text}`
case 'to_do': {
const checked = (blockData.checked as boolean) ? '[x]' : '[ ]'
return `${checked} ${text}`
}
case 'quote':
return `> ${text}`
case 'callout':
return text
case 'toggle':
return text
default:
return text
}
})
.filter(Boolean)
.join('\n\n')
}
/**
* Fetches all block children for a page, handling pagination.
*/
async function fetchAllBlocks(
accessToken: string,
pageId: string
): Promise<Record<string, unknown>[]> {
const allBlocks: Record<string, unknown>[] = []
let cursor: string | undefined
let hasMore = true
while (hasMore) {
const params = new URLSearchParams({ page_size: '100' })
if (cursor) params.append('start_cursor', cursor)
const response = await fetchWithRetry(
`${NOTION_BASE_URL}/blocks/${pageId}/children?${params.toString()}`,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
'Notion-Version': NOTION_API_VERSION,
},
}
)
if (!response.ok) {
logger.warn(`Failed to fetch blocks for page ${pageId}`, { status: response.status })
break
}
const data = await response.json()
allBlocks.push(...(data.results || []))
cursor = data.next_cursor ?? undefined
hasMore = data.has_more === true
}
return allBlocks
}
/**
* Extracts multi_select tags from page properties.
*/
function extractTags(properties: Record<string, unknown>): string[] {
const tags: string[] = []
for (const value of Object.values(properties)) {
const prop = value as Record<string, unknown>
if (prop.type === 'multi_select' && Array.isArray(prop.multi_select)) {
for (const item of prop.multi_select) {
const name = (item as Record<string, unknown>).name as string
if (name) tags.push(name)
}
}
if (prop.type === 'select' && prop.select) {
const name = (prop.select as Record<string, unknown>).name as string
if (name) tags.push(name)
}
}
return tags
}
/**
* Converts a Notion page to a lightweight metadata stub (no content fetching).
*/
function pageToStub(page: Record<string, unknown>): ExternalDocument {
const pageId = page.id as string
const properties = (page.properties || {}) as Record<string, unknown>
const title = extractTitle(properties)
const url = page.url as string
const lastEditedTime = (page.last_edited_time as string) ?? ''
const tags = extractTags(properties)
return {
externalId: pageId,
title: title || 'Untitled',
content: '',
contentDeferred: true,
mimeType: 'text/plain',
sourceUrl: url,
contentHash: `notion:${pageId}:${lastEditedTime}`,
metadata: {
tags,
lastModified: page.last_edited_time as string,
createdTime: page.created_time as string,
parentType: (page.parent as Record<string, unknown>)?.type,
},
}
}
export const notionConnector: ConnectorConfig = {
...notionConnectorMeta,
listDocuments: async (
accessToken: string,
sourceConfig: Record<string, unknown>,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> => {
const scope = (sourceConfig.scope as string) || 'workspace'
const databaseIds = parseMultiValue(sourceConfig.databaseId)
const rootPageId = (sourceConfig.rootPageId as string)?.trim()
const maxPages = sourceConfig.maxPages ? Number(sourceConfig.maxPages) : 0
if (scope === 'database' && databaseIds.length > 0) {
return listFromDatabases(accessToken, databaseIds, maxPages, cursor, syncContext)
}
if (scope === 'page' && rootPageId) {
return listFromParentPage(accessToken, rootPageId, maxPages, cursor, syncContext)
}
// Default: workspace-wide search
const searchQuery = (sourceConfig.searchQuery as string) || ''
return listFromWorkspace(accessToken, searchQuery, maxPages, cursor, syncContext)
},
getDocument: async (
accessToken: string,
_sourceConfig: Record<string, unknown>,
externalId: string,
_syncContext?: Record<string, unknown>
): Promise<ExternalDocument | null> => {
const response = await fetchWithRetry(`${NOTION_BASE_URL}/pages/${externalId}`, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
'Notion-Version': NOTION_API_VERSION,
},
})
if (!response.ok) {
if (response.status === 404) return null
throw new Error(`Failed to get Notion page: ${response.status}`)
}
const page = await response.json()
if (page.archived) return null
try {
const blocks = await fetchAllBlocks(accessToken, externalId)
const blockContent = blocksToPlainText(blocks)
const stub = pageToStub(page)
const content = blockContent.trim() || stub.title
return { ...stub, content, contentDeferred: false }
} catch (error) {
logger.warn(`Failed to fetch content for Notion page: ${externalId}`, {
error: toError(error).message,
})
return null
}
},
validateConfig: async (
accessToken: string,
sourceConfig: Record<string, unknown>
): Promise<{ valid: boolean; error?: string }> => {
const scope = (sourceConfig.scope as string) || 'workspace'
const databaseIds = parseMultiValue(sourceConfig.databaseId)
const rootPageId = (sourceConfig.rootPageId as string)?.trim()
const maxPages = sourceConfig.maxPages as string | undefined
if (maxPages && (Number.isNaN(Number(maxPages)) || Number(maxPages) <= 0)) {
return { valid: false, error: 'Max pages must be a positive number' }
}
if (scope === 'database' && databaseIds.length === 0) {
return {
valid: false,
error: 'At least one database is required when scope is "Specific database"',
}
}
if (scope === 'page' && !rootPageId) {
return { valid: false, error: 'Page ID is required when scope is "Specific page"' }
}
try {
// Verify the token works
if (scope === 'database' && databaseIds.length > 0) {
// Verify every database is accessible
for (const databaseId of databaseIds) {
const response = await fetchWithRetry(
`${NOTION_BASE_URL}/databases/${databaseId}`,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
'Notion-Version': NOTION_API_VERSION,
},
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
return {
valid: false,
error: `Cannot access database ${databaseId}: ${response.status}`,
}
}
}
} else if (scope === 'page' && rootPageId) {
// Verify page is accessible
const response = await fetchWithRetry(
`${NOTION_BASE_URL}/pages/${rootPageId}`,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
'Notion-Version': NOTION_API_VERSION,
},
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
return { valid: false, error: `Cannot access page: ${response.status}` }
}
} else {
// Workspace scope — just verify token works
const response = await fetchWithRetry(
`${NOTION_BASE_URL}/search`,
{
method: 'POST',
headers: {
Authorization: `Bearer ${accessToken}`,
'Notion-Version': NOTION_API_VERSION,
'Content-Type': 'application/json',
},
body: JSON.stringify({ page_size: 1 }),
},
VALIDATE_RETRY_OPTIONS
)
if (!response.ok) {
const errorText = await response.text()
return { valid: false, error: `Cannot access Notion workspace: ${errorText}` }
}
}
return { valid: true }
} catch (error) {
const message = getErrorMessage(error, 'Failed to validate configuration')
return { valid: false, error: message }
}
},
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
const result: Record<string, unknown> = {}
const tags = joinTagArray(metadata.tags)
if (tags) result.tags = tags
const lastModified = parseTagDate(metadata.lastModified)
if (lastModified) result.lastModified = lastModified
const created = parseTagDate(metadata.createdTime)
if (created) result.created = created
return result
},
}
/**
* Lists pages from the entire workspace using the search API.
*/
async function listFromWorkspace(
accessToken: string,
searchQuery: string,
maxPages: number,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> {
const body: Record<string, unknown> = {
page_size: 100,
filter: { value: 'page', property: 'object' },
sort: { direction: 'descending', timestamp: 'last_edited_time' },
}
if (searchQuery.trim()) {
body.query = searchQuery.trim()
}
if (cursor) {
body.start_cursor = cursor
}
logger.info('Listing Notion pages from workspace', { searchQuery, cursor })
const response = await fetchWithRetry(`${NOTION_BASE_URL}/search`, {
method: 'POST',
headers: {
Authorization: `Bearer ${accessToken}`,
'Notion-Version': NOTION_API_VERSION,
'Content-Type': 'application/json',
},
body: JSON.stringify(body),
})
if (!response.ok) {
const errorText = await response.text()
logger.error('Failed to search Notion', { status: response.status, error: errorText })
throw new Error(`Failed to search Notion: ${response.status}`)
}
const data = await response.json()
const results = (data.results || []) as Record<string, unknown>[]
const pages = results.filter((r) => r.object === 'page' && !(r.archived as boolean))
const documents = pages.map(pageToStub)
const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = maxPages > 0 && totalFetched >= maxPages
if (hitLimit && syncContext) syncContext.listingCapped = true
const nextCursor = hitLimit ? undefined : ((data.next_cursor as string) ?? undefined)
return {
documents,
nextCursor,
hasMore: hitLimit ? false : data.has_more === true,
}
}
/**
* Lists pages from one or more Notion databases.
*
* Notion's `/v1/databases/{database_id}/query` endpoint is per-database — there
* is no batch query endpoint — so multiple databases are walked sequentially.
*
* Cursor format:
* - Single database: the Notion `start_cursor` string directly, or undefined.
* - Multiple databases: JSON-encoded `{ databaseIndex, cursor }` where
* `databaseIndex` is the position into `databaseIds` currently being drained
* and `cursor` is the Notion `start_cursor` for that database (or undefined
* when starting a fresh database).
*
* Page IDs returned by Notion are globally-unique UUIDs, so each page's
* `externalId` does not need to be namespaced by database.
*/
async function listFromDatabases(
accessToken: string,
databaseIds: string[],
maxPages: number,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> {
let databaseIndex = 0
let startCursor: string | undefined
if (cursor) {
if (databaseIds.length === 1) {
// Single-database path: cursor is always a bare Notion `next_cursor` string,
// matching the legacy pre-multi-select format. Never JSON-decode here.
startCursor = cursor
} else {
try {
const parsed = JSON.parse(cursor) as unknown
if (
parsed &&
typeof parsed === 'object' &&
typeof (parsed as { databaseIndex?: unknown }).databaseIndex === 'number'
) {
const compound = parsed as { databaseIndex: number; cursor?: string }
databaseIndex = compound.databaseIndex
startCursor = typeof compound.cursor === 'string' ? compound.cursor : undefined
} else {
// Legacy single-DB cursor carried forward into a now-multi-DB config:
// treat it as the start cursor for the first database.
startCursor = cursor
}
} catch {
startCursor = cursor
}
}
}
const documents: ExternalDocument[] = []
let nextCursor: string | undefined
let hasMore = false
while (databaseIndex < databaseIds.length) {
const databaseId = databaseIds[databaseIndex]
const body: Record<string, unknown> = { page_size: 100 }
if (startCursor) body.start_cursor = startCursor
logger.info('Querying Notion database', {
databaseId,
databaseIndex,
databaseCount: databaseIds.length,
startCursor,
})
const response = await fetchWithRetry(`${NOTION_BASE_URL}/databases/${databaseId}/query`, {
method: 'POST',
headers: {
Authorization: `Bearer ${accessToken}`,
'Notion-Version': NOTION_API_VERSION,
'Content-Type': 'application/json',
},
body: JSON.stringify(body),
})
if (!response.ok) {
const errorText = await response.text()
logger.error('Failed to query Notion database', {
databaseId,
status: response.status,
error: errorText,
})
throw new Error(`Failed to query Notion database ${databaseId}: ${response.status}`)
}
const data = await response.json()
const results = (data.results || []) as Record<string, unknown>[]
const pages = results.filter((r) => r.object === 'page' && !(r.archived as boolean))
documents.push(...pages.map(pageToStub))
if (data.has_more === true && typeof data.next_cursor === 'string') {
const nextStart = data.next_cursor as string
nextCursor =
databaseIds.length === 1 ? nextStart : JSON.stringify({ databaseIndex, cursor: nextStart })
hasMore = true
break
}
databaseIndex++
startCursor = undefined
if (databaseIndex < databaseIds.length) {
nextCursor =
databaseIds.length === 1 ? undefined : JSON.stringify({ databaseIndex, cursor: undefined })
hasMore = true
break
}
}
const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = maxPages > 0 && totalFetched >= maxPages
if (hitLimit) {
if (syncContext) syncContext.listingCapped = true
hasMore = false
nextCursor = undefined
}
return {
documents,
nextCursor: hasMore ? nextCursor : undefined,
hasMore,
}
}
/**
* Lists child pages under a specific parent page.
*
* Uses the blocks children endpoint to find child_page blocks,
* then fetches each page's metadata to build lightweight stubs.
*/
async function listFromParentPage(
accessToken: string,
rootPageId: string,
maxPages: number,
cursor?: string,
syncContext?: Record<string, unknown>
): Promise<ExternalDocumentList> {
const params = new URLSearchParams({ page_size: '100' })
if (cursor) params.append('start_cursor', cursor)
logger.info('Listing child pages under root page', { rootPageId, cursor })
const response = await fetchWithRetry(
`${NOTION_BASE_URL}/blocks/${rootPageId}/children?${params.toString()}`,
{
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
'Notion-Version': NOTION_API_VERSION,
},
}
)
if (!response.ok) {
const errorText = await response.text()
logger.error('Failed to list child blocks', { status: response.status, error: errorText })
throw new Error(`Failed to list child blocks: ${response.status}`)
}
const data = await response.json()
const blockResults = (data.results || []) as Record<string, unknown>[]
// Filter to child_page blocks only (child_database blocks cannot be fetched via the Pages API)
const childPageIds = blockResults
.filter((b) => b.type === 'child_page')
.map((b) => b.id as string)
// Also include the root page itself on the first call (no cursor)
const pageIdsToFetch = !cursor ? [rootPageId, ...childPageIds] : childPageIds
// Fetch page metadata (not content) in concurrent batches to build stubs
const CHILD_PAGE_CONCURRENCY = 5
const documents: ExternalDocument[] = []
for (let i = 0; i < pageIdsToFetch.length; i += CHILD_PAGE_CONCURRENCY) {
const cumulativeSoFar = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length
if (maxPages > 0 && cumulativeSoFar >= maxPages) break
const batch = pageIdsToFetch.slice(i, i + CHILD_PAGE_CONCURRENCY)
const results = await Promise.all(
batch.map(async (pageId) => {
try {
const pageResponse = await fetchWithRetry(`${NOTION_BASE_URL}/pages/${pageId}`, {
method: 'GET',
headers: {
Authorization: `Bearer ${accessToken}`,
'Notion-Version': NOTION_API_VERSION,
},
})
if (!pageResponse.ok) {
logger.warn(`Failed to fetch child page ${pageId}`, { status: pageResponse.status })
return null
}
const page = await pageResponse.json()
if (page.archived) return null
return pageToStub(page)
} catch (error) {
logger.warn(`Failed to process child page ${pageId}`, {
error: toError(error).message,
})
return null
}
})
)
documents.push(...(results.filter(Boolean) as ExternalDocument[]))
}
const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length
if (syncContext) syncContext.totalDocsFetched = totalFetched
const hitLimit = maxPages > 0 && totalFetched >= maxPages
if (hitLimit && syncContext) syncContext.listingCapped = true
const nextCursor = hitLimit ? undefined : ((data.next_cursor as string) ?? undefined)
return {
documents,
nextCursor,
hasMore: hitLimit ? false : data.has_more === true,
}
}

Some files were not shown because too many files have changed in this diff Show More