chore: import upstream snapshot with attribution
Publish CLI Package / publish-npm (push) Waiting to run
Publish Python SDK / publish-pypi (push) Waiting to run
Publish TypeScript SDK / publish-npm (push) Waiting to run
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Waiting to run
Publish Python SDK / publish-pypi (push) Waiting to run
Publish TypeScript SDK / publish-npm (push) Waiting to run
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,339 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { airtableConnectorMeta } from '@/connectors/airtable/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { computeContentHash, parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('AirtableConnector')
|
||||
|
||||
const AIRTABLE_API = 'https://api.airtable.com/v0'
|
||||
const PAGE_SIZE = 100
|
||||
|
||||
/**
|
||||
* Flattens a record's fields into a plain-text representation.
|
||||
* Each field is rendered as "Field Name: value" on its own line.
|
||||
*/
|
||||
function recordToPlainText(
|
||||
fields: Record<string, unknown>,
|
||||
fieldNames?: Map<string, string>
|
||||
): string {
|
||||
const lines: string[] = []
|
||||
for (const [key, value] of Object.entries(fields)) {
|
||||
if (value == null) continue
|
||||
const displayName = fieldNames?.get(key) ?? key
|
||||
if (Array.isArray(value)) {
|
||||
// Attachments or linked records
|
||||
const items = value.map((v) => {
|
||||
if (typeof v === 'object' && v !== null) {
|
||||
const obj = v as Record<string, unknown>
|
||||
return (obj.url as string) || (obj.name as string) || JSON.stringify(v)
|
||||
}
|
||||
return String(v)
|
||||
})
|
||||
lines.push(`${displayName}: ${items.join(', ')}`)
|
||||
} else if (typeof value === 'object') {
|
||||
lines.push(`${displayName}: ${JSON.stringify(value)}`)
|
||||
} else {
|
||||
lines.push(`${displayName}: ${String(value)}`)
|
||||
}
|
||||
}
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts a human-readable title from a record's fields.
|
||||
* Prefers the configured title field, then falls back to common field names.
|
||||
*/
|
||||
function extractTitle(fields: Record<string, unknown>, titleField?: string): string {
|
||||
if (titleField && fields[titleField] != null) {
|
||||
return String(fields[titleField])
|
||||
}
|
||||
const candidates = ['Name', 'Title', 'name', 'title', 'Summary', 'summary']
|
||||
for (const candidate of candidates) {
|
||||
if (fields[candidate] != null) {
|
||||
return String(fields[candidate])
|
||||
}
|
||||
}
|
||||
for (const value of Object.values(fields)) {
|
||||
if (typeof value === 'string' && value.trim()) {
|
||||
return value.length > 80 ? `${value.slice(0, 80)}…` : value
|
||||
}
|
||||
}
|
||||
return 'Untitled'
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the cursor format: "offset:<airtable_offset>"
|
||||
*/
|
||||
function parseCursor(cursor?: string): string | undefined {
|
||||
if (!cursor) return undefined
|
||||
if (cursor.startsWith('offset:')) return cursor.slice(7)
|
||||
return cursor
|
||||
}
|
||||
|
||||
export const airtableConnector: ConnectorConfig = {
|
||||
...airtableConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const baseId = sourceConfig.baseId as string
|
||||
const tableIdOrName = sourceConfig.tableIdOrName as string
|
||||
const viewId = sourceConfig.viewId as string | undefined
|
||||
const titleField = sourceConfig.titleField as string | undefined
|
||||
const maxRecords = sourceConfig.maxRecords ? Number(sourceConfig.maxRecords) : 0
|
||||
|
||||
const fieldNames = await fetchFieldNames(accessToken, baseId, tableIdOrName, syncContext)
|
||||
|
||||
const params = new URLSearchParams()
|
||||
params.append('pageSize', String(PAGE_SIZE))
|
||||
if (viewId) params.append('view', viewId)
|
||||
if (maxRecords > 0) params.append('maxRecords', String(maxRecords))
|
||||
|
||||
const offset = parseCursor(cursor)
|
||||
if (offset) params.append('offset', offset)
|
||||
|
||||
const encodedTable = encodeURIComponent(tableIdOrName)
|
||||
const url = `${AIRTABLE_API}/${baseId}/${encodedTable}?${params.toString()}`
|
||||
|
||||
logger.info(`Listing records from ${baseId}/${tableIdOrName}`, {
|
||||
offset: offset ?? 'none',
|
||||
view: viewId ?? 'default',
|
||||
})
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error('Failed to list Airtable records', {
|
||||
status: response.status,
|
||||
error: errorText,
|
||||
})
|
||||
throw new Error(`Failed to list Airtable records: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as {
|
||||
records: AirtableRecord[]
|
||||
offset?: string
|
||||
}
|
||||
|
||||
const records = data.records || []
|
||||
const documents: ExternalDocument[] = await Promise.all(
|
||||
records.map((record) =>
|
||||
recordToDocument(record, baseId, tableIdOrName, titleField, fieldNames)
|
||||
)
|
||||
)
|
||||
|
||||
const nextOffset = data.offset
|
||||
return {
|
||||
documents,
|
||||
nextCursor: nextOffset ? `offset:${nextOffset}` : undefined,
|
||||
hasMore: Boolean(nextOffset),
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
const baseId = sourceConfig.baseId as string
|
||||
const tableIdOrName = sourceConfig.tableIdOrName as string
|
||||
const titleField = sourceConfig.titleField as string | undefined
|
||||
|
||||
const fieldNames = await fetchFieldNames(accessToken, baseId, tableIdOrName)
|
||||
const encodedTable = encodeURIComponent(tableIdOrName)
|
||||
const url = `${AIRTABLE_API}/${baseId}/${encodedTable}/${externalId}`
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404 || response.status === 422) return null
|
||||
throw new Error(`Failed to get Airtable record: ${response.status}`)
|
||||
}
|
||||
|
||||
const record = (await response.json()) as AirtableRecord
|
||||
return recordToDocument(record, baseId, tableIdOrName, titleField, fieldNames)
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const baseId = sourceConfig.baseId as string
|
||||
const tableIdOrName = sourceConfig.tableIdOrName as string
|
||||
|
||||
if (!baseId || !tableIdOrName) {
|
||||
return { valid: false, error: 'Base ID and table name are required' }
|
||||
}
|
||||
|
||||
if (baseId && !baseId.startsWith('app')) {
|
||||
return { valid: false, error: 'Base ID should start with "app"' }
|
||||
}
|
||||
|
||||
const maxRecords = sourceConfig.maxRecords as string | undefined
|
||||
if (maxRecords && (Number.isNaN(Number(maxRecords)) || Number(maxRecords) <= 0)) {
|
||||
return { valid: false, error: 'Max records must be a positive number' }
|
||||
}
|
||||
|
||||
try {
|
||||
const encodedTable = encodeURIComponent(tableIdOrName)
|
||||
const url = `${AIRTABLE_API}/${baseId}/${encodedTable}?pageSize=1`
|
||||
const response = await fetchWithRetry(
|
||||
url,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
if (response.status === 404 || response.status === 422) {
|
||||
return { valid: false, error: `Table "${tableIdOrName}" not found in base "${baseId}"` }
|
||||
}
|
||||
if (response.status === 403) {
|
||||
return { valid: false, error: 'Access denied. Check your Airtable permissions.' }
|
||||
}
|
||||
return { valid: false, error: `Airtable API error: ${response.status} - ${errorText}` }
|
||||
}
|
||||
|
||||
const viewId = sourceConfig.viewId as string | undefined
|
||||
if (viewId) {
|
||||
const viewUrl = `${AIRTABLE_API}/${baseId}/${encodedTable}?pageSize=1&view=${encodeURIComponent(viewId)}`
|
||||
const viewResponse = await fetchWithRetry(
|
||||
viewUrl,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
if (!viewResponse.ok) {
|
||||
return { valid: false, error: `View "${viewId}" not found in table "${tableIdOrName}"` }
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
const createdTime = parseTagDate(metadata.createdTime)
|
||||
if (createdTime) result.createdTime = createdTime
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
|
||||
interface AirtableRecord {
|
||||
id: string
|
||||
fields: Record<string, unknown>
|
||||
createdTime: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts an Airtable record to an ExternalDocument.
|
||||
*/
|
||||
async function recordToDocument(
|
||||
record: AirtableRecord,
|
||||
baseId: string,
|
||||
tableIdOrName: string,
|
||||
titleField: string | undefined,
|
||||
fieldNames: Map<string, string>
|
||||
): Promise<ExternalDocument> {
|
||||
const plainText = recordToPlainText(record.fields, fieldNames)
|
||||
const contentHash = await computeContentHash(plainText)
|
||||
const title = extractTitle(record.fields, titleField)
|
||||
|
||||
const encodedTable = encodeURIComponent(tableIdOrName)
|
||||
const sourceUrl = `https://airtable.com/${baseId}/${encodedTable}/${record.id}`
|
||||
|
||||
return {
|
||||
externalId: record.id,
|
||||
title,
|
||||
content: plainText,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl,
|
||||
contentHash,
|
||||
metadata: {
|
||||
createdTime: record.createdTime,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the table schema to build a field ID → field name mapping.
|
||||
*/
|
||||
async function fetchFieldNames(
|
||||
accessToken: string,
|
||||
baseId: string,
|
||||
tableIdOrName: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<Map<string, string>> {
|
||||
const cacheKey = `fieldNames:${baseId}/${tableIdOrName}`
|
||||
if (syncContext?.[cacheKey]) return syncContext[cacheKey] as Map<string, string>
|
||||
|
||||
const fieldNames = new Map<string, string>()
|
||||
|
||||
try {
|
||||
const url = `${AIRTABLE_API}/meta/bases/${baseId}/tables`
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
logger.warn('Failed to fetch Airtable schema, using raw field keys', {
|
||||
status: response.status,
|
||||
})
|
||||
return fieldNames
|
||||
}
|
||||
|
||||
const data = (await response.json()) as {
|
||||
tables: { id: string; name: string; fields: { id: string; name: string; type: string }[] }[]
|
||||
}
|
||||
|
||||
const table = data.tables.find((t) => t.id === tableIdOrName || t.name === tableIdOrName)
|
||||
|
||||
if (table) {
|
||||
for (const field of table.fields) {
|
||||
fieldNames.set(field.id, field.name)
|
||||
fieldNames.set(field.name, field.name)
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Error fetching Airtable schema', {
|
||||
error: toError(error).message,
|
||||
})
|
||||
}
|
||||
|
||||
if (syncContext) syncContext[cacheKey] = fieldNames
|
||||
return fieldNames
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { airtableConnector } from '@/connectors/airtable/airtable'
|
||||
@@ -0,0 +1,81 @@
|
||||
import { AirtableIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const airtableConnectorMeta: ConnectorMeta = {
|
||||
id: 'airtable',
|
||||
name: 'Airtable',
|
||||
description: 'Sync records from an Airtable table',
|
||||
version: '1.0.0',
|
||||
icon: AirtableIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'oauth',
|
||||
provider: 'airtable',
|
||||
requiredScopes: ['data.records:read', 'schema.bases:read'],
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'baseSelector',
|
||||
title: 'Base',
|
||||
type: 'selector',
|
||||
selectorKey: 'airtable.bases',
|
||||
canonicalParamId: 'baseId',
|
||||
mode: 'basic',
|
||||
placeholder: 'Select a base',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'baseId',
|
||||
title: 'Base ID',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'baseId',
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. appXXXXXXXXXXXXXX',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'tableSelector',
|
||||
title: 'Table',
|
||||
type: 'selector',
|
||||
selectorKey: 'airtable.tables',
|
||||
canonicalParamId: 'tableIdOrName',
|
||||
mode: 'basic',
|
||||
dependsOn: ['baseSelector'],
|
||||
placeholder: 'Select a table',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'tableIdOrName',
|
||||
title: 'Table Name or ID',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'tableIdOrName',
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. Tasks or tblXXXXXXXXXXXXXX',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'viewId',
|
||||
title: 'View',
|
||||
type: 'short-input',
|
||||
placeholder: 'e.g. Grid view or viwXXXXXXXXXXXXXX',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'titleField',
|
||||
title: 'Title Field',
|
||||
type: 'short-input',
|
||||
placeholder: 'e.g. Name',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'maxRecords',
|
||||
title: 'Max Records',
|
||||
type: 'short-input',
|
||||
placeholder: 'e.g. 1000 (default: unlimited)',
|
||||
required: false,
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [{ id: 'createdTime', displayName: 'Created Time', fieldType: 'date' }],
|
||||
}
|
||||
@@ -0,0 +1,343 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { asanaConnectorMeta } from '@/connectors/asana/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { joinTagArray, parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('AsanaConnector')
|
||||
|
||||
const ASANA_API = 'https://app.asana.com/api/1.0'
|
||||
|
||||
const TASK_OPT_FIELDS =
|
||||
'name,notes,completed,completed_at,modified_at,assignee.name,tags.name,permalink_url'
|
||||
|
||||
/**
|
||||
* Asana API response shape for paginated endpoints.
|
||||
*/
|
||||
interface AsanaPageResponse {
|
||||
data: AsanaTask[]
|
||||
next_page: { offset: string; uri: string } | null
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal Asana task shape used by this connector.
|
||||
*/
|
||||
interface AsanaTask {
|
||||
gid: string
|
||||
name: string
|
||||
notes?: string
|
||||
completed: boolean
|
||||
completed_at?: string
|
||||
modified_at?: string
|
||||
assignee?: { name: string }
|
||||
tags?: { name: string }[]
|
||||
permalink_url?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Asana workspace shape.
|
||||
*/
|
||||
interface AsanaWorkspace {
|
||||
gid: string
|
||||
name: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Asana project shape.
|
||||
*/
|
||||
interface AsanaProject {
|
||||
gid: string
|
||||
name: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Makes a GET request to the Asana REST API.
|
||||
*/
|
||||
async function asanaGet<T>(
|
||||
accessToken: string,
|
||||
path: string,
|
||||
retryOptions?: Parameters<typeof fetchWithRetry>[2]
|
||||
): Promise<T> {
|
||||
const response = await fetchWithRetry(
|
||||
`${ASANA_API}${path}`,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
},
|
||||
retryOptions
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error('Asana API request failed', { status: response.status, path, error: errorText })
|
||||
throw new Error(`Asana API error: ${response.status}`)
|
||||
}
|
||||
|
||||
return (await response.json()) as T
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a formatted text document from an Asana task.
|
||||
*/
|
||||
function buildTaskContent(task: AsanaTask): string {
|
||||
const parts: string[] = []
|
||||
|
||||
parts.push(task.name || 'Untitled')
|
||||
|
||||
if (task.assignee?.name) parts.push(`Assignee: ${task.assignee.name}`)
|
||||
|
||||
parts.push(`Completed: ${task.completed ? 'Yes' : 'No'}`)
|
||||
|
||||
const tagNames = task.tags?.map((t) => t.name).filter(Boolean)
|
||||
if (tagNames && tagNames.length > 0) {
|
||||
parts.push(`Labels: ${tagNames.join(', ')}`)
|
||||
}
|
||||
|
||||
if (task.notes) {
|
||||
parts.push('')
|
||||
parts.push(task.notes)
|
||||
}
|
||||
|
||||
return parts.join('\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches all project GIDs in a workspace, used when no specific project is configured.
|
||||
*/
|
||||
async function listWorkspaceProjects(
|
||||
accessToken: string,
|
||||
workspaceGid: string
|
||||
): Promise<AsanaProject[]> {
|
||||
const projects: AsanaProject[] = []
|
||||
let offset: string | undefined
|
||||
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
const offsetParam = offset ? `&offset=${offset}` : ''
|
||||
const result = await asanaGet<{ data: AsanaProject[]; next_page: { offset: string } | null }>(
|
||||
accessToken,
|
||||
`/projects?workspace=${workspaceGid}&limit=100${offsetParam}`
|
||||
)
|
||||
projects.push(...result.data)
|
||||
if (!result.next_page) break
|
||||
offset = result.next_page.offset
|
||||
}
|
||||
|
||||
return projects
|
||||
}
|
||||
|
||||
export const asanaConnector: ConnectorConfig = {
|
||||
...asanaConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const workspaceGid = sourceConfig.workspace as string
|
||||
const projectGid = (sourceConfig.project as string) || ''
|
||||
const maxTasks = sourceConfig.maxTasks ? Number(sourceConfig.maxTasks) : 0
|
||||
const pageSize = maxTasks > 0 ? Math.min(maxTasks, 100) : 100
|
||||
|
||||
/**
|
||||
* Cursor format:
|
||||
* - For a single project: the offset string directly, or undefined
|
||||
* - For all projects: JSON-encoded { projectIndex, offset }
|
||||
*/
|
||||
let projectGids: string[]
|
||||
let projectIndex = 0
|
||||
let offset: string | undefined
|
||||
|
||||
if (projectGid) {
|
||||
projectGids = [projectGid]
|
||||
} else {
|
||||
if (!syncContext?.projectGids) {
|
||||
logger.info('Fetching all projects in workspace', { workspaceGid })
|
||||
const projects = await listWorkspaceProjects(accessToken, workspaceGid)
|
||||
if (syncContext) syncContext.projectGids = projects.map((p) => p.gid)
|
||||
projectGids = projects.map((p) => p.gid)
|
||||
} else {
|
||||
projectGids = syncContext.projectGids as string[]
|
||||
}
|
||||
}
|
||||
|
||||
if (cursor) {
|
||||
try {
|
||||
const parsed = JSON.parse(cursor) as { projectIndex: number; offset?: string }
|
||||
projectIndex = parsed.projectIndex
|
||||
offset = parsed.offset
|
||||
} catch {
|
||||
offset = cursor
|
||||
}
|
||||
}
|
||||
|
||||
logger.info('Listing Asana tasks', {
|
||||
workspaceGid,
|
||||
projectCount: projectGids.length,
|
||||
projectIndex,
|
||||
offset,
|
||||
pageSize,
|
||||
})
|
||||
|
||||
const documents: ExternalDocument[] = []
|
||||
let nextCursor: string | undefined
|
||||
let hasMore = false
|
||||
|
||||
while (projectIndex < projectGids.length) {
|
||||
const currentProjectGid = projectGids[projectIndex]
|
||||
const offsetParam = offset ? `&offset=${offset}` : ''
|
||||
|
||||
const result = await asanaGet<AsanaPageResponse>(
|
||||
accessToken,
|
||||
`/tasks?project=${currentProjectGid}&opt_fields=${TASK_OPT_FIELDS}&limit=${pageSize}${offsetParam}`
|
||||
)
|
||||
|
||||
for (const task of result.data) {
|
||||
const content = buildTaskContent(task)
|
||||
const tagNames = task.tags?.map((t) => t.name).filter(Boolean) || []
|
||||
|
||||
documents.push({
|
||||
externalId: task.gid,
|
||||
title: task.name || 'Untitled',
|
||||
content,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: task.permalink_url || undefined,
|
||||
contentHash: `asana:${task.gid}:${task.modified_at ?? ''}`,
|
||||
metadata: {
|
||||
project: currentProjectGid,
|
||||
assignee: task.assignee?.name,
|
||||
completed: task.completed,
|
||||
lastModified: task.modified_at,
|
||||
labels: tagNames,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
if (result.next_page) {
|
||||
nextCursor = JSON.stringify({ projectIndex, offset: result.next_page.offset })
|
||||
hasMore = true
|
||||
break
|
||||
}
|
||||
|
||||
projectIndex++
|
||||
offset = undefined
|
||||
|
||||
if (projectIndex < projectGids.length) {
|
||||
nextCursor = JSON.stringify({ projectIndex, offset: undefined })
|
||||
hasMore = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
|
||||
if (maxTasks > 0) {
|
||||
const remaining = maxTasks - previouslyFetched
|
||||
if (documents.length > remaining) {
|
||||
documents.splice(remaining)
|
||||
}
|
||||
}
|
||||
|
||||
const totalFetched = previouslyFetched + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
const hitLimit = maxTasks > 0 && totalFetched >= maxTasks
|
||||
|
||||
if (hitLimit) {
|
||||
hasMore = false
|
||||
nextCursor = undefined
|
||||
}
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore ? nextCursor : undefined,
|
||||
hasMore,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
_sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
try {
|
||||
const result = await asanaGet<{ data: AsanaTask }>(
|
||||
accessToken,
|
||||
`/tasks/${externalId}?opt_fields=${TASK_OPT_FIELDS}`
|
||||
)
|
||||
const task = result.data
|
||||
|
||||
if (!task) return null
|
||||
|
||||
const content = buildTaskContent(task)
|
||||
const tagNames = task.tags?.map((t) => t.name).filter(Boolean) || []
|
||||
|
||||
return {
|
||||
externalId: task.gid,
|
||||
title: task.name || 'Untitled',
|
||||
content,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: task.permalink_url || undefined,
|
||||
contentHash: `asana:${task.gid}:${task.modified_at ?? ''}`,
|
||||
metadata: {
|
||||
assignee: task.assignee?.name,
|
||||
completed: task.completed,
|
||||
lastModified: task.modified_at,
|
||||
labels: tagNames,
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to get Asana task', {
|
||||
externalId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const workspaceGid = sourceConfig.workspace as string | undefined
|
||||
if (!workspaceGid) {
|
||||
return { valid: false, error: 'Workspace GID is required' }
|
||||
}
|
||||
|
||||
const maxTasks = sourceConfig.maxTasks as string | undefined
|
||||
if (maxTasks && (Number.isNaN(Number(maxTasks)) || Number(maxTasks) <= 0)) {
|
||||
return { valid: false, error: 'Max tasks must be a positive number' }
|
||||
}
|
||||
|
||||
try {
|
||||
await asanaGet<{ data: AsanaWorkspace }>(
|
||||
accessToken,
|
||||
`/workspaces/${workspaceGid}`,
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.project === 'string') result.project = metadata.project
|
||||
if (typeof metadata.assignee === 'string') result.assignee = metadata.assignee
|
||||
if (typeof metadata.completed === 'boolean') result.completed = metadata.completed
|
||||
|
||||
const lastModified = parseTagDate(metadata.lastModified)
|
||||
if (lastModified) result.lastModified = lastModified
|
||||
|
||||
const labels = joinTagArray(metadata.labels)
|
||||
if (labels) result.labels = labels
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { asanaConnector } from '@/connectors/asana/asana'
|
||||
@@ -0,0 +1,56 @@
|
||||
import { AsanaIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const asanaConnectorMeta: ConnectorMeta = {
|
||||
id: 'asana',
|
||||
name: 'Asana',
|
||||
description: 'Sync tasks from Asana',
|
||||
version: '1.0.0',
|
||||
icon: AsanaIcon,
|
||||
|
||||
auth: { mode: 'oauth', provider: 'asana', requiredScopes: ['default'] },
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'workspaceSelector',
|
||||
title: 'Workspace',
|
||||
type: 'selector',
|
||||
selectorKey: 'asana.workspaces',
|
||||
canonicalParamId: 'workspace',
|
||||
mode: 'basic',
|
||||
placeholder: 'Select a workspace',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'workspace',
|
||||
title: 'Workspace GID',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'workspace',
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. 1234567890',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'project',
|
||||
title: 'Project GID',
|
||||
type: 'short-input',
|
||||
placeholder: 'e.g. 9876543210 (leave empty for all projects)',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'maxTasks',
|
||||
title: 'Max Tasks',
|
||||
type: 'short-input',
|
||||
placeholder: 'e.g. 500 (default: unlimited)',
|
||||
required: false,
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'project', displayName: 'Project', fieldType: 'text' },
|
||||
{ id: 'assignee', displayName: 'Assignee', fieldType: 'text' },
|
||||
{ id: 'completed', displayName: 'Completed', fieldType: 'boolean' },
|
||||
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
|
||||
{ id: 'labels', displayName: 'Labels', fieldType: 'text' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,601 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { ashbyConnectorMeta } from '@/connectors/ashby/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('AshbyConnector')
|
||||
|
||||
const ASHBY_API_BASE = 'https://api.ashbyhq.com'
|
||||
const CANDIDATES_PER_PAGE = 100
|
||||
const NOTES_PER_PAGE = 100
|
||||
const FEEDBACK_PER_PAGE = 100
|
||||
|
||||
/**
|
||||
* Hard cap on the number of applications whose interview feedback is fetched for a
|
||||
* single candidate document. Candidates with many applications are rare, but this
|
||||
* bounds the number of feedback API calls per `getDocument` invocation.
|
||||
*/
|
||||
const MAX_APPLICATIONS_FOR_FEEDBACK = 10
|
||||
|
||||
type UnknownRecord = Record<string, unknown>
|
||||
|
||||
/**
|
||||
* Builds the standard Ashby Authorization header. Ashby uses HTTP Basic auth with
|
||||
* the API key as the username and an empty password, i.e. `Basic base64(apiKey + ':')`.
|
||||
*/
|
||||
function ashbyHeaders(accessToken: string): Record<string, string> {
|
||||
return {
|
||||
'Content-Type': 'application/json',
|
||||
Accept: 'application/json; version=1',
|
||||
Authorization: `Basic ${Buffer.from(`${accessToken}:`).toString('base64')}`,
|
||||
}
|
||||
}
|
||||
|
||||
interface AshbyEnvelope {
|
||||
success: boolean
|
||||
results?: unknown
|
||||
moreDataAvailable?: boolean
|
||||
nextCursor?: string | null
|
||||
errors?: unknown
|
||||
errorInfo?: { message?: string }
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts a human-readable error message from an Ashby error envelope. Ashby returns
|
||||
* errors as either `errorInfo.message` or an `errors` string array.
|
||||
*/
|
||||
function ashbyErrorMessage(data: AshbyEnvelope, fallback: string): string {
|
||||
if (data.errorInfo?.message) return data.errorInfo.message
|
||||
if (Array.isArray(data.errors) && data.errors.length > 0) {
|
||||
return data.errors.map((e) => String(e)).join('; ')
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes an Ashby RPC-style POST request and returns the parsed envelope.
|
||||
* Ashby exposes a flat set of POST endpoints under `https://api.ashbyhq.com`.
|
||||
*/
|
||||
async function ashbyPost(
|
||||
accessToken: string,
|
||||
endpoint: string,
|
||||
body: UnknownRecord,
|
||||
retryOptions?: Parameters<typeof fetchWithRetry>[2]
|
||||
): Promise<AshbyEnvelope> {
|
||||
const response = await fetchWithRetry(
|
||||
`${ASHBY_API_BASE}/${endpoint}`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: ashbyHeaders(accessToken),
|
||||
body: JSON.stringify(body),
|
||||
},
|
||||
retryOptions
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
throw new Error(
|
||||
`Ashby ${endpoint} HTTP error: ${response.status}${errorText ? ` — ${errorText.slice(0, 300)}` : ''}`
|
||||
)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as AshbyEnvelope
|
||||
if (!data.success) {
|
||||
throw new Error(ashbyErrorMessage(data, `Ashby ${endpoint} request failed`))
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
interface AshbyCandidateSummary {
|
||||
id: string
|
||||
name: string
|
||||
position: string | null
|
||||
company: string | null
|
||||
school: string | null
|
||||
location: string | null
|
||||
source: string | null
|
||||
emailDomain: string | null
|
||||
profileUrl: string | null
|
||||
applicationIds: string[]
|
||||
createdAt: string | null
|
||||
updatedAt: string | null
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts a human-readable location string from an Ashby candidate's `location`
|
||||
* object. Prefers the API-provided `locationSummary`. Falls back to joining the
|
||||
* `name` values of the `locationComponents` array (each entry is `{ type, name }`
|
||||
* ordered city → region → country, per the candidate entity returned by
|
||||
* `candidate.list`/`candidate.info`). As a final fallback, supports the flat
|
||||
* `{ city, region, country }` shape used by candidate write inputs.
|
||||
*/
|
||||
function extractLocation(raw: UnknownRecord): string | null {
|
||||
const location = raw.location as UnknownRecord | undefined
|
||||
if (!location) return null
|
||||
|
||||
const summary = location.locationSummary as string | undefined
|
||||
if (summary?.trim()) return summary.trim()
|
||||
|
||||
if (Array.isArray(location.locationComponents)) {
|
||||
const parts = (location.locationComponents as UnknownRecord[])
|
||||
.map((c) => c?.name)
|
||||
.filter((n): n is string => typeof n === 'string' && n.trim().length > 0)
|
||||
.map((n) => n.trim())
|
||||
if (parts.length > 0) return parts.join(', ')
|
||||
}
|
||||
|
||||
const parts = [location.city, location.region, location.country]
|
||||
.filter((p): p is string => typeof p === 'string' && p.trim().length > 0)
|
||||
.map((p) => p.trim())
|
||||
return parts.length > 0 ? parts.join(', ') : null
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the source title from an Ashby candidate's `source` object, which
|
||||
* references the organization's sources list (e.g. "LinkedIn", "Referral").
|
||||
*/
|
||||
function extractSource(raw: UnknownRecord): string | null {
|
||||
const source = raw.source as UnknownRecord | undefined
|
||||
const title = source?.title as string | undefined
|
||||
return title?.trim() || null
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the lowercased domain from an Ashby candidate's primary email address
|
||||
* (`primaryEmailAddress.value`), enabling filtering candidates by email domain.
|
||||
*/
|
||||
function extractEmailDomain(raw: UnknownRecord): string | null {
|
||||
const email = raw.primaryEmailAddress as UnknownRecord | undefined
|
||||
const value = email?.value as string | undefined
|
||||
const at = value?.lastIndexOf('@') ?? -1
|
||||
if (!value || at < 0 || at === value.length - 1) return null
|
||||
return (
|
||||
value
|
||||
.slice(at + 1)
|
||||
.trim()
|
||||
.toLowerCase() || null
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes a raw Ashby candidate record into the fields this connector cares about.
|
||||
* Field names mirror the Ashby candidate object returned by `candidate.list` and
|
||||
* `candidate.info` (`position`, `company`, `school`, `location`, `source`,
|
||||
* `primaryEmailAddress`, `profileUrl`, `applicationIds`, `createdAt`, `updatedAt`).
|
||||
* Stage and status live on applications rather than candidates, so they are
|
||||
* intentionally not surfaced here.
|
||||
*/
|
||||
function mapCandidate(raw: unknown): AshbyCandidateSummary {
|
||||
const c = (raw ?? {}) as UnknownRecord
|
||||
return {
|
||||
id: (c.id as string) ?? '',
|
||||
name: (c.name as string) ?? '',
|
||||
position: (c.position as string) ?? null,
|
||||
company: (c.company as string) ?? null,
|
||||
school: (c.school as string) ?? null,
|
||||
location: extractLocation(c),
|
||||
source: extractSource(c),
|
||||
emailDomain: extractEmailDomain(c),
|
||||
profileUrl: (c.profileUrl as string) ?? null,
|
||||
applicationIds: Array.isArray(c.applicationIds) ? (c.applicationIds as string[]) : [],
|
||||
createdAt: (c.createdAt as string) ?? null,
|
||||
updatedAt: (c.updatedAt as string) ?? null,
|
||||
}
|
||||
}
|
||||
|
||||
interface AshbyNote {
|
||||
content: string | null
|
||||
authorName: string | null
|
||||
createdAt: string | null
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps a raw Ashby candidate note into a plain-text-friendly shape, combining the
|
||||
* author's first and last name into a single display name.
|
||||
*/
|
||||
function mapNote(raw: unknown): AshbyNote {
|
||||
const n = (raw ?? {}) as UnknownRecord
|
||||
const author = n.author as UnknownRecord | undefined
|
||||
const first = (author?.firstName as string) ?? ''
|
||||
const last = (author?.lastName as string) ?? ''
|
||||
const authorName = `${first} ${last}`.trim() || (author?.email as string) || null
|
||||
return {
|
||||
content: (n.content as string) ?? null,
|
||||
authorName,
|
||||
createdAt: (n.createdAt as string) ?? null,
|
||||
}
|
||||
}
|
||||
|
||||
interface AshbyFeedbackSummary {
|
||||
submittedByName: string | null
|
||||
submittedAt: string | null
|
||||
lines: string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Collects `{ field.path -> field.title }` entries from a feedback form definition.
|
||||
* Ashby's `formDefinition` exposes fields either flat under `fields[]` or grouped
|
||||
* under `sections[].fields[]`, and individual entries are sometimes wrapped in a
|
||||
* `{ field }` envelope — all variants are handled.
|
||||
*/
|
||||
function collectFieldTitles(formDefinition: UnknownRecord | undefined): Map<string, string> {
|
||||
const titleByPath = new Map<string, string>()
|
||||
if (!formDefinition) return titleByPath
|
||||
|
||||
const addField = (entry: UnknownRecord): void => {
|
||||
const field = (entry?.field ?? entry) as UnknownRecord
|
||||
const path = field?.path as string | undefined
|
||||
const title = (field?.title as string) || (field?.humanReadablePath as string)
|
||||
if (path && title) titleByPath.set(path, title)
|
||||
}
|
||||
|
||||
if (Array.isArray(formDefinition.fields)) {
|
||||
for (const entry of formDefinition.fields as UnknownRecord[]) addField(entry)
|
||||
}
|
||||
|
||||
if (Array.isArray(formDefinition.sections)) {
|
||||
for (const section of formDefinition.sections as UnknownRecord[]) {
|
||||
const fields = Array.isArray(section?.fields) ? (section.fields as UnknownRecord[]) : []
|
||||
for (const entry of fields) addField(entry)
|
||||
}
|
||||
}
|
||||
|
||||
return titleByPath
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps a raw Ashby application feedback submission into a flat list of
|
||||
* `Title: value` lines, resolving each `submittedValues` key (the field's `path`)
|
||||
* to its human-readable title via the form definition. Falls back to the raw path
|
||||
* when no title is found.
|
||||
*/
|
||||
function mapFeedback(raw: unknown): AshbyFeedbackSummary {
|
||||
const f = (raw ?? {}) as UnknownRecord
|
||||
const submittedBy = f.submittedByUser as UnknownRecord | undefined
|
||||
const first = (submittedBy?.firstName as string) ?? ''
|
||||
const last = (submittedBy?.lastName as string) ?? ''
|
||||
const submittedByName = `${first} ${last}`.trim() || (submittedBy?.email as string) || null
|
||||
|
||||
const titleByPath = collectFieldTitles(f.formDefinition as UnknownRecord | undefined)
|
||||
|
||||
const submittedValues = (f.submittedValues as UnknownRecord | undefined) ?? {}
|
||||
const lines: string[] = []
|
||||
for (const [path, value] of Object.entries(submittedValues)) {
|
||||
if (value == null) continue
|
||||
const label = titleByPath.get(path) ?? path
|
||||
const rendered = renderFeedbackValue(value)
|
||||
if (rendered) lines.push(`${label}: ${rendered}`)
|
||||
}
|
||||
|
||||
const submittedAt =
|
||||
(f.submittedAt as string) ?? (f.completedAt as string) ?? (f.createdAt as string) ?? null
|
||||
|
||||
return { submittedByName, submittedAt, lines }
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders an arbitrary submitted feedback value (string, number, boolean, or a
|
||||
* rich-text / structured object) into a single-line plain-text string.
|
||||
*/
|
||||
function renderFeedbackValue(value: unknown): string {
|
||||
if (typeof value === 'string') return value.trim()
|
||||
if (typeof value === 'number' || typeof value === 'boolean') return String(value)
|
||||
if (Array.isArray(value)) {
|
||||
return value
|
||||
.map((v) => renderFeedbackValue(v))
|
||||
.filter(Boolean)
|
||||
.join(', ')
|
||||
}
|
||||
if (value && typeof value === 'object') {
|
||||
const obj = value as UnknownRecord
|
||||
const label = obj.label ?? obj.value ?? obj.text ?? obj.content
|
||||
if (typeof label === 'string') return label.trim()
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
/**
|
||||
* Stable, metadata-based content hash for a candidate document. Identical between the
|
||||
* listing stub and the fully-fetched document so unchanged candidates are skipped,
|
||||
* which keeps the `getDocument` re-hydration (notes + feedback fetches) cheap: the
|
||||
* sync engine only re-hydrates a deferred stub when this hash differs from the stored
|
||||
* document's hash (see `lib/knowledge/connectors/sync-engine.ts`).
|
||||
*
|
||||
* Known limitation — notes/feedback freshness depends on `candidate.updatedAt`.
|
||||
* Candidate notes (`candidate.listNotes`) and interview feedback
|
||||
* (`applicationFeedback.list`) are separate Ashby objects, not candidate fields. This
|
||||
* hash is derived solely from the candidate's own `updatedAt`, so a new note or newly
|
||||
* submitted feedback is only re-synced if Ashby advances `candidate.updatedAt` as a
|
||||
* side effect of that write.
|
||||
*
|
||||
* As of this writing Ashby's public API docs do not specify what counts as a
|
||||
* "modification" for `candidate.updatedAt` or for `candidate.list` syncToken
|
||||
* incremental sync, and no third-party ATS-integration vendor (Merge, Nango, Knit)
|
||||
* documents it either — so this behavior is unverified. If Ashby does NOT touch
|
||||
* `candidate.updatedAt` on note/feedback writes, those additions will not be picked up
|
||||
* until some other candidate field changes; a forced full sync re-hydrates everything
|
||||
* regardless. No cheaper listing-time signal exists to fold into this hash: the
|
||||
* `candidate.list` object exposes no note/feedback count, and syncToken carries the
|
||||
* same unspecified change semantics as `updatedAt`.
|
||||
*
|
||||
* Refs:
|
||||
* - https://developers.ashbyhq.com/reference/candidatelist
|
||||
* - https://developers.ashbyhq.com/reference/candidatecreatenote
|
||||
* - https://developers.ashbyhq.com/docs/pagination-and-incremental-sync
|
||||
*/
|
||||
function buildContentHash(id: string, updatedAt: string | null): string {
|
||||
return `ashby:${id}:${updatedAt ?? ''}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a lightweight document stub from a candidate listing entry. Content is
|
||||
* deferred and only fetched (via `getDocument`) for new or changed candidates.
|
||||
*/
|
||||
function candidateToStub(candidate: AshbyCandidateSummary): ExternalDocument {
|
||||
return {
|
||||
externalId: candidate.id,
|
||||
title: candidate.name || 'Unnamed Candidate',
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: candidate.profileUrl ?? undefined,
|
||||
contentHash: buildContentHash(candidate.id, candidate.updatedAt),
|
||||
metadata: candidateMetadata(candidate),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the tag-carrying metadata block shared by the listing stub and the
|
||||
* fully-fetched document, keeping the keys aligned with `mapTags`/`tagDefinitions`.
|
||||
*/
|
||||
function candidateMetadata(candidate: AshbyCandidateSummary): Record<string, unknown> {
|
||||
return {
|
||||
candidateName: candidate.name,
|
||||
company: candidate.company,
|
||||
school: candidate.school,
|
||||
location: candidate.location,
|
||||
source: candidate.source,
|
||||
emailDomain: candidate.emailDomain,
|
||||
createdAt: candidate.createdAt,
|
||||
updatedAt: candidate.updatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches all notes for a candidate, following cursor pagination.
|
||||
*/
|
||||
async function fetchAllNotes(accessToken: string, candidateId: string): Promise<AshbyNote[]> {
|
||||
const notes: AshbyNote[] = []
|
||||
let cursor: string | undefined
|
||||
let hasMore = true
|
||||
|
||||
while (hasMore) {
|
||||
const body: UnknownRecord = { candidateId, limit: NOTES_PER_PAGE }
|
||||
if (cursor) body.cursor = cursor
|
||||
const data = await ashbyPost(accessToken, 'candidate.listNotes', body)
|
||||
const results = Array.isArray(data.results) ? data.results : []
|
||||
for (const raw of results) notes.push(mapNote(raw))
|
||||
cursor = data.nextCursor ?? undefined
|
||||
hasMore = Boolean(data.moreDataAvailable) && Boolean(cursor)
|
||||
}
|
||||
|
||||
return notes
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches all interview feedback submissions for a single application, following
|
||||
* cursor pagination.
|
||||
*/
|
||||
async function fetchFeedbackForApplication(
|
||||
accessToken: string,
|
||||
applicationId: string
|
||||
): Promise<AshbyFeedbackSummary[]> {
|
||||
const feedback: AshbyFeedbackSummary[] = []
|
||||
let cursor: string | undefined
|
||||
let hasMore = true
|
||||
|
||||
while (hasMore) {
|
||||
const body: UnknownRecord = { applicationId, limit: FEEDBACK_PER_PAGE }
|
||||
if (cursor) body.cursor = cursor
|
||||
const data = await ashbyPost(accessToken, 'applicationFeedback.list', body)
|
||||
const results = Array.isArray(data.results) ? data.results : []
|
||||
for (const raw of results) feedback.push(mapFeedback(raw))
|
||||
cursor = data.nextCursor ?? undefined
|
||||
hasMore = Boolean(data.moreDataAvailable) && Boolean(cursor)
|
||||
}
|
||||
|
||||
return feedback
|
||||
}
|
||||
|
||||
/**
|
||||
* Assembles a candidate's profile, notes, and interview feedback into a single
|
||||
* plain-text document body for indexing.
|
||||
*/
|
||||
function formatCandidateContent(
|
||||
candidate: AshbyCandidateSummary,
|
||||
notes: AshbyNote[],
|
||||
feedback: AshbyFeedbackSummary[]
|
||||
): string {
|
||||
const parts: string[] = []
|
||||
|
||||
parts.push(`Candidate: ${candidate.name || 'Unnamed Candidate'}`)
|
||||
if (candidate.position) parts.push(`Current Role: ${candidate.position}`)
|
||||
if (candidate.company) parts.push(`Current Company: ${candidate.company}`)
|
||||
if (candidate.school) parts.push(`School: ${candidate.school}`)
|
||||
if (candidate.location) parts.push(`Location: ${candidate.location}`)
|
||||
if (candidate.source) parts.push(`Source: ${candidate.source}`)
|
||||
if (candidate.createdAt) parts.push(`Created: ${candidate.createdAt}`)
|
||||
if (candidate.updatedAt) parts.push(`Last Updated: ${candidate.updatedAt}`)
|
||||
|
||||
const nonEmptyNotes = notes.filter((n) => n.content?.trim())
|
||||
if (nonEmptyNotes.length > 0) {
|
||||
parts.push('')
|
||||
parts.push('--- Notes ---')
|
||||
for (const note of nonEmptyNotes) {
|
||||
const header = [note.authorName, note.createdAt].filter(Boolean).join(' — ')
|
||||
if (header) parts.push(`[${header}]`)
|
||||
parts.push((note.content ?? '').trim())
|
||||
parts.push('')
|
||||
}
|
||||
}
|
||||
|
||||
const nonEmptyFeedback = feedback.filter((f) => f.lines.length > 0)
|
||||
if (nonEmptyFeedback.length > 0) {
|
||||
parts.push('--- Interview Feedback ---')
|
||||
for (const f of nonEmptyFeedback) {
|
||||
const header = [f.submittedByName, f.submittedAt].filter(Boolean).join(' — ')
|
||||
if (header) parts.push(`[${header}]`)
|
||||
for (const line of f.lines) parts.push(line)
|
||||
parts.push('')
|
||||
}
|
||||
}
|
||||
|
||||
return parts.join('\n').trim()
|
||||
}
|
||||
|
||||
export const ashbyConnector: ConnectorConfig = {
|
||||
...ashbyConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const maxCandidates = sourceConfig.maxCandidates ? Number(sourceConfig.maxCandidates) : 0
|
||||
const createdAfterMs = (() => {
|
||||
const raw = sourceConfig.createdAfter
|
||||
if (typeof raw !== 'string' || !raw.trim()) return undefined
|
||||
const ms = new Date(raw.trim()).getTime()
|
||||
return Number.isNaN(ms) ? undefined : ms
|
||||
})()
|
||||
|
||||
const prevFetched = (syncContext?.totalCandidatesFetched as number) ?? 0
|
||||
if (maxCandidates > 0 && prevFetched >= maxCandidates) {
|
||||
if (syncContext) syncContext.listingCapped = true
|
||||
return { documents: [], hasMore: false }
|
||||
}
|
||||
|
||||
const body: UnknownRecord = { limit: CANDIDATES_PER_PAGE }
|
||||
if (cursor) body.cursor = cursor
|
||||
if (createdAfterMs !== undefined) body.createdAfter = createdAfterMs
|
||||
|
||||
logger.info('Listing Ashby candidates', {
|
||||
cursor: cursor ?? 'initial',
|
||||
maxCandidates: maxCandidates || 'unlimited',
|
||||
})
|
||||
|
||||
const data = await ashbyPost(accessToken, 'candidate.list', body)
|
||||
const results = Array.isArray(data.results) ? data.results : []
|
||||
const candidates = results.map(mapCandidate).filter((c) => c.id)
|
||||
|
||||
let documents = candidates.map(candidateToStub)
|
||||
if (maxCandidates > 0) {
|
||||
const remaining = Math.max(0, maxCandidates - prevFetched)
|
||||
if (documents.length > remaining) documents = documents.slice(0, remaining)
|
||||
}
|
||||
|
||||
const totalFetched = prevFetched + documents.length
|
||||
if (syncContext) syncContext.totalCandidatesFetched = totalFetched
|
||||
const hitLimit = maxCandidates > 0 && totalFetched >= maxCandidates
|
||||
if (hitLimit && syncContext) syncContext.listingCapped = true
|
||||
|
||||
const nextCursor = data.nextCursor ?? undefined
|
||||
const hasMore = !hitLimit && Boolean(data.moreDataAvailable) && Boolean(nextCursor)
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore ? nextCursor : undefined,
|
||||
hasMore,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
_sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
try {
|
||||
if (!externalId) return null
|
||||
|
||||
const infoData = await ashbyPost(accessToken, 'candidate.info', { id: externalId })
|
||||
if (!infoData.results) return null
|
||||
const candidate = mapCandidate(infoData.results)
|
||||
if (!candidate.id) return null
|
||||
|
||||
const notes = await fetchAllNotes(accessToken, candidate.id)
|
||||
|
||||
const feedback: AshbyFeedbackSummary[] = []
|
||||
const applicationIds = candidate.applicationIds.slice(0, MAX_APPLICATIONS_FOR_FEEDBACK)
|
||||
for (const applicationId of applicationIds) {
|
||||
try {
|
||||
const applicationFeedback = await fetchFeedbackForApplication(accessToken, applicationId)
|
||||
feedback.push(...applicationFeedback)
|
||||
} catch (error) {
|
||||
logger.warn('Failed to fetch Ashby feedback for application', {
|
||||
applicationId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const content = formatCandidateContent(candidate, notes, feedback)
|
||||
if (!content.trim()) return null
|
||||
|
||||
return {
|
||||
externalId: candidate.id,
|
||||
title: candidate.name || 'Unnamed Candidate',
|
||||
content,
|
||||
contentDeferred: false,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: candidate.profileUrl ?? undefined,
|
||||
contentHash: buildContentHash(candidate.id, candidate.updatedAt),
|
||||
metadata: candidateMetadata(candidate),
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Failed to get Ashby candidate', {
|
||||
externalId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const maxCandidates = sourceConfig.maxCandidates as string | undefined
|
||||
if (maxCandidates && (Number.isNaN(Number(maxCandidates)) || Number(maxCandidates) < 0)) {
|
||||
return { valid: false, error: 'Max candidates must be a non-negative number' }
|
||||
}
|
||||
|
||||
try {
|
||||
await ashbyPost(accessToken, 'candidate.list', { limit: 1 }, VALIDATE_RETRY_OPTIONS)
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
const textTags = ['candidateName', 'company', 'school', 'location', 'source', 'emailDomain']
|
||||
for (const key of textTags) {
|
||||
const value = metadata[key]
|
||||
if (typeof value === 'string' && value.trim()) result[key] = value.trim()
|
||||
}
|
||||
|
||||
const createdAt = parseTagDate(metadata.createdAt)
|
||||
if (createdAt) result.createdAt = createdAt
|
||||
|
||||
const updatedAt = parseTagDate(metadata.updatedAt)
|
||||
if (updatedAt) result.updatedAt = updatedAt
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { ashbyConnector } from '@/connectors/ashby/ashby'
|
||||
@@ -0,0 +1,49 @@
|
||||
import { AshbyIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const ashbyConnectorMeta: ConnectorMeta = {
|
||||
id: 'ashby',
|
||||
name: 'Ashby',
|
||||
description: 'Sync candidate notes and interview feedback from Ashby',
|
||||
version: '1.0.0',
|
||||
icon: AshbyIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'apiKey',
|
||||
label: 'API Key',
|
||||
placeholder: 'Enter your Ashby API key',
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'maxCandidates',
|
||||
title: 'Max Candidates',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 500 (default: unlimited)',
|
||||
description:
|
||||
'Cap the number of candidates synced. Leave empty to sync ALL candidates in the organization.',
|
||||
},
|
||||
{
|
||||
id: 'createdAfter',
|
||||
title: 'Created After',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. 2025-01-01 or 2025-01-01T00:00:00Z',
|
||||
description:
|
||||
'Only sync candidates created on or after this date (ISO 8601). Leave blank to sync candidates regardless of creation date.',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'candidateName', displayName: 'Candidate Name', fieldType: 'text' },
|
||||
{ id: 'company', displayName: 'Current Company', fieldType: 'text' },
|
||||
{ id: 'school', displayName: 'School', fieldType: 'text' },
|
||||
{ id: 'location', displayName: 'Location', fieldType: 'text' },
|
||||
{ id: 'source', displayName: 'Source', fieldType: 'text' },
|
||||
{ id: 'emailDomain', displayName: 'Email Domain', fieldType: 'text' },
|
||||
{ id: 'createdAt', displayName: 'Created', fieldType: 'date' },
|
||||
{ id: 'updatedAt', displayName: 'Last Updated', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
||||
export { azureDevopsConnector } from '@/connectors/azure-devops/azure-devops'
|
||||
@@ -0,0 +1,177 @@
|
||||
import { AzureIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const azureDevopsConnectorMeta: ConnectorMeta = {
|
||||
id: 'azure_devops',
|
||||
name: 'Azure DevOps',
|
||||
description:
|
||||
'Sync wiki pages, work items, and repository files from an Azure DevOps project into your knowledge base',
|
||||
version: '1.1.0',
|
||||
icon: AzureIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'apiKey',
|
||||
label: 'Personal Access Token',
|
||||
placeholder: 'Enter your Azure DevOps PAT (scopes: Wiki Read, Work Items Read, Code Read)',
|
||||
},
|
||||
|
||||
/**
|
||||
* Incremental sync applies to work items only, via a `System.ChangedDate`
|
||||
* WIQL filter derived from lastSyncAt. Wiki pages have no change timestamp on
|
||||
* listing, so they are always re-listed and reconciled by ETag content hash.
|
||||
* Repository files are likewise always re-listed in full and reconciled by the
|
||||
* git blob objectId hash — a commit-diff incremental path is intentionally
|
||||
* avoided to match the github/gitlab full-listing approach, keeping change
|
||||
* detection correct without tracking per-branch commit state. Unchanged
|
||||
* documents are skipped without a content fetch in every case.
|
||||
*/
|
||||
supportsIncrementalSync: true,
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'organization',
|
||||
title: 'Organization',
|
||||
type: 'short-input',
|
||||
placeholder: 'e.g. my-org',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'project',
|
||||
title: 'Project',
|
||||
type: 'short-input',
|
||||
placeholder: 'e.g. my-project',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'contentType',
|
||||
title: 'Content',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
options: [
|
||||
{ label: 'Wiki pages only', id: 'wiki' },
|
||||
{ label: 'Work items only', id: 'workitems' },
|
||||
{ label: 'Repository files only', id: 'files' },
|
||||
{ label: 'Wiki pages and work items', id: 'both' },
|
||||
{ label: 'Wiki, work items, and files', id: 'all' },
|
||||
],
|
||||
description: 'Which content to index from the project.',
|
||||
},
|
||||
{
|
||||
id: 'wikiName',
|
||||
title: 'Wiki',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'Wiki name or ID (all wikis if blank)',
|
||||
description:
|
||||
'Restrict syncing to a single wiki by name or ID. Applies only when syncing wiki pages.',
|
||||
},
|
||||
{
|
||||
id: 'workItemType',
|
||||
title: 'Work Item Type',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. Bug, Task, User Story',
|
||||
description: 'Only sync work items of this type. Applies only when syncing work items.',
|
||||
},
|
||||
{
|
||||
id: 'state',
|
||||
title: 'State',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. Active, Closed',
|
||||
description: 'Only sync work items in this state. Applies only when syncing work items.',
|
||||
},
|
||||
{
|
||||
id: 'areaPath',
|
||||
title: 'Area Path',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. MyProject\\Team A',
|
||||
description:
|
||||
'Only sync work items under this area path (and its children). Applies only when syncing work items.',
|
||||
},
|
||||
{
|
||||
id: 'workItemTags',
|
||||
title: 'Tags',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. customer, urgent (comma-separated)',
|
||||
description:
|
||||
'Only sync work items containing all of these tags (comma-separated). Applies only when syncing work items.',
|
||||
},
|
||||
{
|
||||
id: 'customWiql',
|
||||
title: 'Custom WIQL Query',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'SELECT [System.Id] FROM workitems WHERE ...',
|
||||
description:
|
||||
'Advanced: a full WIQL query selecting [System.Id]. Overrides the type, state, area path, and tag filters when set. Custom queries always run as full listings on every sync (the incremental changed-date filter is not applied).',
|
||||
},
|
||||
{
|
||||
id: 'repositoryName',
|
||||
title: 'Repository',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'Repository name or ID (all repos if blank)',
|
||||
description:
|
||||
'Restrict syncing to a single repository by name or ID. Applies only when syncing repository files.',
|
||||
},
|
||||
{
|
||||
id: 'branch',
|
||||
title: 'Branch',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: "Each repo's default branch",
|
||||
description:
|
||||
'Branch to sync repository files from. Defaults to each repository’s default branch. Applies only when syncing repository files.',
|
||||
},
|
||||
{
|
||||
id: 'pathPrefix',
|
||||
title: 'Path Filter',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. docs/, src/',
|
||||
description:
|
||||
'Only sync repository files under this path prefix. Applies only when syncing repository files.',
|
||||
},
|
||||
{
|
||||
id: 'fileExtensions',
|
||||
title: 'File Extensions',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. .md, .txt, .ts',
|
||||
description:
|
||||
'Only sync repository files with these extensions (comma-separated). Leave blank for all text files. Applies only when syncing repository files.',
|
||||
},
|
||||
{
|
||||
id: 'maxItems',
|
||||
title: 'Max Items',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 500 (default: unlimited)',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'kind', displayName: 'Type', fieldType: 'text' },
|
||||
{ id: 'wikiName', displayName: 'Wiki', fieldType: 'text' },
|
||||
{ id: 'workItemType', displayName: 'Work Item Type', fieldType: 'text' },
|
||||
{ id: 'state', displayName: 'State', fieldType: 'text' },
|
||||
{ id: 'areaPath', displayName: 'Area Path', fieldType: 'text' },
|
||||
{ id: 'tags', displayName: 'Tags', fieldType: 'text' },
|
||||
{ id: 'repository', displayName: 'Repository', fieldType: 'text' },
|
||||
{ id: 'path', displayName: 'File Path', fieldType: 'text' },
|
||||
{ id: 'changedDate', displayName: 'Changed Date', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { escapeCql } from '@/connectors/confluence/confluence'
|
||||
|
||||
describe('escapeCql', () => {
|
||||
it.concurrent('returns plain strings unchanged', () => {
|
||||
expect(escapeCql('Engineering')).toBe('Engineering')
|
||||
})
|
||||
|
||||
it.concurrent('escapes double quotes', () => {
|
||||
expect(escapeCql('say "hello"')).toBe('say \\"hello\\"')
|
||||
})
|
||||
|
||||
it.concurrent('escapes backslashes', () => {
|
||||
expect(escapeCql('path\\to\\file')).toBe('path\\\\to\\\\file')
|
||||
})
|
||||
|
||||
it.concurrent('escapes backslashes before quotes', () => {
|
||||
expect(escapeCql('a\\"b')).toBe('a\\\\\\"b')
|
||||
})
|
||||
|
||||
it.concurrent('handles empty string', () => {
|
||||
expect(escapeCql('')).toBe('')
|
||||
})
|
||||
|
||||
it.concurrent('leaves other special chars unchanged', () => {
|
||||
expect(escapeCql("it's a test & <tag>")).toBe("it's a test & <tag>")
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,629 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { confluenceConnectorMeta } from '@/connectors/confluence/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { htmlToPlainText, joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils'
|
||||
import { getConfluenceCloudId, normalizeConfluenceDomainHost } from '@/tools/confluence/utils'
|
||||
|
||||
const logger = createLogger('ConfluenceConnector')
|
||||
|
||||
/**
|
||||
* Escapes a value for use inside CQL double-quoted strings.
|
||||
*/
|
||||
export function escapeCql(value: string): string {
|
||||
return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"')
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a CQL clause restricting content to the given space keys.
|
||||
* Single key uses `space = "X"`; multiple keys use `space in ("X","Y")`.
|
||||
*/
|
||||
function buildSpaceClause(spaceKeys: string[]): string {
|
||||
if (spaceKeys.length === 1) {
|
||||
return `space="${escapeCql(spaceKeys[0])}"`
|
||||
}
|
||||
const list = spaceKeys.map((k) => `"${escapeCql(k)}"`).join(',')
|
||||
return `space in (${list})`
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches labels for a batch of page IDs using the v2 labels endpoint.
|
||||
*/
|
||||
const LABEL_FETCH_CONCURRENCY = 5
|
||||
|
||||
async function fetchLabelsForPages(
|
||||
cloudId: string,
|
||||
accessToken: string,
|
||||
pageIds: string[]
|
||||
): Promise<Map<string, string[]>> {
|
||||
const labelsByPageId = new Map<string, string[]>()
|
||||
|
||||
for (let i = 0; i < pageIds.length; i += LABEL_FETCH_CONCURRENCY) {
|
||||
const batch = pageIds.slice(i, i + LABEL_FETCH_CONCURRENCY)
|
||||
const results = await Promise.all(
|
||||
batch.map(async (pageId) => {
|
||||
try {
|
||||
let data: Record<string, unknown> | null = null
|
||||
for (const contentType of ['pages', 'blogposts']) {
|
||||
const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/${contentType}/${pageId}/labels`
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
|
||||
if (response.ok) {
|
||||
data = await response.json()
|
||||
break
|
||||
}
|
||||
if (response.status !== 404) {
|
||||
logger.warn(`Failed to fetch labels for ${contentType} ${pageId}`, {
|
||||
status: response.status,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if (!data) {
|
||||
return { pageId, labels: [] as string[] }
|
||||
}
|
||||
|
||||
const labels = ((data.results as Record<string, unknown>[]) || []).map(
|
||||
(label) => label.name as string
|
||||
)
|
||||
return { pageId, labels }
|
||||
} catch (error) {
|
||||
logger.warn(`Error fetching labels for page ${pageId}`, {
|
||||
error: toError(error).message,
|
||||
})
|
||||
return { pageId, labels: [] as string[] }
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
for (const { pageId, labels } of results) {
|
||||
labelsByPageId.set(pageId, labels)
|
||||
}
|
||||
}
|
||||
|
||||
return labelsByPageId
|
||||
}
|
||||
|
||||
/**
|
||||
* Produces a canonical metadata stub with a deterministic contentHash that
|
||||
* does not depend on which API surface (v1 CQL or v2) returned the page.
|
||||
*/
|
||||
function pageToStub(
|
||||
page: Record<string, unknown>,
|
||||
options: {
|
||||
spaceId?: unknown
|
||||
labels?: string[]
|
||||
sourceUrl?: string
|
||||
} = {}
|
||||
): ExternalDocument {
|
||||
const version = page.version as Record<string, unknown> | undefined
|
||||
const versionNumber = version?.number as number | undefined
|
||||
const lastModified = (version?.createdAt ?? version?.when ?? '') as string
|
||||
const versionKey = versionNumber ?? lastModified
|
||||
|
||||
return {
|
||||
externalId: String(page.id),
|
||||
title: (page.title as string) || 'Untitled',
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: options.sourceUrl,
|
||||
contentHash: `confluence:${page.id}:${versionKey}`,
|
||||
metadata: {
|
||||
spaceId: options.spaceId,
|
||||
status: page.status,
|
||||
version: versionNumber,
|
||||
labels: options.labels ?? [],
|
||||
lastModified,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a v1 CQL search result item to a lightweight metadata stub.
|
||||
*/
|
||||
function cqlResultToStub(item: Record<string, unknown>, domain: string): ExternalDocument {
|
||||
const links = item._links as Record<string, string> | undefined
|
||||
const metadata = item.metadata as Record<string, unknown> | undefined
|
||||
const labelsWrapper = metadata?.labels as Record<string, unknown> | undefined
|
||||
const labelResults = (labelsWrapper?.results || []) as Record<string, unknown>[]
|
||||
const labels = labelResults.map((l) => l.name as string)
|
||||
|
||||
return pageToStub(item, {
|
||||
spaceId: (item.space as Record<string, unknown>)?.key,
|
||||
labels,
|
||||
sourceUrl: links?.webui ? `https://${domain}/wiki${links.webui}` : undefined,
|
||||
})
|
||||
}
|
||||
|
||||
export const confluenceConnector: ConnectorConfig = {
|
||||
...confluenceConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const domain = normalizeConfluenceDomainHost(sourceConfig.domain as string)
|
||||
const spaceKeys = parseMultiValue(sourceConfig.spaceKey)
|
||||
const contentType = (sourceConfig.contentType as string) || 'page'
|
||||
const labelFilter = (sourceConfig.labelFilter as string) || ''
|
||||
const maxPages = sourceConfig.maxPages ? Number(sourceConfig.maxPages) : 0
|
||||
|
||||
if (spaceKeys.length === 0) {
|
||||
throw new Error('At least one space key is required')
|
||||
}
|
||||
|
||||
let cloudId = syncContext?.cloudId as string | undefined
|
||||
if (!cloudId) {
|
||||
cloudId = await getConfluenceCloudId(domain, accessToken)
|
||||
if (syncContext) syncContext.cloudId = cloudId
|
||||
}
|
||||
|
||||
/**
|
||||
* Route through CQL when a label filter is set or when multiple spaces are
|
||||
* selected — the v2 `/spaces/{spaceId}/pages` endpoint is single-space only,
|
||||
* but CQL natively supports `space in (...)`.
|
||||
*/
|
||||
if (labelFilter.trim() || spaceKeys.length > 1) {
|
||||
return listDocumentsViaCql(
|
||||
cloudId,
|
||||
accessToken,
|
||||
domain,
|
||||
spaceKeys,
|
||||
contentType,
|
||||
labelFilter,
|
||||
maxPages,
|
||||
cursor,
|
||||
syncContext
|
||||
)
|
||||
}
|
||||
|
||||
const spaceKey = spaceKeys[0]
|
||||
let spaceId = syncContext?.spaceId as string | undefined
|
||||
if (!spaceId) {
|
||||
spaceId = await resolveSpaceId(cloudId, accessToken, spaceKey)
|
||||
if (syncContext) syncContext.spaceId = spaceId
|
||||
}
|
||||
|
||||
if (contentType === 'all') {
|
||||
return listAllContentTypes(
|
||||
cloudId,
|
||||
accessToken,
|
||||
domain,
|
||||
spaceId,
|
||||
spaceKey,
|
||||
maxPages,
|
||||
cursor,
|
||||
syncContext
|
||||
)
|
||||
}
|
||||
|
||||
return listDocumentsV2(
|
||||
cloudId,
|
||||
accessToken,
|
||||
domain,
|
||||
spaceId,
|
||||
spaceKey,
|
||||
contentType,
|
||||
maxPages,
|
||||
cursor,
|
||||
syncContext
|
||||
)
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
externalId: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocument | null> => {
|
||||
const domain = normalizeConfluenceDomainHost(sourceConfig.domain as string)
|
||||
let cloudId = syncContext?.cloudId as string | undefined
|
||||
if (!cloudId) {
|
||||
cloudId = await getConfluenceCloudId(domain, accessToken)
|
||||
if (syncContext) syncContext.cloudId = cloudId
|
||||
}
|
||||
|
||||
// Try pages first, fall back to blogposts if not found
|
||||
let page: Record<string, unknown> | null = null
|
||||
for (const endpoint of ['pages', 'blogposts']) {
|
||||
const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/${endpoint}/${externalId}?body-format=storage`
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
|
||||
if (response.ok) {
|
||||
page = await response.json()
|
||||
break
|
||||
}
|
||||
if (response.status !== 404) {
|
||||
throw new Error(`Failed to get Confluence content: ${response.status}`)
|
||||
}
|
||||
}
|
||||
|
||||
if (!page) return null
|
||||
const body = page.body as Record<string, unknown> | undefined
|
||||
const storage = body?.storage as Record<string, unknown> | undefined
|
||||
const rawContent = (storage?.value as string) || ''
|
||||
const plainText = htmlToPlainText(rawContent)
|
||||
|
||||
const labelMap = await fetchLabelsForPages(cloudId, accessToken, [String(page.id)])
|
||||
const labels = labelMap.get(String(page.id)) ?? []
|
||||
|
||||
const links = page._links as Record<string, unknown> | undefined
|
||||
const stub = pageToStub(page, {
|
||||
spaceId: page.spaceId,
|
||||
labels,
|
||||
sourceUrl: links?.webui ? `https://${domain}/wiki${links.webui}` : undefined,
|
||||
})
|
||||
|
||||
return {
|
||||
...stub,
|
||||
content: plainText,
|
||||
contentDeferred: false,
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const domain = sourceConfig.domain as string
|
||||
const spaceKeys = parseMultiValue(sourceConfig.spaceKey)
|
||||
|
||||
if (!domain || spaceKeys.length === 0) {
|
||||
return { valid: false, error: 'Domain and at least one space key are required' }
|
||||
}
|
||||
|
||||
const maxPages = sourceConfig.maxPages as string | undefined
|
||||
if (maxPages && (Number.isNaN(Number(maxPages)) || Number(maxPages) <= 0)) {
|
||||
return { valid: false, error: 'Max pages must be a positive number' }
|
||||
}
|
||||
|
||||
try {
|
||||
const cloudId = await getConfluenceCloudId(domain, accessToken, VALIDATE_RETRY_OPTIONS)
|
||||
const params = new URLSearchParams()
|
||||
for (const key of spaceKeys) params.append('keys', key)
|
||||
params.append('limit', String(Math.max(spaceKeys.length, 1)))
|
||||
const spaceUrl = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces?${params.toString()}`
|
||||
const response = await fetchWithRetry(
|
||||
spaceUrl,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
if (!response.ok) {
|
||||
return { valid: false, error: `Failed to validate spaces: ${response.status}` }
|
||||
}
|
||||
const data = await response.json()
|
||||
const results = (data.results as Array<Record<string, unknown>> | undefined) ?? []
|
||||
const foundKeys = new Set(results.map((r) => String(r.key)))
|
||||
const missing = spaceKeys.filter((k) => !foundKeys.has(k))
|
||||
if (missing.length > 0) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `Space${missing.length > 1 ? 's' : ''} not found: ${missing.join(', ')}`,
|
||||
}
|
||||
}
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
return { valid: false, error: toError(error).message || 'Failed to validate configuration' }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
const joined = joinTagArray(metadata.labels)
|
||||
if (joined) result.labels = joined
|
||||
|
||||
if (metadata.version != null) {
|
||||
const num = Number(metadata.version)
|
||||
if (!Number.isNaN(num)) result.version = num
|
||||
}
|
||||
|
||||
const lastModified = parseTagDate(metadata.lastModified)
|
||||
if (lastModified) result.lastModified = lastModified
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists documents using the v2 API for a single content type (pages or blogposts).
|
||||
*/
|
||||
async function listDocumentsV2(
|
||||
cloudId: string,
|
||||
accessToken: string,
|
||||
domain: string,
|
||||
spaceId: string,
|
||||
spaceKey: string,
|
||||
contentType: string,
|
||||
maxPages: number,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> {
|
||||
const queryParams = new URLSearchParams()
|
||||
queryParams.append('limit', '250')
|
||||
if (cursor) {
|
||||
queryParams.append('cursor', cursor)
|
||||
}
|
||||
|
||||
const endpoint = contentType === 'blogpost' ? 'blogposts' : 'pages'
|
||||
const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces/${spaceId}/${endpoint}?${queryParams.toString()}`
|
||||
|
||||
logger.info(`Listing ${endpoint} in space ${spaceKey} (ID: ${spaceId})`)
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error(`Failed to list Confluence ${endpoint}`, {
|
||||
status: response.status,
|
||||
error: errorText,
|
||||
})
|
||||
throw new Error(`Failed to list Confluence ${endpoint}: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const results = data.results || []
|
||||
|
||||
const documents: ExternalDocument[] = results.map((page: Record<string, unknown>) => {
|
||||
const links = page._links as Record<string, string> | undefined
|
||||
return pageToStub(page, {
|
||||
spaceId: page.spaceId,
|
||||
sourceUrl: links?.webui ? `https://${domain}/wiki${links.webui}` : undefined,
|
||||
})
|
||||
})
|
||||
|
||||
let nextCursor: string | undefined
|
||||
const nextLink = (data._links as Record<string, string>)?.next
|
||||
if (nextLink) {
|
||||
try {
|
||||
nextCursor = new URL(nextLink, 'https://placeholder').searchParams.get('cursor') || undefined
|
||||
} catch {
|
||||
// Ignore malformed URLs
|
||||
}
|
||||
}
|
||||
|
||||
const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
const hitLimit = maxPages > 0 && totalFetched >= maxPages
|
||||
if (hitLimit && syncContext) syncContext.listingCapped = true
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hitLimit ? undefined : nextCursor,
|
||||
hasMore: hitLimit ? false : Boolean(nextCursor),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists both pages and blogposts using a compound cursor that tracks
|
||||
* pagination state for each content type independently.
|
||||
*/
|
||||
async function listAllContentTypes(
|
||||
cloudId: string,
|
||||
accessToken: string,
|
||||
domain: string,
|
||||
spaceId: string,
|
||||
spaceKey: string,
|
||||
maxPages: number,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> {
|
||||
let pageCursor: string | undefined
|
||||
let blogCursor: string | undefined
|
||||
let pagesDone = false
|
||||
let blogsDone = false
|
||||
|
||||
if (cursor) {
|
||||
try {
|
||||
const parsed = JSON.parse(cursor)
|
||||
pageCursor = parsed.page
|
||||
blogCursor = parsed.blog
|
||||
pagesDone = parsed.pagesDone === true
|
||||
blogsDone = parsed.blogsDone === true
|
||||
} catch {
|
||||
/**
|
||||
* Older bare-string cursors are no longer emitted; fall through and
|
||||
* restart instead of silently re-listing blogposts from page 0.
|
||||
*/
|
||||
logger.warn('Ignoring unparseable Confluence cursor; restarting listing')
|
||||
}
|
||||
}
|
||||
|
||||
const results: ExternalDocumentList = { documents: [], hasMore: false }
|
||||
|
||||
if (!pagesDone) {
|
||||
const pagesResult = await listDocumentsV2(
|
||||
cloudId,
|
||||
accessToken,
|
||||
domain,
|
||||
spaceId,
|
||||
spaceKey,
|
||||
'page',
|
||||
maxPages,
|
||||
pageCursor,
|
||||
syncContext
|
||||
)
|
||||
results.documents.push(...pagesResult.documents)
|
||||
pageCursor = pagesResult.nextCursor
|
||||
pagesDone = !pagesResult.hasMore
|
||||
}
|
||||
|
||||
if (!blogsDone) {
|
||||
const blogResult = await listDocumentsV2(
|
||||
cloudId,
|
||||
accessToken,
|
||||
domain,
|
||||
spaceId,
|
||||
spaceKey,
|
||||
'blogpost',
|
||||
maxPages,
|
||||
blogCursor,
|
||||
syncContext
|
||||
)
|
||||
results.documents.push(...blogResult.documents)
|
||||
blogCursor = blogResult.nextCursor
|
||||
blogsDone = !blogResult.hasMore
|
||||
}
|
||||
|
||||
results.hasMore = !pagesDone || !blogsDone
|
||||
|
||||
if (results.hasMore) {
|
||||
results.nextCursor = JSON.stringify({
|
||||
page: pageCursor,
|
||||
blog: blogCursor,
|
||||
pagesDone,
|
||||
blogsDone,
|
||||
})
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists documents using CQL search via the v1 API (used when label filtering is enabled).
|
||||
*/
|
||||
async function listDocumentsViaCql(
|
||||
cloudId: string,
|
||||
accessToken: string,
|
||||
domain: string,
|
||||
spaceKeys: string[],
|
||||
contentType: string,
|
||||
labelFilter: string,
|
||||
maxPages: number,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> {
|
||||
const labels = labelFilter
|
||||
.split(',')
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean)
|
||||
|
||||
// Build CQL query
|
||||
let cql = buildSpaceClause(spaceKeys)
|
||||
|
||||
if (contentType === 'blogpost') {
|
||||
cql += ' AND type="blogpost"'
|
||||
} else if (contentType === 'page' || !contentType) {
|
||||
cql += ' AND type="page"'
|
||||
}
|
||||
// contentType === 'all' — no type filter
|
||||
|
||||
if (labels.length === 1) {
|
||||
cql += ` AND label="${escapeCql(labels[0])}"`
|
||||
} else if (labels.length > 1) {
|
||||
const labelList = labels.map((l) => `"${escapeCql(l)}"`).join(',')
|
||||
cql += ` AND label in (${labelList})`
|
||||
}
|
||||
|
||||
const limit = maxPages > 0 ? Math.min(maxPages, 50) : 50
|
||||
const start = cursor ? Number(cursor) : 0
|
||||
|
||||
const queryParams = new URLSearchParams()
|
||||
queryParams.append('cql', cql)
|
||||
queryParams.append('limit', String(limit))
|
||||
queryParams.append('start', String(start))
|
||||
queryParams.append('expand', 'version,metadata.labels')
|
||||
|
||||
const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/rest/api/content/search?${queryParams.toString()}`
|
||||
|
||||
logger.info(`Searching Confluence via CQL: ${cql}`, { start, limit })
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error('Failed to search Confluence via CQL', {
|
||||
status: response.status,
|
||||
error: errorText,
|
||||
})
|
||||
throw new Error(`Failed to search Confluence via CQL: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const results = data.results || []
|
||||
|
||||
const documents: ExternalDocument[] = results.map((item: Record<string, unknown>) =>
|
||||
cqlResultToStub(item, domain)
|
||||
)
|
||||
|
||||
const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
const hitLimit = maxPages > 0 && totalFetched >= maxPages
|
||||
if (hitLimit && syncContext) syncContext.listingCapped = true
|
||||
|
||||
const totalSize = (data.totalSize as number) ?? 0
|
||||
const nextStart = start + results.length
|
||||
const hasMore = !hitLimit && nextStart < totalSize
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore ? String(nextStart) : undefined,
|
||||
hasMore,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a Confluence space key to its numeric space ID.
|
||||
*/
|
||||
async function resolveSpaceId(
|
||||
cloudId: string,
|
||||
accessToken: string,
|
||||
spaceKey: string
|
||||
): Promise<string> {
|
||||
const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces?keys=${encodeURIComponent(spaceKey)}&limit=1`
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to resolve space key "${spaceKey}": ${response.status}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const results = data.results || []
|
||||
|
||||
if (results.length === 0) {
|
||||
throw new Error(`Space "${spaceKey}" not found`)
|
||||
}
|
||||
|
||||
return String(results[0].id)
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { confluenceConnector } from '@/connectors/confluence/confluence'
|
||||
@@ -0,0 +1,87 @@
|
||||
import { ConfluenceIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const confluenceConnectorMeta: ConnectorMeta = {
|
||||
id: 'confluence',
|
||||
name: 'Confluence',
|
||||
description: 'Sync pages from a Confluence space',
|
||||
version: '1.1.0',
|
||||
icon: ConfluenceIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'oauth',
|
||||
provider: 'confluence',
|
||||
requiredScopes: [
|
||||
'read:confluence-content.all',
|
||||
'read:page:confluence',
|
||||
'read:blogpost:confluence',
|
||||
'read:space:confluence',
|
||||
'read:label:confluence',
|
||||
'search:confluence',
|
||||
'offline_access',
|
||||
],
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'domain',
|
||||
title: 'Confluence Domain',
|
||||
type: 'short-input',
|
||||
placeholder: 'yoursite.atlassian.net',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'spaceSelector',
|
||||
title: 'Spaces',
|
||||
type: 'selector',
|
||||
selectorKey: 'confluence.spaces',
|
||||
canonicalParamId: 'spaceKey',
|
||||
mode: 'basic',
|
||||
multi: true,
|
||||
dependsOn: ['domain'],
|
||||
placeholder: 'Select one or more spaces',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'spaceKey',
|
||||
title: 'Space Keys',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'spaceKey',
|
||||
mode: 'advanced',
|
||||
multi: true,
|
||||
placeholder: 'e.g. ENG, PRODUCT (comma-separated for multiple)',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'contentType',
|
||||
title: 'Content Type',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
options: [
|
||||
{ label: 'Pages only', id: 'page' },
|
||||
{ label: 'Blog posts only', id: 'blogpost' },
|
||||
{ label: 'All content', id: 'all' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'labelFilter',
|
||||
title: 'Filter by Label',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. published, engineering',
|
||||
},
|
||||
{
|
||||
id: 'maxPages',
|
||||
title: 'Max Pages',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 500 (default: unlimited)',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'labels', displayName: 'Labels', fieldType: 'text' },
|
||||
{ id: 'version', displayName: 'Version', fieldType: 'number' },
|
||||
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { DEFAULT_MAX_MESSAGES, discordConnectorMeta } from '@/connectors/discord/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { computeContentHash, parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('DiscordConnector')
|
||||
|
||||
const DISCORD_API_BASE = 'https://discord.com/api/v10'
|
||||
const MESSAGES_PER_PAGE = 100
|
||||
|
||||
interface DiscordMessage {
|
||||
id: string
|
||||
channel_id: string
|
||||
author: {
|
||||
id: string
|
||||
username: string
|
||||
discriminator?: string
|
||||
bot?: boolean
|
||||
}
|
||||
content: string
|
||||
timestamp: string
|
||||
edited_timestamp?: string | null
|
||||
type: number
|
||||
}
|
||||
|
||||
interface DiscordChannel {
|
||||
id: string
|
||||
name?: string
|
||||
topic?: string | null
|
||||
guild_id?: string
|
||||
type: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Calls the Discord REST API with Bot token auth.
|
||||
* Unlike Slack, Discord returns proper HTTP status codes for errors.
|
||||
*/
|
||||
async function discordApiGet(
|
||||
path: string,
|
||||
botToken: string,
|
||||
params?: Record<string, string>,
|
||||
retryOptions?: Parameters<typeof fetchWithRetry>[2]
|
||||
): Promise<unknown> {
|
||||
const queryParams = params ? `?${new URLSearchParams(params).toString()}` : ''
|
||||
const url = `${DISCORD_API_BASE}${path}${queryParams}`
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
url,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bot ${botToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
},
|
||||
retryOptions
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const body = await response.text().catch(() => '')
|
||||
throw new Error(`Discord API error ${response.status}: ${body}`)
|
||||
}
|
||||
|
||||
return response.json()
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches all messages from a channel, up to a maximum count, using `before`-based pagination.
|
||||
* Discord returns messages newest-first; we collect them all then reverse for chronological order.
|
||||
*/
|
||||
async function fetchChannelMessages(
|
||||
botToken: string,
|
||||
channelId: string,
|
||||
maxMessages: number
|
||||
): Promise<{ messages: DiscordMessage[]; lastActivityTs?: string }> {
|
||||
const allMessages: DiscordMessage[] = []
|
||||
let beforeId: string | undefined
|
||||
let lastActivityTs: string | undefined
|
||||
|
||||
while (allMessages.length < maxMessages) {
|
||||
const limit = Math.min(MESSAGES_PER_PAGE, maxMessages - allMessages.length)
|
||||
const params: Record<string, string> = { limit: String(limit) }
|
||||
if (beforeId) {
|
||||
params.before = beforeId
|
||||
}
|
||||
|
||||
const messages = (await discordApiGet(
|
||||
`/channels/${channelId}/messages`,
|
||||
botToken,
|
||||
params
|
||||
)) as DiscordMessage[]
|
||||
|
||||
if (!messages || messages.length === 0) break
|
||||
|
||||
if (!lastActivityTs && messages.length > 0) {
|
||||
lastActivityTs = messages[0].timestamp
|
||||
}
|
||||
|
||||
allMessages.push(...messages)
|
||||
|
||||
// The last message in the batch is the oldest; use its ID for the next page
|
||||
beforeId = messages[messages.length - 1].id
|
||||
|
||||
// If we got fewer than requested, there are no more messages
|
||||
if (messages.length < limit) break
|
||||
}
|
||||
|
||||
return { messages: allMessages.slice(0, maxMessages), lastActivityTs }
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts fetched messages into a single document content string.
|
||||
* Each line: "[ISO timestamp] username: message content"
|
||||
* Messages are returned chronologically (oldest first).
|
||||
*/
|
||||
function formatMessages(messages: DiscordMessage[]): string {
|
||||
const lines: string[] = []
|
||||
|
||||
// Discord returns newest first; reverse for chronological order
|
||||
const chronological = [...messages].reverse()
|
||||
|
||||
for (const msg of chronological) {
|
||||
// Skip system messages (type 0 = DEFAULT, type 19 = REPLY are user messages)
|
||||
if (msg.type !== 0 && msg.type !== 19) continue
|
||||
if (!msg.content) continue
|
||||
|
||||
const userName = msg.author.username
|
||||
lines.push(`[${msg.timestamp}] ${userName}: ${msg.content}`)
|
||||
}
|
||||
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
export const discordConnector: ConnectorConfig = {
|
||||
...discordConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
_cursor?: string,
|
||||
_syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const channelId = sourceConfig.channelId as string
|
||||
if (!channelId?.trim()) {
|
||||
throw new Error('Channel ID is required')
|
||||
}
|
||||
|
||||
const maxMessages = sourceConfig.maxMessages
|
||||
? Number(sourceConfig.maxMessages)
|
||||
: DEFAULT_MAX_MESSAGES
|
||||
|
||||
logger.info('Syncing Discord channel', { channelId, maxMessages })
|
||||
|
||||
const channel = (await discordApiGet(
|
||||
`/channels/${channelId.trim()}`,
|
||||
accessToken
|
||||
)) as DiscordChannel
|
||||
|
||||
const { messages, lastActivityTs } = await fetchChannelMessages(
|
||||
accessToken,
|
||||
channel.id,
|
||||
maxMessages
|
||||
)
|
||||
|
||||
const content = formatMessages(messages)
|
||||
if (!content.trim()) {
|
||||
logger.info('No messages found in Discord channel', { channelId: channel.id })
|
||||
return { documents: [], hasMore: false }
|
||||
}
|
||||
|
||||
const contentHash = await computeContentHash(content)
|
||||
const channelName = channel.name || channel.id
|
||||
const sourceUrl = `https://discord.com/channels/${channel.guild_id || '@me'}/${channel.id}`
|
||||
|
||||
const document: ExternalDocument = {
|
||||
externalId: channel.id,
|
||||
title: `#${channelName}`,
|
||||
content,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl,
|
||||
contentHash,
|
||||
metadata: {
|
||||
channelName,
|
||||
messageCount: messages.length,
|
||||
lastActivity: lastActivityTs,
|
||||
topic: channel.topic ?? undefined,
|
||||
},
|
||||
}
|
||||
|
||||
return {
|
||||
documents: [document],
|
||||
hasMore: false,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
const maxMessages = sourceConfig.maxMessages
|
||||
? Number(sourceConfig.maxMessages)
|
||||
: DEFAULT_MAX_MESSAGES
|
||||
|
||||
try {
|
||||
const channel = (await discordApiGet(
|
||||
`/channels/${externalId}`,
|
||||
accessToken
|
||||
)) as DiscordChannel
|
||||
|
||||
const { messages, lastActivityTs } = await fetchChannelMessages(
|
||||
accessToken,
|
||||
externalId,
|
||||
maxMessages
|
||||
)
|
||||
|
||||
const content = formatMessages(messages)
|
||||
if (!content.trim()) return null
|
||||
|
||||
const contentHash = await computeContentHash(content)
|
||||
const channelName = channel.name || channel.id
|
||||
const sourceUrl = `https://discord.com/channels/${channel.guild_id || '@me'}/${channel.id}`
|
||||
|
||||
return {
|
||||
externalId: channel.id,
|
||||
title: `#${channelName}`,
|
||||
content,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl,
|
||||
contentHash,
|
||||
metadata: {
|
||||
channelName,
|
||||
messageCount: messages.length,
|
||||
lastActivity: lastActivityTs,
|
||||
topic: channel.topic ?? undefined,
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Failed to get Discord channel document', {
|
||||
externalId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const channelId = sourceConfig.channelId as string | undefined
|
||||
const maxMessages = sourceConfig.maxMessages as string | undefined
|
||||
|
||||
if (!channelId?.trim()) {
|
||||
return { valid: false, error: 'Channel ID is required' }
|
||||
}
|
||||
|
||||
if (maxMessages && (Number.isNaN(Number(maxMessages)) || Number(maxMessages) <= 0)) {
|
||||
return { valid: false, error: 'Max messages must be a positive number' }
|
||||
}
|
||||
|
||||
try {
|
||||
await discordApiGet(
|
||||
`/channels/${channelId.trim()}`,
|
||||
accessToken,
|
||||
undefined,
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
if (message.includes('401') || message.includes('403')) {
|
||||
return { valid: false, error: 'Invalid bot token or missing permissions for this channel' }
|
||||
}
|
||||
if (message.includes('404')) {
|
||||
return { valid: false, error: `Channel not found: ${channelId}` }
|
||||
}
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.channelName === 'string') {
|
||||
result.channelName = metadata.channelName
|
||||
}
|
||||
|
||||
if (typeof metadata.messageCount === 'number') {
|
||||
result.messageCount = metadata.messageCount
|
||||
}
|
||||
|
||||
const lastActivity = parseTagDate(metadata.lastActivity)
|
||||
if (lastActivity) {
|
||||
result.lastActivity = lastActivity
|
||||
}
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { discordConnector } from '@/connectors/discord/discord'
|
||||
@@ -0,0 +1,42 @@
|
||||
import { DiscordIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const DEFAULT_MAX_MESSAGES = 1000
|
||||
|
||||
export const discordConnectorMeta: ConnectorMeta = {
|
||||
id: 'discord',
|
||||
name: 'Discord',
|
||||
description: 'Sync channel messages from Discord',
|
||||
version: '1.0.0',
|
||||
icon: DiscordIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'apiKey',
|
||||
label: 'Bot Token',
|
||||
placeholder: 'Enter your Discord bot token',
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'channelId',
|
||||
title: 'Channel ID',
|
||||
type: 'short-input',
|
||||
placeholder: 'e.g. 123456789012345678',
|
||||
required: true,
|
||||
description: 'The Discord channel ID to sync messages from',
|
||||
},
|
||||
{
|
||||
id: 'maxMessages',
|
||||
title: 'Max Messages',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: `e.g. 500 (default: ${DEFAULT_MAX_MESSAGES})`,
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'channelName', displayName: 'Channel Name', fieldType: 'text' },
|
||||
{ id: 'messageCount', displayName: 'Message Count', fieldType: 'number' },
|
||||
{ id: 'lastActivity', displayName: 'Last Activity', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,620 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { docusignConnectorMeta } from '@/connectors/docusign/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('DocuSignConnector')
|
||||
|
||||
/**
|
||||
* DocuSign OAuth userinfo endpoint. Sim's DocuSign OAuth integration is wired to the
|
||||
* demo/sandbox authorization server (`account-d.docusign.com`, see lib/oauth/oauth.ts token
|
||||
* endpoint), so the connector resolves account info from the matching demo userinfo host.
|
||||
* The production host is `https://account.docusign.com/oauth/userinfo`.
|
||||
*/
|
||||
const DOCUSIGN_USERINFO_URL = 'https://account-d.docusign.com/oauth/userinfo'
|
||||
|
||||
/**
|
||||
* DocuSign web-app base for envelope deep links. MUST match the same environment as
|
||||
* {@link DOCUSIGN_USERINFO_URL}: demo/sandbox envelopes only exist in the demo web app
|
||||
* (`appdemo.docusign.com`), not production (`app.docusign.com`). Keep these in lockstep
|
||||
* if the OAuth environment ever changes.
|
||||
*/
|
||||
const DOCUSIGN_WEB_BASE = 'https://appdemo.docusign.com'
|
||||
|
||||
const DEFAULT_LOOKBACK_DAYS = 90
|
||||
const MAX_PAGE_SIZE = 100
|
||||
const DEFAULT_MAX_ENVELOPES = 0
|
||||
/**
|
||||
* Days of overlap added to the incremental sync window. DocuSign status changes are
|
||||
* indexed by `statusChangedDateTime`, but webhook/processing lag and clock skew can let
|
||||
* a change land slightly before the recorded sync time. A small overlap re-scans recent
|
||||
* envelopes so late-recorded changes are not missed.
|
||||
*/
|
||||
const INCREMENTAL_OVERLAP_DAYS = 2
|
||||
const MS_PER_DAY = 24 * 60 * 60 * 1000
|
||||
|
||||
interface DocuSignAccount {
|
||||
account_id?: string
|
||||
base_uri?: string
|
||||
is_default?: boolean
|
||||
}
|
||||
|
||||
interface DocuSignUserInfo {
|
||||
accounts?: DocuSignAccount[]
|
||||
}
|
||||
|
||||
interface ResolvedAccount {
|
||||
accountId: string
|
||||
baseUri: string
|
||||
}
|
||||
|
||||
interface DocuSignSigner {
|
||||
recipientId?: string
|
||||
name?: string
|
||||
email?: string
|
||||
status?: string
|
||||
}
|
||||
|
||||
interface DocuSignRecipients {
|
||||
signers?: DocuSignSigner[]
|
||||
carbonCopies?: DocuSignSigner[]
|
||||
agents?: DocuSignSigner[]
|
||||
editors?: DocuSignSigner[]
|
||||
certifiedDeliveries?: DocuSignSigner[]
|
||||
}
|
||||
|
||||
interface DocuSignCustomField {
|
||||
name?: string
|
||||
value?: string
|
||||
}
|
||||
|
||||
interface DocuSignCustomFields {
|
||||
textCustomFields?: DocuSignCustomField[]
|
||||
listCustomFields?: DocuSignCustomField[]
|
||||
}
|
||||
|
||||
interface DocuSignDocument {
|
||||
documentId?: string
|
||||
name?: string
|
||||
}
|
||||
|
||||
interface DocuSignEnvelope {
|
||||
envelopeId?: string
|
||||
status?: string
|
||||
emailSubject?: string
|
||||
emailBlurb?: string
|
||||
sentDateTime?: string
|
||||
completedDateTime?: string
|
||||
createdDateTime?: string
|
||||
statusChangedDateTime?: string
|
||||
lastModifiedDateTime?: string
|
||||
sender?: { userName?: string; email?: string }
|
||||
recipients?: DocuSignRecipients
|
||||
customFields?: DocuSignCustomFields
|
||||
envelopeDocuments?: DocuSignDocument[]
|
||||
}
|
||||
|
||||
interface DocuSignEnvelopesListResponse {
|
||||
envelopes?: DocuSignEnvelope[]
|
||||
resultSetSize?: string
|
||||
totalSetSize?: string
|
||||
endPosition?: string
|
||||
nextUri?: string
|
||||
}
|
||||
|
||||
interface DocuSignFormValue {
|
||||
name?: string
|
||||
value?: string
|
||||
}
|
||||
|
||||
interface DocuSignRecipientFormData {
|
||||
formData?: DocuSignFormValue[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Response shape of the envelope `form_data` endpoint. The envelope-level entered tab
|
||||
* values are returned as a top-level `formData` array of `{ name, value }` pairs (there
|
||||
* is no nested `formValues` property). Per-recipient values live under `recipientFormData`.
|
||||
*/
|
||||
interface DocuSignFormData {
|
||||
formData?: DocuSignFormValue[]
|
||||
recipientFormData?: DocuSignRecipientFormData[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a Date as a UTC ISO 8601 string with explicit time zone offset, the format
|
||||
* DocuSign recommends for the `from_date` filter on listStatusChanges.
|
||||
*/
|
||||
function formatFromDate(date: Date): string {
|
||||
return date.toISOString()
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the effective lookback window in days, narrowing to the time since the last
|
||||
* successful sync (plus an overlap to catch late-recorded status changes) when incremental
|
||||
* sync is active.
|
||||
*/
|
||||
function computeLookbackDays(
|
||||
sourceConfig: Record<string, unknown>,
|
||||
lastSyncAt: Date | undefined
|
||||
): number {
|
||||
const raw = sourceConfig.lookback as string | undefined
|
||||
const configured = Number(raw)
|
||||
const baseline =
|
||||
Number.isFinite(configured) && configured > 0 ? Math.floor(configured) : DEFAULT_LOOKBACK_DAYS
|
||||
|
||||
if (!lastSyncAt) return baseline
|
||||
|
||||
const sinceLastSync = Math.ceil((Date.now() - lastSyncAt.getTime()) / MS_PER_DAY)
|
||||
const incremental = Math.max(sinceLastSync + INCREMENTAL_OVERLAP_DAYS, INCREMENTAL_OVERLAP_DAYS)
|
||||
return Math.min(incremental, baseline)
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves and caches the user's DocuSign account ID and base URI in the sync context.
|
||||
* The userinfo lookup is expensive and identical across every page of a sync run, so it is
|
||||
* cached on the shared `syncContext` (mirrors the Gmail connector's label cache pattern).
|
||||
*/
|
||||
async function resolveAccount(
|
||||
accessToken: string,
|
||||
syncContext: Record<string, unknown> | undefined,
|
||||
retryOptions?: Parameters<typeof fetchWithRetry>[2]
|
||||
): Promise<ResolvedAccount> {
|
||||
const cacheKey = '_docusignAccount'
|
||||
const cached = syncContext?.[cacheKey] as ResolvedAccount | undefined
|
||||
if (cached) return cached
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
DOCUSIGN_USERINFO_URL,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
},
|
||||
retryOptions
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
throw new Error(
|
||||
`Failed to resolve DocuSign account: ${response.status}${
|
||||
errorText ? ` — ${errorText.slice(0, 200)}` : ''
|
||||
}`
|
||||
)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as DocuSignUserInfo
|
||||
const accounts = Array.isArray(data.accounts) ? data.accounts : []
|
||||
const account = accounts.find((a) => a.is_default) ?? accounts[0]
|
||||
|
||||
if (!account?.account_id || !account.base_uri) {
|
||||
throw new Error('No accessible DocuSign account found for this user')
|
||||
}
|
||||
|
||||
const resolved: ResolvedAccount = {
|
||||
accountId: account.account_id,
|
||||
baseUri: account.base_uri,
|
||||
}
|
||||
if (syncContext) syncContext[cacheKey] = resolved
|
||||
return resolved
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the REST API base for an account, e.g.
|
||||
* `https://demo.docusign.net/restapi/v2.1/accounts/{accountId}`.
|
||||
*/
|
||||
function apiBaseFor(account: ResolvedAccount): string {
|
||||
return `${account.baseUri}/restapi/v2.1/accounts/${account.accountId}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a metadata-based content hash. Identical between the listDocuments stub and the
|
||||
* getDocument result so the sync engine can detect changes without downloading content.
|
||||
*/
|
||||
function buildContentHash(envelope: DocuSignEnvelope): string {
|
||||
const envelopeId = envelope.envelopeId ?? ''
|
||||
const changeIndicator =
|
||||
envelope.statusChangedDateTime ?? envelope.lastModifiedDateTime ?? envelope.status ?? ''
|
||||
return `docusign:${envelopeId}:${changeIndicator}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a DocuSign web console link to the envelope, when an envelope ID is present.
|
||||
*/
|
||||
function buildSourceUrl(envelopeId: string | undefined): string | undefined {
|
||||
if (!envelopeId) return undefined
|
||||
return `${DOCUSIGN_WEB_BASE}/documents/details/${envelopeId}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the sender's display name. The envelope summary exposes sender details only via
|
||||
* the nested `sender` (userInfo) object — there is no top-level `senderName` field.
|
||||
*/
|
||||
function resolveSenderName(envelope: DocuSignEnvelope): string | undefined {
|
||||
return envelope.sender?.userName ?? undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the sender's email. As with the name, this is only available on the nested
|
||||
* `sender` (userInfo) object.
|
||||
*/
|
||||
function resolveSenderEmail(envelope: DocuSignEnvelope): string | undefined {
|
||||
return envelope.sender?.email ?? undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Collects all named recipients across roles into a flat list of name/email/status entries.
|
||||
*/
|
||||
function collectRecipients(
|
||||
recipients: DocuSignRecipients | undefined
|
||||
): { name: string; email: string; status: string }[] {
|
||||
if (!recipients) return []
|
||||
const groups = [
|
||||
recipients.signers,
|
||||
recipients.agents,
|
||||
recipients.editors,
|
||||
recipients.carbonCopies,
|
||||
recipients.certifiedDeliveries,
|
||||
]
|
||||
const out: { name: string; email: string; status: string }[] = []
|
||||
for (const group of groups) {
|
||||
if (!Array.isArray(group)) continue
|
||||
for (const r of group) {
|
||||
out.push({
|
||||
name: r.name?.trim() ?? '',
|
||||
email: r.email?.trim() ?? '',
|
||||
status: r.status?.trim() ?? '',
|
||||
})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds shared metadata used by both the stub and the full document (and by mapTags).
|
||||
*
|
||||
* Every field consumed by `mapTags` is read from the envelope object that the list
|
||||
* endpoint (`GET /envelopes?from_date=...`) returns directly on each list entry — the
|
||||
* list response carries the full envelope summary (status, subject, sender, and all
|
||||
* lifecycle timestamps), not a thin stub. The sync engine calls `mapTags` on this stub
|
||||
* metadata, so tags are populated without any per-envelope detail fetch.
|
||||
*/
|
||||
function buildMetadata(envelope: DocuSignEnvelope): Record<string, unknown> {
|
||||
const recipients = collectRecipients(envelope.recipients)
|
||||
return {
|
||||
status: envelope.status,
|
||||
subject: envelope.emailSubject,
|
||||
senderName: resolveSenderName(envelope),
|
||||
senderEmail: resolveSenderEmail(envelope),
|
||||
sentDate: envelope.sentDateTime,
|
||||
completedDate: envelope.completedDateTime,
|
||||
recipientNames: recipients.map((r) => r.name).filter(Boolean),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a lightweight document stub from an envelope list entry. No per-envelope API
|
||||
* calls are made — content is fetched lazily by getDocument for new/changed envelopes only.
|
||||
*/
|
||||
function envelopeToStub(envelope: DocuSignEnvelope): ExternalDocument {
|
||||
return {
|
||||
externalId: envelope.envelopeId ?? '',
|
||||
title: envelope.emailSubject?.trim() || 'Untitled DocuSign Envelope',
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: buildSourceUrl(envelope.envelopeId),
|
||||
contentHash: buildContentHash(envelope),
|
||||
metadata: buildMetadata(envelope),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders the envelope's text metadata into a plain-text document. Envelope documents are
|
||||
* PDFs/binary and are intentionally NOT downloaded or text-extracted — only the document
|
||||
* names are indexed alongside subject, status, sender, recipients, custom fields, and form
|
||||
* data field name/value pairs.
|
||||
*/
|
||||
function buildContent(
|
||||
envelope: DocuSignEnvelope,
|
||||
customFields: DocuSignCustomField[],
|
||||
documents: DocuSignDocument[],
|
||||
formValues: DocuSignFormValue[]
|
||||
): string {
|
||||
const parts: string[] = []
|
||||
|
||||
if (envelope.emailSubject) parts.push(`Subject: ${envelope.emailSubject}`)
|
||||
if (envelope.emailBlurb) parts.push(`Message: ${envelope.emailBlurb}`)
|
||||
if (envelope.status) parts.push(`Status: ${envelope.status}`)
|
||||
|
||||
const senderName = resolveSenderName(envelope)
|
||||
const senderEmail = resolveSenderEmail(envelope)
|
||||
if (senderName || senderEmail) {
|
||||
parts.push(`Sender: ${[senderName, senderEmail].filter(Boolean).join(' ')}`.trim())
|
||||
}
|
||||
if (envelope.sentDateTime) parts.push(`Sent: ${envelope.sentDateTime}`)
|
||||
if (envelope.completedDateTime) parts.push(`Completed: ${envelope.completedDateTime}`)
|
||||
|
||||
const recipients = collectRecipients(envelope.recipients)
|
||||
if (recipients.length > 0) {
|
||||
parts.push('')
|
||||
parts.push('--- Recipients ---')
|
||||
for (const r of recipients) {
|
||||
const label = [r.name, r.email ? `<${r.email}>` : '', r.status ? `(${r.status})` : '']
|
||||
.filter(Boolean)
|
||||
.join(' ')
|
||||
if (label) parts.push(label)
|
||||
}
|
||||
}
|
||||
|
||||
const fields = customFields.filter((f) => f.name?.trim())
|
||||
if (fields.length > 0) {
|
||||
parts.push('')
|
||||
parts.push('--- Custom Fields ---')
|
||||
for (const f of fields) {
|
||||
parts.push(`${f.name}: ${f.value ?? ''}`)
|
||||
}
|
||||
}
|
||||
|
||||
const docNames = documents.map((d) => d.name?.trim()).filter((n): n is string => Boolean(n))
|
||||
if (docNames.length > 0) {
|
||||
parts.push('')
|
||||
parts.push('--- Documents ---')
|
||||
for (const name of docNames) parts.push(name)
|
||||
}
|
||||
|
||||
const formPairs = formValues.filter((v) => v.name?.trim())
|
||||
if (formPairs.length > 0) {
|
||||
parts.push('')
|
||||
parts.push('--- Form Data ---')
|
||||
for (const v of formPairs) {
|
||||
parts.push(`${v.name}: ${v.value ?? ''}`)
|
||||
}
|
||||
}
|
||||
|
||||
return parts.join('\n').trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the envelope's form data (signer-entered tab values). Returns an empty list on
|
||||
* 404 or any error — form data is supplementary and a missing endpoint must not fail the doc.
|
||||
*/
|
||||
async function fetchFormValues(
|
||||
apiBase: string,
|
||||
accessToken: string,
|
||||
envelopeId: string
|
||||
): Promise<DocuSignFormValue[]> {
|
||||
try {
|
||||
const response = await fetchWithRetry(`${apiBase}/envelopes/${envelopeId}/form_data`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
if (!response.ok) return []
|
||||
const data = (await response.json()) as DocuSignFormData
|
||||
const values: DocuSignFormValue[] = []
|
||||
if (Array.isArray(data.formData)) values.push(...data.formData)
|
||||
if (Array.isArray(data.recipientFormData)) {
|
||||
for (const recipient of data.recipientFormData) {
|
||||
if (Array.isArray(recipient.formData)) values.push(...recipient.formData)
|
||||
}
|
||||
}
|
||||
return values
|
||||
} catch (error) {
|
||||
logger.warn('Failed to fetch DocuSign form data', {
|
||||
envelopeId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
export const docusignConnector: ConnectorConfig = {
|
||||
...docusignConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>,
|
||||
lastSyncAt?: Date
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const account = await resolveAccount(accessToken, syncContext)
|
||||
const apiBase = apiBaseFor(account)
|
||||
|
||||
const lookbackDays = computeLookbackDays(sourceConfig, lastSyncAt)
|
||||
const maxEnvelopes = sourceConfig.maxEnvelopes
|
||||
? Number(sourceConfig.maxEnvelopes)
|
||||
: DEFAULT_MAX_ENVELOPES
|
||||
|
||||
const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
|
||||
if (maxEnvelopes > 0 && prevFetched >= maxEnvelopes) {
|
||||
return { documents: [], hasMore: false }
|
||||
}
|
||||
|
||||
const startPosition = cursor ? Number(cursor) : 0
|
||||
const cachedFromDate = syncContext?.docusignFromDate as string | undefined
|
||||
const fromDate = cachedFromDate
|
||||
? new Date(cachedFromDate)
|
||||
: new Date(Date.now() - lookbackDays * MS_PER_DAY)
|
||||
if (syncContext && !cachedFromDate) syncContext.docusignFromDate = fromDate.toISOString()
|
||||
|
||||
const queryParams = new URLSearchParams({
|
||||
from_date: formatFromDate(fromDate),
|
||||
include: 'recipients,custom_fields',
|
||||
count: String(MAX_PAGE_SIZE),
|
||||
start_position: String(startPosition),
|
||||
})
|
||||
const statusFilter = typeof sourceConfig.status === 'string' ? sourceConfig.status.trim() : ''
|
||||
if (statusFilter) queryParams.set('status', statusFilter)
|
||||
|
||||
const url = `${apiBase}/envelopes?${queryParams.toString()}`
|
||||
|
||||
logger.info('Listing DocuSign envelopes', {
|
||||
from: formatFromDate(fromDate),
|
||||
startPosition,
|
||||
incremental: Boolean(lastSyncAt),
|
||||
})
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
logger.error('Failed to list DocuSign envelopes', {
|
||||
status: response.status,
|
||||
error: errorText.slice(0, 500),
|
||||
})
|
||||
throw new Error(`Failed to list DocuSign envelopes: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as DocuSignEnvelopesListResponse
|
||||
const envelopes = (data.envelopes ?? []).filter((e) => e.envelopeId)
|
||||
const pageDocuments = envelopes.map(envelopeToStub)
|
||||
|
||||
let documents = pageDocuments
|
||||
if (maxEnvelopes > 0) {
|
||||
const remaining = Math.max(0, maxEnvelopes - prevFetched)
|
||||
if (pageDocuments.length > remaining) {
|
||||
documents = pageDocuments.slice(0, remaining)
|
||||
}
|
||||
}
|
||||
|
||||
const totalFetched = prevFetched + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
|
||||
const hitLimit = maxEnvelopes > 0 && totalFetched >= maxEnvelopes
|
||||
if (hitLimit && syncContext) syncContext.listingCapped = true
|
||||
|
||||
const endPosition = Number(data.endPosition)
|
||||
const totalSetSize = Number(data.totalSetSize)
|
||||
const hasNextPage =
|
||||
pageDocuments.length === MAX_PAGE_SIZE &&
|
||||
Number.isFinite(endPosition) &&
|
||||
Number.isFinite(totalSetSize) &&
|
||||
endPosition + 1 < totalSetSize
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: !hitLimit && hasNextPage ? String(endPosition + 1) : undefined,
|
||||
hasMore: !hitLimit && hasNextPage,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
_sourceConfig: Record<string, unknown>,
|
||||
externalId: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocument | null> => {
|
||||
try {
|
||||
if (!externalId) return null
|
||||
|
||||
const account = await resolveAccount(accessToken, syncContext)
|
||||
const apiBase = apiBaseFor(account)
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
`${apiBase}/envelopes/${externalId}?include=recipients,custom_fields,documents`,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404 || response.status === 410) return null
|
||||
throw new Error(`Failed to fetch DocuSign envelope: ${response.status}`)
|
||||
}
|
||||
|
||||
const envelope = (await response.json()) as DocuSignEnvelope
|
||||
if (!envelope.envelopeId) return null
|
||||
|
||||
const customFields: DocuSignCustomField[] = [
|
||||
...(envelope.customFields?.textCustomFields ?? []),
|
||||
...(envelope.customFields?.listCustomFields ?? []),
|
||||
]
|
||||
|
||||
const documents = Array.isArray(envelope.envelopeDocuments) ? envelope.envelopeDocuments : []
|
||||
|
||||
const formValues = await fetchFormValues(apiBase, accessToken, externalId)
|
||||
|
||||
const content = buildContent(envelope, customFields, documents, formValues)
|
||||
if (!content.trim()) return null
|
||||
|
||||
return {
|
||||
externalId: envelope.envelopeId,
|
||||
title: envelope.emailSubject?.trim() || 'Untitled DocuSign Envelope',
|
||||
content,
|
||||
contentDeferred: false,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: buildSourceUrl(envelope.envelopeId),
|
||||
contentHash: buildContentHash(envelope),
|
||||
metadata: buildMetadata(envelope),
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Failed to get DocuSign envelope', {
|
||||
externalId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const maxEnvelopes = sourceConfig.maxEnvelopes as string | undefined
|
||||
if (maxEnvelopes && (Number.isNaN(Number(maxEnvelopes)) || Number(maxEnvelopes) < 0)) {
|
||||
return { valid: false, error: 'Max envelopes must be a non-negative number' }
|
||||
}
|
||||
|
||||
try {
|
||||
await resolveAccount(accessToken, undefined, VALIDATE_RETRY_OPTIONS)
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.status === 'string' && metadata.status.trim()) {
|
||||
result.status = metadata.status
|
||||
}
|
||||
|
||||
const sender = [metadata.senderName, metadata.senderEmail]
|
||||
.filter((v): v is string => typeof v === 'string' && v.trim().length > 0)
|
||||
.join(' ')
|
||||
.trim()
|
||||
if (sender) result.sender = sender
|
||||
|
||||
if (typeof metadata.subject === 'string' && metadata.subject.trim()) {
|
||||
result.subject = metadata.subject.trim()
|
||||
}
|
||||
|
||||
const sentDate = parseTagDate(metadata.sentDate)
|
||||
if (sentDate) result.sentDate = sentDate
|
||||
|
||||
const completedDate = parseTagDate(metadata.completedDate)
|
||||
if (completedDate) result.completedDate = completedDate
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { docusignConnector } from '@/connectors/docusign/docusign'
|
||||
@@ -0,0 +1,68 @@
|
||||
import { DocuSignIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const docusignConnectorMeta: ConnectorMeta = {
|
||||
id: 'docusign',
|
||||
name: 'DocuSign',
|
||||
description: 'Sync envelope and agreement metadata from DocuSign into your knowledge base',
|
||||
version: '1.0.0',
|
||||
icon: DocuSignIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'oauth',
|
||||
provider: 'docusign',
|
||||
requiredScopes: ['signature'],
|
||||
},
|
||||
|
||||
supportsIncrementalSync: true,
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'lookback',
|
||||
title: 'Date Range',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
options: [
|
||||
{ label: 'Last 30 days', id: '30' },
|
||||
{ label: 'Last 90 days (recommended)', id: '90' },
|
||||
{ label: 'Last 6 months', id: '180' },
|
||||
],
|
||||
description:
|
||||
'On initial sync only. Filters envelopes by when their status last changed (from_date).',
|
||||
},
|
||||
{
|
||||
id: 'status',
|
||||
title: 'Filter by Status',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. completed (or completed,sent)',
|
||||
description:
|
||||
'Only sync envelopes with these statuses (comma-separated: created, sent, delivered, completed, declined, voided). Leave blank to sync all.',
|
||||
},
|
||||
{
|
||||
id: 'maxEnvelopes',
|
||||
title: 'Max Envelopes',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 200 (default: unlimited)',
|
||||
},
|
||||
],
|
||||
|
||||
/**
|
||||
* Tag definitions are constrained by the document table's slot pools: 7 text slots but
|
||||
* only 2 date slots (`date1`, `date2`). The two highest-value envelope dates — when it was
|
||||
* sent and when it completed — claim both date slots. `createdDateTime` is intentionally
|
||||
* NOT exposed as a date tag: it nearly always equals `sentDateTime` for sent envelopes, so
|
||||
* adding it would consume a (non-existent) third date slot and be silently dropped by the
|
||||
* slot allocator. `emailSubject` is exposed as a filterable text tag (distinct from the
|
||||
* document title) since text slots are plentiful.
|
||||
*/
|
||||
tagDefinitions: [
|
||||
{ id: 'status', displayName: 'Status', fieldType: 'text' },
|
||||
{ id: 'sender', displayName: 'Sender', fieldType: 'text' },
|
||||
{ id: 'subject', displayName: 'Subject', fieldType: 'text' },
|
||||
{ id: 'sentDate', displayName: 'Sent Date', fieldType: 'date' },
|
||||
{ id: 'completedDate', displayName: 'Completed Date', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { dropboxConnectorMeta } from '@/connectors/dropbox/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import {
|
||||
CONNECTOR_MAX_FILE_BYTES,
|
||||
ConnectorFileTooLargeError,
|
||||
htmlToPlainText,
|
||||
isSkippedDocument,
|
||||
markSkipped,
|
||||
parseTagDate,
|
||||
readBodyWithLimit,
|
||||
sizeLimitSkipReason,
|
||||
stubOrSkipBySize,
|
||||
takeIndexableWithinCap,
|
||||
} from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('DropboxConnector')
|
||||
|
||||
const SUPPORTED_EXTENSIONS = new Set([
|
||||
'.txt',
|
||||
'.md',
|
||||
'.markdown',
|
||||
'.html',
|
||||
'.htm',
|
||||
'.csv',
|
||||
'.json',
|
||||
'.xml',
|
||||
'.yaml',
|
||||
'.yml',
|
||||
'.log',
|
||||
'.rst',
|
||||
'.tsv',
|
||||
])
|
||||
|
||||
const MAX_FILE_SIZE = CONNECTOR_MAX_FILE_BYTES
|
||||
|
||||
interface DropboxFileEntry {
|
||||
'.tag': 'file' | 'folder' | 'deleted'
|
||||
id: string
|
||||
name: string
|
||||
path_lower: string
|
||||
path_display: string
|
||||
client_modified?: string
|
||||
server_modified?: string
|
||||
size?: number
|
||||
content_hash?: string
|
||||
is_downloadable?: boolean
|
||||
}
|
||||
|
||||
interface DropboxListFolderResponse {
|
||||
entries: DropboxFileEntry[]
|
||||
cursor: string
|
||||
has_more: boolean
|
||||
}
|
||||
|
||||
function hasSupportedExtension(name: string): boolean {
|
||||
const lower = name.toLowerCase()
|
||||
const dotIndex = lower.lastIndexOf('.')
|
||||
if (dotIndex === -1) return false
|
||||
return SUPPORTED_EXTENSIONS.has(lower.slice(dotIndex))
|
||||
}
|
||||
|
||||
/** A downloadable file with a supported extension, regardless of size. */
|
||||
function isDownloadableFile(entry: DropboxFileEntry): boolean {
|
||||
return (
|
||||
entry['.tag'] === 'file' && entry.is_downloadable !== false && hasSupportedExtension(entry.name)
|
||||
)
|
||||
}
|
||||
|
||||
async function downloadFileContent(accessToken: string, filePath: string): Promise<string> {
|
||||
const response = await fetchWithRetry('https://content.dropboxapi.com/2/files/download', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Dropbox-API-Arg': JSON.stringify({ path: filePath }),
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to download file ${filePath}: ${response.status}`)
|
||||
}
|
||||
|
||||
// Stream with a hard byte cap so a file whose listing metadata under-reported
|
||||
// (or omitted) its size can never be fully buffered into memory. Oversize raises
|
||||
// so getDocument can surface it as a skipped (failed) row rather than dropping it.
|
||||
const buffer = await readBodyWithLimit(response, MAX_FILE_SIZE)
|
||||
if (!buffer) {
|
||||
throw new ConnectorFileTooLargeError(MAX_FILE_SIZE)
|
||||
}
|
||||
|
||||
const text = buffer.toString('utf8')
|
||||
|
||||
if (filePath.endsWith('.html') || filePath.endsWith('.htm')) {
|
||||
return htmlToPlainText(text)
|
||||
}
|
||||
|
||||
return text
|
||||
}
|
||||
|
||||
function fileToStub(entry: DropboxFileEntry): ExternalDocument {
|
||||
return {
|
||||
externalId: entry.id,
|
||||
title: entry.name,
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: `https://www.dropbox.com/home${entry.path_display}`,
|
||||
contentHash: `dropbox:${entry.id}:${entry.content_hash ?? entry.server_modified ?? ''}`,
|
||||
metadata: {
|
||||
path: entry.path_display,
|
||||
lastModified: entry.server_modified || entry.client_modified,
|
||||
fileSize: entry.size,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export const dropboxConnector: ConnectorConfig = {
|
||||
...dropboxConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
let data: DropboxListFolderResponse
|
||||
|
||||
if (cursor) {
|
||||
const response = await fetchWithRetry(
|
||||
'https://api.dropboxapi.com/2/files/list_folder/continue',
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({ cursor }),
|
||||
}
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error('Failed to continue listing Dropbox folder', {
|
||||
status: response.status,
|
||||
error: errorText,
|
||||
})
|
||||
throw new Error(`Failed to continue listing Dropbox folder: ${response.status}`)
|
||||
}
|
||||
|
||||
data = await response.json()
|
||||
} else {
|
||||
const folderPath = (sourceConfig.folderPath as string)?.trim() || ''
|
||||
const path = folderPath.startsWith('/') ? folderPath : folderPath ? `/${folderPath}` : ''
|
||||
|
||||
logger.info('Listing Dropbox folder', { path: path || '(root)' })
|
||||
|
||||
const response = await fetchWithRetry('https://api.dropboxapi.com/2/files/list_folder', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
path,
|
||||
recursive: true,
|
||||
include_deleted: false,
|
||||
include_non_downloadable_files: false,
|
||||
limit: 2000,
|
||||
}),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error('Failed to list Dropbox folder', {
|
||||
status: response.status,
|
||||
error: errorText,
|
||||
})
|
||||
throw new Error(`Failed to list Dropbox folder: ${response.status}`)
|
||||
}
|
||||
|
||||
data = await response.json()
|
||||
}
|
||||
|
||||
// Keep oversized files and surface them as skipped (failed) documents instead
|
||||
// of dropping them silently at listing time.
|
||||
const candidateFiles = data.entries.filter(isDownloadableFile)
|
||||
|
||||
const maxFiles = sourceConfig.maxFiles ? Number(sourceConfig.maxFiles) : 0
|
||||
const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
|
||||
|
||||
const stubs = candidateFiles.map((entry) =>
|
||||
stubOrSkipBySize(fileToStub(entry), entry.size, MAX_FILE_SIZE)
|
||||
)
|
||||
|
||||
const { documents, indexableCount, capReached } = takeIndexableWithinCap(
|
||||
stubs,
|
||||
isSkippedDocument,
|
||||
maxFiles,
|
||||
previouslyFetched
|
||||
)
|
||||
|
||||
const totalFetched = previouslyFetched + indexableCount
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
const hitLimit = capReached
|
||||
if (hitLimit && syncContext) syncContext.listingCapped = true
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hitLimit ? undefined : data.has_more ? data.cursor : undefined,
|
||||
hasMore: hitLimit ? false : data.has_more,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
_sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
try {
|
||||
const response = await fetchWithRetry('https://api.dropboxapi.com/2/files/get_metadata', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({ path: externalId }),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 409) return null
|
||||
throw new Error(`Failed to get metadata: ${response.status}`)
|
||||
}
|
||||
|
||||
const entry = (await response.json()) as DropboxFileEntry
|
||||
|
||||
if (!isDownloadableFile(entry)) return null
|
||||
|
||||
const stub = fileToStub(entry)
|
||||
if (entry.size && entry.size > MAX_FILE_SIZE) {
|
||||
return markSkipped(stub, sizeLimitSkipReason(MAX_FILE_SIZE))
|
||||
}
|
||||
|
||||
let content: string
|
||||
try {
|
||||
content = await downloadFileContent(accessToken, entry.path_lower)
|
||||
} catch (error) {
|
||||
if (error instanceof ConnectorFileTooLargeError) {
|
||||
return markSkipped(stub, sizeLimitSkipReason(error.limitBytes))
|
||||
}
|
||||
throw error
|
||||
}
|
||||
if (!content.trim()) return null
|
||||
|
||||
return { ...stub, content, contentDeferred: false }
|
||||
} catch (error) {
|
||||
logger.warn(`Failed to fetch document ${externalId}`, {
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const maxFiles = sourceConfig.maxFiles as string | undefined
|
||||
if (maxFiles && (Number.isNaN(Number(maxFiles)) || Number(maxFiles) <= 0)) {
|
||||
return { valid: false, error: 'Max files must be a positive number' }
|
||||
}
|
||||
|
||||
try {
|
||||
const folderPath = (sourceConfig.folderPath as string)?.trim() || ''
|
||||
const path = folderPath.startsWith('/') ? folderPath : folderPath ? `/${folderPath}` : ''
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
'https://api.dropboxapi.com/2/files/list_folder',
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
path,
|
||||
limit: 1,
|
||||
recursive: false,
|
||||
}),
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
if (errorText.includes('not_found')) {
|
||||
return { valid: false, error: 'Folder not found. Check the path and try again.' }
|
||||
}
|
||||
return { valid: false, error: `Failed to access Dropbox: ${response.status}` }
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.path === 'string') {
|
||||
result.path = metadata.path
|
||||
}
|
||||
|
||||
const lastModified = parseTagDate(metadata.lastModified)
|
||||
if (lastModified) result.lastModified = lastModified
|
||||
|
||||
if (metadata.fileSize != null) {
|
||||
const num = Number(metadata.fileSize)
|
||||
if (!Number.isNaN(num)) result.fileSize = num
|
||||
}
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { dropboxConnector } from '@/connectors/dropbox/dropbox'
|
||||
@@ -0,0 +1,40 @@
|
||||
import { DropboxIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const dropboxConnectorMeta: ConnectorMeta = {
|
||||
id: 'dropbox',
|
||||
name: 'Dropbox',
|
||||
description: 'Sync text files from Dropbox',
|
||||
version: '1.0.0',
|
||||
icon: DropboxIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'oauth',
|
||||
provider: 'dropbox',
|
||||
requiredScopes: ['files.metadata.read', 'files.content.read'],
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'folderPath',
|
||||
title: 'Folder Path',
|
||||
type: 'short-input',
|
||||
placeholder: 'e.g. /Documents (default: entire Dropbox)',
|
||||
required: false,
|
||||
description: 'Leave empty to sync all supported files',
|
||||
},
|
||||
{
|
||||
id: 'maxFiles',
|
||||
title: 'Max Files',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 500 (default: unlimited)',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'path', displayName: 'File Path', fieldType: 'text' },
|
||||
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
|
||||
{ id: 'fileSize', displayName: 'File Size (bytes)', fieldType: 'number' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,545 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import {
|
||||
ThriftReader,
|
||||
ThriftWriter,
|
||||
TYPE_I32,
|
||||
TYPE_I64,
|
||||
TYPE_LIST,
|
||||
TYPE_STRING,
|
||||
TYPE_STRUCT,
|
||||
} from '@/app/api/tools/evernote/lib/thrift'
|
||||
import { evernoteConnectorMeta } from '@/connectors/evernote/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { htmlToPlainText, joinTagArray, parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('EvernoteConnector')
|
||||
|
||||
const NOTES_PER_PAGE = 50
|
||||
|
||||
/**
|
||||
* Extracts the shard ID from an Evernote developer token.
|
||||
* Token format: "S=s1:U=12345:..." where s1 is the shard.
|
||||
*/
|
||||
function extractShardId(token: string): string {
|
||||
const match = token.match(/S=s(\d+)/)
|
||||
if (!match) {
|
||||
throw new Error('Invalid Evernote token format: cannot extract shard ID')
|
||||
}
|
||||
return `s${match[1]}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the user ID from an Evernote developer token.
|
||||
* Token format: "S=s1:U=12345:..." where 12345 is the user ID.
|
||||
*/
|
||||
function extractUserId(token: string): string {
|
||||
const match = token.match(/:U=(\d+)/)
|
||||
if (!match) {
|
||||
throw new Error('Invalid Evernote token format: cannot extract user ID')
|
||||
}
|
||||
return match[1]
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the Evernote API host based on the token type.
|
||||
* Sandbox tokens contain `:Sandbox` and route to sandbox.evernote.com.
|
||||
*/
|
||||
function getHost(token: string): string {
|
||||
return token.includes(':Sandbox') ? 'sandbox.evernote.com' : 'www.evernote.com'
|
||||
}
|
||||
|
||||
/**
|
||||
* Derives the NoteStore URL from a developer token.
|
||||
*/
|
||||
function getNoteStoreUrl(token: string): string {
|
||||
const shardId = extractShardId(token)
|
||||
return `https://${getHost(token)}/shard/${shardId}/notestore`
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends a Thrift RPC call to the Evernote NoteStore via HTTP POST.
|
||||
*/
|
||||
async function callNoteStore(
|
||||
token: string,
|
||||
writer: ThriftWriter,
|
||||
retryOptions?: Parameters<typeof fetchWithRetry>[2]
|
||||
): Promise<ThriftReader> {
|
||||
const url = getNoteStoreUrl(token)
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
url,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/x-thrift',
|
||||
Accept: 'application/x-thrift',
|
||||
},
|
||||
body: new Uint8Array(writer.toBuffer()),
|
||||
},
|
||||
retryOptions
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Evernote HTTP ${response.status}: ${response.statusText}`)
|
||||
}
|
||||
|
||||
const reader = new ThriftReader(await response.arrayBuffer())
|
||||
const msg = reader.readMessageBegin()
|
||||
|
||||
if (reader.isException(msg.type)) {
|
||||
const ex = reader.readException()
|
||||
throw new Error(`Evernote API error: ${ex.message}`)
|
||||
}
|
||||
|
||||
return reader
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks for Evernote-specific exceptions in response struct fields 1-3.
|
||||
*/
|
||||
function checkException(r: ThriftReader, fieldId: number, fieldType: number): boolean {
|
||||
if ((fieldId === 1 || fieldId === 2) && fieldType === TYPE_STRUCT) {
|
||||
let errorCode = 0
|
||||
let message = ''
|
||||
r.readStruct((r2, fid, ftype) => {
|
||||
if (fid === 1 && ftype === TYPE_I32) errorCode = r2.readI32()
|
||||
else if (fid === 2 && ftype === TYPE_STRING) message = r2.readString()
|
||||
else r2.skip(ftype)
|
||||
})
|
||||
throw new Error(`Evernote error (${errorCode}): ${message}`)
|
||||
}
|
||||
if (fieldId === 3 && fieldType === TYPE_STRUCT) {
|
||||
let identifier = ''
|
||||
r.readStruct((r2, fid, ftype) => {
|
||||
if (fid === 1 && ftype === TYPE_STRING) identifier = r2.readString()
|
||||
else r2.skip(ftype)
|
||||
})
|
||||
throw new Error(`Evernote not found: ${identifier}`)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
interface Notebook {
|
||||
guid: string
|
||||
name: string
|
||||
}
|
||||
|
||||
interface Tag {
|
||||
guid: string
|
||||
name: string
|
||||
}
|
||||
|
||||
interface NoteMetadata {
|
||||
guid: string
|
||||
title: string
|
||||
created: number
|
||||
updated: number
|
||||
notebookGuid: string
|
||||
tagGuids: string[]
|
||||
}
|
||||
|
||||
interface Note {
|
||||
guid: string
|
||||
title: string
|
||||
content: string
|
||||
created: number
|
||||
updated: number
|
||||
notebookGuid: string
|
||||
tagGuids: string[]
|
||||
}
|
||||
|
||||
async function apiListNotebooks(
|
||||
token: string,
|
||||
retryOptions?: Parameters<typeof fetchWithRetry>[2]
|
||||
): Promise<Notebook[]> {
|
||||
const w = new ThriftWriter()
|
||||
w.writeMessageBegin('listNotebooks', 0)
|
||||
w.writeStringField(1, token)
|
||||
w.writeFieldStop()
|
||||
|
||||
const r = await callNoteStore(token, w, retryOptions)
|
||||
const notebooks: Notebook[] = []
|
||||
|
||||
r.readStruct((r2, fid, ftype) => {
|
||||
if (fid === 0 && ftype === TYPE_LIST) {
|
||||
const { size } = r2.readListBegin()
|
||||
for (let i = 0; i < size; i++) {
|
||||
let guid = ''
|
||||
let name = ''
|
||||
r2.readStruct((r3, fid3, ftype3) => {
|
||||
if (fid3 === 1 && ftype3 === TYPE_STRING) guid = r3.readString()
|
||||
else if (fid3 === 2 && ftype3 === TYPE_STRING) name = r3.readString()
|
||||
else r3.skip(ftype3)
|
||||
})
|
||||
notebooks.push({ guid, name })
|
||||
}
|
||||
} else if (!checkException(r2, fid, ftype)) {
|
||||
r2.skip(ftype)
|
||||
}
|
||||
})
|
||||
|
||||
return notebooks
|
||||
}
|
||||
|
||||
async function apiListTags(
|
||||
token: string,
|
||||
retryOptions?: Parameters<typeof fetchWithRetry>[2]
|
||||
): Promise<Tag[]> {
|
||||
const w = new ThriftWriter()
|
||||
w.writeMessageBegin('listTags', 0)
|
||||
w.writeStringField(1, token)
|
||||
w.writeFieldStop()
|
||||
|
||||
const r = await callNoteStore(token, w, retryOptions)
|
||||
const tags: Tag[] = []
|
||||
|
||||
r.readStruct((r2, fid, ftype) => {
|
||||
if (fid === 0 && ftype === TYPE_LIST) {
|
||||
const { size } = r2.readListBegin()
|
||||
for (let i = 0; i < size; i++) {
|
||||
let guid = ''
|
||||
let name = ''
|
||||
r2.readStruct((r3, fid3, ftype3) => {
|
||||
if (fid3 === 1 && ftype3 === TYPE_STRING) guid = r3.readString()
|
||||
else if (fid3 === 2 && ftype3 === TYPE_STRING) name = r3.readString()
|
||||
else r3.skip(ftype3)
|
||||
})
|
||||
tags.push({ guid, name })
|
||||
}
|
||||
} else if (!checkException(r2, fid, ftype)) {
|
||||
r2.skip(ftype)
|
||||
}
|
||||
})
|
||||
|
||||
return tags
|
||||
}
|
||||
|
||||
/**
|
||||
* Calls NoteStore.findNotesMetadata with offset-based pagination.
|
||||
*
|
||||
* Thrift field numbers (from NoteStore.thrift):
|
||||
* findNotesMetadata(1:token, 2:NoteFilter, 3:offset, 4:maxNotes, 5:ResultSpec)
|
||||
* NoteFilter: 4:notebookGuid
|
||||
* NotesMetadataResultSpec: 2:includeTitle, 6:includeCreated, 7:includeUpdated,
|
||||
* 11:includeNotebookGuid, 12:includeTagGuids
|
||||
* NotesMetadataList: 1:startIndex, 2:totalNotes, 3:list<NoteMetadata>
|
||||
* NoteMetadata: 1:guid, 2:title, 6:created(i64), 7:updated(i64),
|
||||
* 11:notebookGuid, 12:list<tagGuids>
|
||||
*/
|
||||
async function apiFindNotesMetadata(
|
||||
token: string,
|
||||
offset: number,
|
||||
maxNotes: number,
|
||||
notebookGuid?: string,
|
||||
retryOptions?: Parameters<typeof fetchWithRetry>[2]
|
||||
): Promise<{ totalNotes: number; notes: NoteMetadata[] }> {
|
||||
const w = new ThriftWriter()
|
||||
w.writeMessageBegin('findNotesMetadata', 0)
|
||||
w.writeStringField(1, token)
|
||||
|
||||
w.writeFieldBegin(TYPE_STRUCT, 2) // NoteFilter
|
||||
if (notebookGuid) {
|
||||
w.writeStringField(4, notebookGuid)
|
||||
}
|
||||
w.writeFieldStop()
|
||||
|
||||
w.writeI32Field(3, offset)
|
||||
w.writeI32Field(4, maxNotes)
|
||||
|
||||
w.writeFieldBegin(TYPE_STRUCT, 5) // NotesMetadataResultSpec
|
||||
w.writeBoolField(2, true) // includeTitle
|
||||
w.writeBoolField(6, true) // includeCreated
|
||||
w.writeBoolField(7, true) // includeUpdated
|
||||
w.writeBoolField(11, true) // includeNotebookGuid
|
||||
w.writeBoolField(12, true) // includeTagGuids
|
||||
w.writeFieldStop()
|
||||
|
||||
w.writeFieldStop()
|
||||
|
||||
const r = await callNoteStore(token, w, retryOptions)
|
||||
|
||||
let totalNotes = 0
|
||||
const notes: NoteMetadata[] = []
|
||||
|
||||
r.readStruct((r2, fid, ftype) => {
|
||||
if (fid === 0 && ftype === TYPE_STRUCT) {
|
||||
r2.readStruct((r3, fid3, ftype3) => {
|
||||
if (fid3 === 1 && ftype3 === TYPE_I32) {
|
||||
r3.readI32()
|
||||
} else if (fid3 === 2 && ftype3 === TYPE_I32) {
|
||||
totalNotes = r3.readI32()
|
||||
} else if (fid3 === 3 && ftype3 === TYPE_LIST) {
|
||||
const { size } = r3.readListBegin()
|
||||
for (let i = 0; i < size; i++) {
|
||||
let guid = ''
|
||||
let title = ''
|
||||
let created = 0
|
||||
let updated = 0
|
||||
let nbGuid = ''
|
||||
const tagGuids: string[] = []
|
||||
|
||||
r3.readStruct((r4, fid4, ftype4) => {
|
||||
if (fid4 === 1 && ftype4 === TYPE_STRING) guid = r4.readString()
|
||||
else if (fid4 === 2 && ftype4 === TYPE_STRING) title = r4.readString()
|
||||
else if (fid4 === 6 && ftype4 === TYPE_I64) created = Number(r4.readI64())
|
||||
else if (fid4 === 7 && ftype4 === TYPE_I64) updated = Number(r4.readI64())
|
||||
else if (fid4 === 11 && ftype4 === TYPE_STRING) nbGuid = r4.readString()
|
||||
else if (fid4 === 12 && ftype4 === TYPE_LIST) {
|
||||
const { size: tagCount } = r4.readListBegin()
|
||||
for (let t = 0; t < tagCount; t++) tagGuids.push(r4.readString())
|
||||
} else {
|
||||
r4.skip(ftype4)
|
||||
}
|
||||
})
|
||||
notes.push({ guid, title, created, updated, notebookGuid: nbGuid, tagGuids })
|
||||
}
|
||||
} else {
|
||||
r3.skip(ftype3)
|
||||
}
|
||||
})
|
||||
} else if (!checkException(r2, fid, ftype)) {
|
||||
r2.skip(ftype)
|
||||
}
|
||||
})
|
||||
|
||||
return { totalNotes, notes }
|
||||
}
|
||||
|
||||
/**
|
||||
* Calls NoteStore.getNote to fetch a single note with content.
|
||||
*
|
||||
* Thrift: getNote(1:token, 2:guid, 3:withContent, 4:withResourcesData,
|
||||
* 5:withResourcesRecognition, 6:withResourcesAlternateData)
|
||||
* Note: 1:guid, 2:title, 3:content, 6:created, 7:updated, 11:notebookGuid, 12:tagGuids
|
||||
*/
|
||||
async function apiGetNote(
|
||||
token: string,
|
||||
guid: string,
|
||||
retryOptions?: Parameters<typeof fetchWithRetry>[2]
|
||||
): Promise<Note> {
|
||||
const w = new ThriftWriter()
|
||||
w.writeMessageBegin('getNote', 0)
|
||||
w.writeStringField(1, token)
|
||||
w.writeStringField(2, guid)
|
||||
w.writeBoolField(3, true) // withContent
|
||||
w.writeBoolField(4, false) // withResourcesData
|
||||
w.writeBoolField(5, false) // withResourcesRecognition
|
||||
w.writeBoolField(6, false) // withResourcesAlternateData
|
||||
w.writeFieldStop()
|
||||
|
||||
const r = await callNoteStore(token, w, retryOptions)
|
||||
|
||||
let noteGuid = ''
|
||||
let title = ''
|
||||
let content = ''
|
||||
let created = 0
|
||||
let updated = 0
|
||||
let notebookGuid = ''
|
||||
const tagGuids: string[] = []
|
||||
|
||||
r.readStruct((r2, fid, ftype) => {
|
||||
if (fid === 0 && ftype === TYPE_STRUCT) {
|
||||
r2.readStruct((r3, fid3, ftype3) => {
|
||||
if (fid3 === 1 && ftype3 === TYPE_STRING) noteGuid = r3.readString()
|
||||
else if (fid3 === 2 && ftype3 === TYPE_STRING) title = r3.readString()
|
||||
else if (fid3 === 3 && ftype3 === TYPE_STRING) content = r3.readString()
|
||||
else if (fid3 === 6 && ftype3 === TYPE_I64) created = Number(r3.readI64())
|
||||
else if (fid3 === 7 && ftype3 === TYPE_I64) updated = Number(r3.readI64())
|
||||
else if (fid3 === 11 && ftype3 === TYPE_STRING) notebookGuid = r3.readString()
|
||||
else if (fid3 === 12 && ftype3 === TYPE_LIST) {
|
||||
const { size } = r3.readListBegin()
|
||||
for (let t = 0; t < size; t++) tagGuids.push(r3.readString())
|
||||
} else {
|
||||
r3.skip(ftype3)
|
||||
}
|
||||
})
|
||||
} else if (!checkException(r2, fid, ftype)) {
|
||||
r2.skip(ftype)
|
||||
}
|
||||
})
|
||||
|
||||
return { guid: noteGuid || guid, title, content, created, updated, notebookGuid, tagGuids }
|
||||
}
|
||||
|
||||
export const evernoteConnector: ConnectorConfig = {
|
||||
...evernoteConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const notebookGuid = (sourceConfig.notebookGuid as string) || undefined
|
||||
const retryOptions = { maxRetries: 3, initialDelayMs: 500 }
|
||||
|
||||
if (syncContext && !syncContext.tagMap) {
|
||||
const tags = await apiListTags(accessToken, retryOptions)
|
||||
syncContext.tagMap = Object.fromEntries(tags.map((t) => [t.guid, t.name]))
|
||||
}
|
||||
if (syncContext && !syncContext.notebookMap) {
|
||||
const notebooks = await apiListNotebooks(accessToken, retryOptions)
|
||||
syncContext.notebookMap = Object.fromEntries(notebooks.map((nb) => [nb.guid, nb.name]))
|
||||
}
|
||||
|
||||
const tagMap = (syncContext?.tagMap as Record<string, string>) || {}
|
||||
const notebookMap = (syncContext?.notebookMap as Record<string, string>) || {}
|
||||
const offset = cursor ? Number(cursor) : 0
|
||||
const shardId = extractShardId(accessToken)
|
||||
const userId = extractUserId(accessToken)
|
||||
const host = getHost(accessToken)
|
||||
|
||||
logger.info('Listing Evernote notes', { offset, maxNotes: NOTES_PER_PAGE })
|
||||
|
||||
const result = await apiFindNotesMetadata(
|
||||
accessToken,
|
||||
offset,
|
||||
NOTES_PER_PAGE,
|
||||
notebookGuid,
|
||||
retryOptions
|
||||
)
|
||||
|
||||
const documents: ExternalDocument[] = result.notes.map((meta) => {
|
||||
const tagNames = meta.tagGuids.map((g) => tagMap[g]).filter(Boolean)
|
||||
|
||||
return {
|
||||
externalId: meta.guid,
|
||||
title: meta.title || 'Untitled',
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: `https://${host}/shard/${shardId}/nl/${userId}/${meta.guid}/`,
|
||||
contentHash: `evernote:${meta.guid}:${meta.updated}`,
|
||||
metadata: {
|
||||
tags: tagNames,
|
||||
notebook: notebookMap[meta.notebookGuid] || '',
|
||||
createdAt: meta.created ? new Date(meta.created).toISOString() : undefined,
|
||||
updatedAt: meta.updated ? new Date(meta.updated).toISOString() : undefined,
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
const nextOffset = offset + result.notes.length
|
||||
const hasMore = nextOffset < result.totalNotes
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore ? String(nextOffset) : undefined,
|
||||
hasMore,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
_sourceConfig: Record<string, unknown>,
|
||||
externalId: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocument | null> => {
|
||||
try {
|
||||
const retryOptions = { maxRetries: 3, initialDelayMs: 500 }
|
||||
const note = await apiGetNote(accessToken, externalId, retryOptions)
|
||||
const plainText = htmlToPlainText(note.content)
|
||||
const title = note.title || 'Untitled'
|
||||
const content = plainText.trim() ? plainText : title
|
||||
|
||||
const shardId = extractShardId(accessToken)
|
||||
const userId = extractUserId(accessToken)
|
||||
const host = getHost(accessToken)
|
||||
|
||||
if (syncContext && !syncContext.tagMap) {
|
||||
const tags = await apiListTags(accessToken, retryOptions)
|
||||
syncContext.tagMap = Object.fromEntries(tags.map((t) => [t.guid, t.name]))
|
||||
}
|
||||
if (syncContext && !syncContext.notebookMap) {
|
||||
const notebooks = await apiListNotebooks(accessToken, retryOptions)
|
||||
syncContext.notebookMap = Object.fromEntries(notebooks.map((nb) => [nb.guid, nb.name]))
|
||||
}
|
||||
|
||||
let tagMap: Record<string, string>
|
||||
let notebookMap: Record<string, string>
|
||||
if (syncContext) {
|
||||
tagMap = syncContext.tagMap as Record<string, string>
|
||||
notebookMap = syncContext.notebookMap as Record<string, string>
|
||||
} else {
|
||||
const tags = await apiListTags(accessToken, retryOptions)
|
||||
tagMap = Object.fromEntries(tags.map((t) => [t.guid, t.name]))
|
||||
const notebooks = await apiListNotebooks(accessToken, retryOptions)
|
||||
notebookMap = Object.fromEntries(notebooks.map((nb) => [nb.guid, nb.name]))
|
||||
}
|
||||
|
||||
const tagNames = note.tagGuids.map((g) => tagMap[g]).filter(Boolean)
|
||||
const notebookName = notebookMap[note.notebookGuid] || ''
|
||||
|
||||
return {
|
||||
externalId,
|
||||
title,
|
||||
content,
|
||||
contentDeferred: false,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: `https://${host}/shard/${shardId}/nl/${userId}/${externalId}/`,
|
||||
contentHash: `evernote:${note.guid}:${note.updated}`,
|
||||
metadata: {
|
||||
tags: tagNames,
|
||||
notebook: notebookName,
|
||||
createdAt: note.created ? new Date(note.created).toISOString() : undefined,
|
||||
updatedAt: note.updated ? new Date(note.updated).toISOString() : undefined,
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Failed to get Evernote note', {
|
||||
externalId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
try {
|
||||
extractShardId(accessToken)
|
||||
} catch {
|
||||
return { valid: false, error: 'Invalid developer token format — must start with S=s{number}' }
|
||||
}
|
||||
|
||||
try {
|
||||
const notebooks = await apiListNotebooks(accessToken, VALIDATE_RETRY_OPTIONS)
|
||||
|
||||
const notebookGuid = (sourceConfig.notebookGuid as string) || ''
|
||||
if (notebookGuid.trim()) {
|
||||
const found = notebooks.some((nb) => nb.guid === notebookGuid.trim())
|
||||
if (!found) {
|
||||
return { valid: false, error: `Notebook with GUID "${notebookGuid}" not found` }
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = toError(error).message || 'Failed to connect to Evernote'
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
const tags = joinTagArray(metadata.tags)
|
||||
if (tags) result.tags = tags
|
||||
|
||||
if (typeof metadata.notebook === 'string' && metadata.notebook) {
|
||||
result.notebook = metadata.notebook
|
||||
}
|
||||
|
||||
const updatedAt = parseTagDate(metadata.updatedAt)
|
||||
if (updatedAt) result.updatedAt = updatedAt
|
||||
|
||||
const createdAt = parseTagDate(metadata.createdAt)
|
||||
if (createdAt) result.createdAt = createdAt
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { evernoteConnector } from '@/connectors/evernote/evernote'
|
||||
@@ -0,0 +1,34 @@
|
||||
import { EvernoteIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const evernoteConnectorMeta: ConnectorMeta = {
|
||||
id: 'evernote',
|
||||
name: 'Evernote',
|
||||
description: 'Sync notes from Evernote',
|
||||
version: '1.0.0',
|
||||
icon: EvernoteIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'apiKey',
|
||||
label: 'Developer Token',
|
||||
placeholder: 'Enter your Evernote developer token (starts with S=)',
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'notebookGuid',
|
||||
title: 'Notebook GUID',
|
||||
type: 'short-input',
|
||||
placeholder: 'Leave empty to sync all notebooks',
|
||||
required: false,
|
||||
description: 'Sync only notes from this notebook (optional)',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'tags', displayName: 'Tags', fieldType: 'text' },
|
||||
{ id: 'notebook', displayName: 'Notebook', fieldType: 'text' },
|
||||
{ id: 'updatedAt', displayName: 'Last Updated', fieldType: 'date' },
|
||||
{ id: 'createdAt', displayName: 'Created', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,542 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { fathomConnectorMeta } from '@/connectors/fathom/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('FathomConnector')
|
||||
|
||||
const FATHOM_API_BASE = 'https://api.fathom.ai/external/v1'
|
||||
|
||||
const MS_PER_DAY = 24 * 60 * 60 * 1000
|
||||
|
||||
/**
|
||||
* Days subtracted from `lastSyncAt` when computing the incremental `created_after`
|
||||
* window. Fathom's list endpoint only filters by creation time (no update-based
|
||||
* filter), so a meeting whose transcript was not yet ready on the sync that first
|
||||
* saw it would otherwise never be re-listed. The overlap keeps recently-created
|
||||
* meetings in the window long enough for late transcripts to be retried — the sync
|
||||
* engine re-attempts meetings whose `getDocument` previously returned null, since
|
||||
* those are never persisted. Matches the Gong connector's overlap approach.
|
||||
*/
|
||||
const INCREMENTAL_OVERLAP_DAYS = 14
|
||||
|
||||
/**
|
||||
* Fathom authenticates external API requests with the `X-Api-Key` header.
|
||||
* (The API also accepts `Authorization: Bearer` for OAuth-connected apps, but
|
||||
* the api-key flow this connector uses requires `X-Api-Key`.)
|
||||
*/
|
||||
function buildHeaders(apiKey: string): Record<string, string> {
|
||||
return {
|
||||
'X-Api-Key': apiKey,
|
||||
'Content-Type': 'application/json',
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A meeting object as returned by `GET /meetings`. Only the fields this
|
||||
* connector reads are typed; the API returns additional fields.
|
||||
*/
|
||||
interface FathomMeeting {
|
||||
recording_id?: number
|
||||
title?: string
|
||||
meeting_title?: string | null
|
||||
url?: string
|
||||
share_url?: string
|
||||
created_at?: string
|
||||
scheduled_start_time?: string | null
|
||||
scheduled_end_time?: string | null
|
||||
recording_start_time?: string | null
|
||||
recording_end_time?: string | null
|
||||
transcript_language?: string
|
||||
calendar_invitees_domains_type?: 'only_internal' | 'one_or_more_external' | null
|
||||
recorded_by?: {
|
||||
name?: string
|
||||
email?: string
|
||||
email_domain?: string
|
||||
team?: string | null
|
||||
} | null
|
||||
}
|
||||
|
||||
interface FathomMeetingsListResponse {
|
||||
items?: FathomMeeting[]
|
||||
next_cursor?: string | null
|
||||
}
|
||||
|
||||
/**
|
||||
* A single transcript entry as returned by `GET /recordings/{id}/transcript`.
|
||||
*/
|
||||
interface FathomTranscriptEntry {
|
||||
speaker?: {
|
||||
display_name?: string
|
||||
matched_calendar_invitee_email?: string | null
|
||||
}
|
||||
text?: string
|
||||
timestamp?: string
|
||||
}
|
||||
|
||||
interface FathomTranscriptResponse {
|
||||
transcript?: FathomTranscriptEntry[]
|
||||
}
|
||||
|
||||
interface FathomSummary {
|
||||
template_name?: string | null
|
||||
markdown_formatted?: string | null
|
||||
}
|
||||
|
||||
interface FathomSummaryResponse {
|
||||
summary?: FathomSummary | null
|
||||
}
|
||||
|
||||
/**
|
||||
* Header fields cached per recording during `listDocuments` so `getDocument`
|
||||
* can render an identical document header. Fathom exposes no single-meeting
|
||||
* GET and no `recording_ids` filter, so this metadata cannot be refetched once
|
||||
* listing has moved past the page that contained it — it is carried forward in
|
||||
* the shared `syncContext` instead.
|
||||
*/
|
||||
interface FathomMeetingHeader {
|
||||
title: string
|
||||
meetingDate?: string
|
||||
durationSeconds?: number
|
||||
recordedByEmail?: string
|
||||
recordedByName?: string
|
||||
team?: string
|
||||
sourceUrl?: string
|
||||
contentHash: string
|
||||
metadata: FathomMeetingMetadata
|
||||
}
|
||||
|
||||
/**
|
||||
* Metadata describing a Fathom meeting, attached to the listing stub. The sync
|
||||
* engine merges this onto the hydrated document, so `getDocument` never needs
|
||||
* to reproduce it.
|
||||
*/
|
||||
interface FathomMeetingMetadata {
|
||||
recordingId?: string
|
||||
recordedByEmail?: string
|
||||
recordedByName?: string
|
||||
team?: string
|
||||
meetingType?: string
|
||||
meetingDate?: string
|
||||
durationSeconds?: number
|
||||
transcriptLanguage?: string
|
||||
title?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps Fathom's `calendar_invitees_domains_type` to a human-readable meeting
|
||||
* type. This is the only meeting-type signal the API exposes: `only_internal`
|
||||
* means every invitee shares the recorder's domain; `one_or_more_external`
|
||||
* means at least one external attendee (customer-facing).
|
||||
*/
|
||||
function resolveMeetingType(meeting: FathomMeeting): string | undefined {
|
||||
switch (meeting.calendar_invitees_domains_type) {
|
||||
case 'only_internal':
|
||||
return 'internal'
|
||||
case 'one_or_more_external':
|
||||
return 'external'
|
||||
default:
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the meeting duration in whole seconds from the recording window,
|
||||
* or undefined when either bound is missing or unparseable.
|
||||
*/
|
||||
function computeDurationSeconds(meeting: FathomMeeting): number | undefined {
|
||||
const start = meeting.recording_start_time ?? meeting.scheduled_start_time ?? undefined
|
||||
const end = meeting.recording_end_time ?? meeting.scheduled_end_time ?? undefined
|
||||
if (!start || !end) return undefined
|
||||
const startMs = new Date(start).getTime()
|
||||
const endMs = new Date(end).getTime()
|
||||
if (Number.isNaN(startMs) || Number.isNaN(endMs) || endMs < startMs) return undefined
|
||||
return Math.round((endMs - startMs) / 1000)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the best title for a meeting, falling back through the title fields.
|
||||
*/
|
||||
function resolveTitle(meeting: FathomMeeting): string {
|
||||
const title = meeting.title?.trim() || meeting.meeting_title?.trim()
|
||||
return title || 'Untitled Fathom Meeting'
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the connector metadata bag attached to a listing stub.
|
||||
*/
|
||||
function buildMetadata(meeting: FathomMeeting): FathomMeetingMetadata {
|
||||
return {
|
||||
recordingId: meeting.recording_id != null ? String(meeting.recording_id) : undefined,
|
||||
recordedByEmail: meeting.recorded_by?.email,
|
||||
recordedByName: meeting.recorded_by?.name,
|
||||
team: meeting.recorded_by?.team ?? undefined,
|
||||
meetingType: resolveMeetingType(meeting),
|
||||
meetingDate: meeting.recording_start_time ?? meeting.created_at ?? undefined,
|
||||
durationSeconds: computeDurationSeconds(meeting),
|
||||
transcriptLanguage: meeting.transcript_language,
|
||||
title: resolveTitle(meeting),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the lightweight header fields cached for `getDocument`.
|
||||
*/
|
||||
function buildHeader(meeting: FathomMeeting): FathomMeetingHeader {
|
||||
return {
|
||||
title: resolveTitle(meeting),
|
||||
meetingDate: meeting.recording_start_time ?? meeting.created_at ?? undefined,
|
||||
durationSeconds: computeDurationSeconds(meeting),
|
||||
recordedByEmail: meeting.recorded_by?.email,
|
||||
recordedByName: meeting.recorded_by?.name,
|
||||
team: meeting.recorded_by?.team ?? undefined,
|
||||
sourceUrl: buildSourceUrl(meeting),
|
||||
contentHash: buildContentHash(meeting),
|
||||
metadata: buildMetadata(meeting),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a metadata-based content hash. Fathom recordings are immutable once
|
||||
* processed, so the recording id plus its end/creation timestamps fully identify
|
||||
* a version. The same value is cached in the header and returned by `getDocument`,
|
||||
* so the stub and hydrated document hash identically.
|
||||
*/
|
||||
function buildContentHash(meeting: FathomMeeting): string {
|
||||
return `fathom:${meeting.recording_id ?? ''}:${meeting.recording_end_time ?? ''}:${meeting.created_at ?? ''}`
|
||||
}
|
||||
|
||||
function buildSourceUrl(meeting: FathomMeeting): string | undefined {
|
||||
return meeting.share_url || meeting.url || undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the cached header for a recording out of the shared sync context.
|
||||
*/
|
||||
function readCachedHeader(
|
||||
syncContext: Record<string, unknown> | undefined,
|
||||
recordingId: string
|
||||
): FathomMeetingHeader | undefined {
|
||||
const cache = syncContext?.meetingHeaders as Record<string, FathomMeetingHeader> | undefined
|
||||
return cache?.[recordingId]
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores the header for a recording in the shared sync context.
|
||||
*/
|
||||
function cacheHeader(
|
||||
syncContext: Record<string, unknown> | undefined,
|
||||
recordingId: string,
|
||||
header: FathomMeetingHeader
|
||||
): void {
|
||||
if (!syncContext) return
|
||||
const cache =
|
||||
(syncContext.meetingHeaders as Record<string, FathomMeetingHeader> | undefined) ?? {}
|
||||
cache[recordingId] = header
|
||||
syncContext.meetingHeaders = cache
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats the meeting header, optional summary, and transcript into a single
|
||||
* plain-text document with one `Speaker: text` line per transcript entry.
|
||||
*/
|
||||
function formatMeetingContent(
|
||||
header: FathomMeetingHeader | undefined,
|
||||
transcript: FathomTranscriptEntry[],
|
||||
summary: FathomSummary | null
|
||||
): string {
|
||||
const parts: string[] = []
|
||||
|
||||
parts.push(`Meeting: ${header?.title ?? 'Untitled Fathom Meeting'}`)
|
||||
|
||||
if (header?.meetingDate) parts.push(`Date: ${header.meetingDate}`)
|
||||
|
||||
if (header?.durationSeconds != null) {
|
||||
parts.push(`Duration: ${Math.round(header.durationSeconds / 60)} minutes`)
|
||||
}
|
||||
|
||||
if (header?.recordedByEmail) {
|
||||
parts.push(`Recorded by: ${header.recordedByName ?? header.recordedByEmail}`)
|
||||
}
|
||||
|
||||
if (header?.team) parts.push(`Team: ${header.team}`)
|
||||
|
||||
if (summary?.markdown_formatted?.trim()) {
|
||||
parts.push('')
|
||||
parts.push('--- Summary ---')
|
||||
parts.push(summary.markdown_formatted.trim())
|
||||
}
|
||||
|
||||
if (transcript.length > 0) {
|
||||
parts.push('')
|
||||
parts.push('--- Transcript ---')
|
||||
for (const entry of transcript) {
|
||||
const speaker = entry.speaker?.display_name?.trim() || 'Unknown'
|
||||
const text = entry.text?.trim()
|
||||
if (text) parts.push(`${speaker}: ${text}`)
|
||||
}
|
||||
}
|
||||
|
||||
return parts.join('\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a listing meeting into a deferred stub. Content is fetched lazily
|
||||
* via `getDocument` only for new or changed meetings.
|
||||
*/
|
||||
function meetingToStub(meeting: FathomMeeting): ExternalDocument {
|
||||
const metadata = buildMetadata(meeting)
|
||||
return {
|
||||
externalId: String(meeting.recording_id),
|
||||
title: resolveTitle(meeting),
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: buildSourceUrl(meeting),
|
||||
contentHash: buildContentHash(meeting),
|
||||
metadata: { ...metadata },
|
||||
}
|
||||
}
|
||||
|
||||
export const fathomConnector: ConnectorConfig = {
|
||||
...fathomConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>,
|
||||
lastSyncAt?: Date
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const recordedBy = (sourceConfig.recordedBy as string | undefined)?.trim()
|
||||
const teams = (sourceConfig.teams as string | undefined)?.trim()
|
||||
const meetingType = (sourceConfig.meetingType as string | undefined)?.trim()
|
||||
const inviteeDomain = (sourceConfig.inviteeDomains as string | undefined)?.trim()
|
||||
const maxMeetings = sourceConfig.maxMeetings ? Number(sourceConfig.maxMeetings) : 0
|
||||
|
||||
const url = new URL(`${FATHOM_API_BASE}/meetings`)
|
||||
if (recordedBy) url.searchParams.append('recorded_by[]', recordedBy)
|
||||
if (teams) url.searchParams.append('teams[]', teams)
|
||||
if (meetingType && meetingType !== 'all') {
|
||||
url.searchParams.append('calendar_invitees_domains_type', meetingType)
|
||||
}
|
||||
if (inviteeDomain) url.searchParams.append('calendar_invitees_domains[]', inviteeDomain)
|
||||
if (cursor) url.searchParams.append('cursor', cursor)
|
||||
if (lastSyncAt) {
|
||||
const createdAfter = new Date(lastSyncAt.getTime() - INCREMENTAL_OVERLAP_DAYS * MS_PER_DAY)
|
||||
url.searchParams.append('created_after', createdAfter.toISOString())
|
||||
}
|
||||
|
||||
logger.info('Listing Fathom meetings', {
|
||||
hasCursor: Boolean(cursor),
|
||||
recordedBy,
|
||||
teams,
|
||||
meetingType,
|
||||
inviteeDomain,
|
||||
incremental: Boolean(lastSyncAt),
|
||||
})
|
||||
|
||||
const response = await fetchWithRetry(url.toString(), {
|
||||
method: 'GET',
|
||||
headers: buildHeaders(accessToken),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
logger.error('Failed to list Fathom meetings', {
|
||||
status: response.status,
|
||||
error: errorText.slice(0, 500),
|
||||
})
|
||||
throw new Error(`Failed to list Fathom meetings: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as FathomMeetingsListResponse
|
||||
const meetings = data.items ?? []
|
||||
const nextCursor = data.next_cursor?.trim() || undefined
|
||||
|
||||
const allDocuments: ExternalDocument[] = []
|
||||
for (const meeting of meetings) {
|
||||
if (meeting.recording_id == null) continue
|
||||
const externalId = String(meeting.recording_id)
|
||||
cacheHeader(syncContext, externalId, buildHeader(meeting))
|
||||
allDocuments.push(meetingToStub(meeting))
|
||||
}
|
||||
|
||||
const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
|
||||
let documents = allDocuments
|
||||
if (maxMeetings > 0) {
|
||||
const remaining = Math.max(0, maxMeetings - prevFetched)
|
||||
if (allDocuments.length > remaining) {
|
||||
documents = allDocuments.slice(0, remaining)
|
||||
}
|
||||
}
|
||||
|
||||
const totalFetched = prevFetched + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
const hitLimit = maxMeetings > 0 && totalFetched >= maxMeetings
|
||||
if (hitLimit && syncContext) syncContext.listingCapped = true
|
||||
|
||||
const hasMore = !hitLimit && Boolean(nextCursor)
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore ? nextCursor : undefined,
|
||||
hasMore,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
_sourceConfig: Record<string, unknown>,
|
||||
externalId: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocument | null> => {
|
||||
try {
|
||||
if (!externalId) return null
|
||||
|
||||
const transcriptUrl = `${FATHOM_API_BASE}/recordings/${encodeURIComponent(externalId)}/transcript`
|
||||
const transcriptResponse = await fetchWithRetry(transcriptUrl, {
|
||||
method: 'GET',
|
||||
headers: buildHeaders(accessToken),
|
||||
})
|
||||
|
||||
if (!transcriptResponse.ok) {
|
||||
if (transcriptResponse.status === 404) return null
|
||||
throw new Error(`Failed to fetch Fathom transcript: ${transcriptResponse.status}`)
|
||||
}
|
||||
|
||||
const transcriptData = (await transcriptResponse.json()) as FathomTranscriptResponse
|
||||
const transcript = transcriptData.transcript ?? []
|
||||
|
||||
let summary: FathomSummary | null = null
|
||||
try {
|
||||
const summaryUrl = `${FATHOM_API_BASE}/recordings/${encodeURIComponent(externalId)}/summary`
|
||||
const summaryResponse = await fetchWithRetry(summaryUrl, {
|
||||
method: 'GET',
|
||||
headers: buildHeaders(accessToken),
|
||||
})
|
||||
if (summaryResponse.ok) {
|
||||
const summaryData = (await summaryResponse.json()) as FathomSummaryResponse
|
||||
summary = summaryData.summary ?? null
|
||||
}
|
||||
} catch (summaryError) {
|
||||
logger.warn('Failed to fetch Fathom summary', {
|
||||
externalId,
|
||||
error: toError(summaryError).message,
|
||||
})
|
||||
}
|
||||
|
||||
const hasTranscript = transcript.some((entry) => entry.text?.trim())
|
||||
const hasSummary = Boolean(summary?.markdown_formatted?.trim())
|
||||
if (!hasTranscript && !hasSummary) {
|
||||
logger.info('No transcript or summary yet for Fathom meeting', { externalId })
|
||||
return null
|
||||
}
|
||||
|
||||
const header = readCachedHeader(syncContext, externalId)
|
||||
if (!header) {
|
||||
logger.warn(
|
||||
'No cached header for Fathom meeting; skipping to avoid an un-refreshable record',
|
||||
{
|
||||
externalId,
|
||||
}
|
||||
)
|
||||
return null
|
||||
}
|
||||
|
||||
const content = formatMeetingContent(header, transcript, summary).trim()
|
||||
if (!content) return null
|
||||
|
||||
return {
|
||||
externalId,
|
||||
title: header.title,
|
||||
content,
|
||||
contentDeferred: false,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: header.sourceUrl,
|
||||
contentHash: header.contentHash,
|
||||
metadata: { ...header.metadata },
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Failed to get Fathom meeting', {
|
||||
externalId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const maxMeetings = sourceConfig.maxMeetings as string | undefined
|
||||
if (maxMeetings && (Number.isNaN(Number(maxMeetings)) || Number(maxMeetings) < 0)) {
|
||||
return { valid: false, error: 'Max meetings must be a non-negative number' }
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetchWithRetry(
|
||||
`${FATHOM_API_BASE}/meetings`,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: buildHeaders(accessToken),
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
return {
|
||||
valid: false,
|
||||
error: `Fathom access failed: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}`,
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.title === 'string' && metadata.title.trim()) {
|
||||
result.title = metadata.title
|
||||
}
|
||||
|
||||
if (typeof metadata.recordedByEmail === 'string' && metadata.recordedByEmail.trim()) {
|
||||
result.recordedByEmail = metadata.recordedByEmail
|
||||
}
|
||||
|
||||
if (typeof metadata.recordedByName === 'string' && metadata.recordedByName.trim()) {
|
||||
result.recordedByName = metadata.recordedByName
|
||||
}
|
||||
|
||||
if (typeof metadata.team === 'string' && metadata.team.trim()) {
|
||||
result.team = metadata.team
|
||||
}
|
||||
|
||||
if (typeof metadata.meetingType === 'string' && metadata.meetingType.trim()) {
|
||||
result.meetingType = metadata.meetingType
|
||||
}
|
||||
|
||||
if (typeof metadata.transcriptLanguage === 'string' && metadata.transcriptLanguage.trim()) {
|
||||
result.transcriptLanguage = metadata.transcriptLanguage
|
||||
}
|
||||
|
||||
if (metadata.durationSeconds != null) {
|
||||
const num = Number(metadata.durationSeconds)
|
||||
if (!Number.isNaN(num)) result.durationSeconds = num
|
||||
}
|
||||
|
||||
const meetingDate = parseTagDate(metadata.meetingDate)
|
||||
if (meetingDate) result.meetingDate = meetingDate
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { fathomConnector } from '@/connectors/fathom/fathom'
|
||||
@@ -0,0 +1,79 @@
|
||||
import { FathomIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const fathomConnectorMeta: ConnectorMeta = {
|
||||
id: 'fathom',
|
||||
name: 'Fathom',
|
||||
description: 'Sync meeting transcripts and summaries from Fathom',
|
||||
version: '1.0.0',
|
||||
icon: FathomIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'apiKey',
|
||||
label: 'API Key',
|
||||
placeholder: 'Enter your Fathom API key',
|
||||
},
|
||||
|
||||
supportsIncrementalSync: true,
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'recordedBy',
|
||||
title: 'Filter by Recorder Email',
|
||||
type: 'short-input',
|
||||
placeholder: 'e.g. john@example.com',
|
||||
required: false,
|
||||
description: 'Only sync meetings recorded by this email',
|
||||
},
|
||||
{
|
||||
id: 'teams',
|
||||
title: 'Filter by Team',
|
||||
type: 'short-input',
|
||||
placeholder: 'e.g. Sales',
|
||||
required: false,
|
||||
description: 'Only sync meetings belonging to this team',
|
||||
},
|
||||
{
|
||||
id: 'meetingType',
|
||||
title: 'Filter by Meeting Type',
|
||||
type: 'dropdown',
|
||||
mode: 'advanced',
|
||||
required: false,
|
||||
description:
|
||||
'Only sync internal meetings (everyone shares the recorder’s domain) or external meetings (at least one outside attendee). Leave as All to sync both.',
|
||||
options: [
|
||||
{ id: 'all', label: 'All meetings' },
|
||||
{ id: 'one_or_more_external', label: 'External (customer-facing) only' },
|
||||
{ id: 'only_internal', label: 'Internal only' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'inviteeDomains',
|
||||
title: 'Filter by Attendee Domain',
|
||||
type: 'short-input',
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. acme.com',
|
||||
required: false,
|
||||
description:
|
||||
'Only sync meetings that include a calendar invitee from this company email domain (exact match).',
|
||||
},
|
||||
{
|
||||
id: 'maxMeetings',
|
||||
title: 'Max Meetings',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 200 (default: unlimited)',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'title', displayName: 'Title', fieldType: 'text' },
|
||||
{ id: 'recordedByEmail', displayName: 'Recorded By (Email)', fieldType: 'text' },
|
||||
{ id: 'recordedByName', displayName: 'Recorded By (Name)', fieldType: 'text' },
|
||||
{ id: 'team', displayName: 'Team', fieldType: 'text' },
|
||||
{ id: 'meetingType', displayName: 'Meeting Type', fieldType: 'text' },
|
||||
{ id: 'transcriptLanguage', displayName: 'Language', fieldType: 'text' },
|
||||
{ id: 'durationSeconds', displayName: 'Duration (seconds)', fieldType: 'number' },
|
||||
{ id: 'meetingDate', displayName: 'Meeting Date', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { firefliesConnectorMeta } from '@/connectors/fireflies/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('FirefliesConnector')
|
||||
|
||||
const FIREFLIES_GRAPHQL_URL = 'https://api.fireflies.ai/graphql'
|
||||
const TRANSCRIPTS_PER_PAGE = 50
|
||||
|
||||
interface FirefliesTranscript {
|
||||
id: string
|
||||
title: string
|
||||
date: number
|
||||
duration: number
|
||||
host_email?: string
|
||||
organizer_email?: string
|
||||
participants?: string[]
|
||||
transcript_url?: string
|
||||
speakers?: { id: number; name: string }[]
|
||||
sentences?: { index: number; speaker_name: string; text: string }[]
|
||||
summary?: {
|
||||
keywords?: string[]
|
||||
action_items?: string
|
||||
overview?: string
|
||||
short_summary?: string
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes a GraphQL query against the Fireflies API.
|
||||
*/
|
||||
async function firefliesGraphQL(
|
||||
accessToken: string,
|
||||
query: string,
|
||||
variables: Record<string, unknown> = {},
|
||||
retryOptions?: Parameters<typeof fetchWithRetry>[2]
|
||||
): Promise<Record<string, unknown>> {
|
||||
const response = await fetchWithRetry(
|
||||
FIREFLIES_GRAPHQL_URL,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
body: JSON.stringify({ query, variables }),
|
||||
},
|
||||
retryOptions
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Fireflies API HTTP error: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
|
||||
if (data.errors) {
|
||||
const message = (data.errors as { message: string }[])[0]?.message || 'Unknown GraphQL error'
|
||||
throw new Error(`Fireflies API error: ${message}`)
|
||||
}
|
||||
|
||||
return data.data as Record<string, unknown>
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats transcript sentences into plain text content.
|
||||
*/
|
||||
function formatTranscriptContent(transcript: FirefliesTranscript): string {
|
||||
const parts: string[] = []
|
||||
|
||||
if (transcript.title) {
|
||||
parts.push(`Meeting: ${transcript.title}`)
|
||||
}
|
||||
|
||||
if (transcript.date) {
|
||||
parts.push(`Date: ${new Date(transcript.date).toISOString()}`)
|
||||
}
|
||||
|
||||
if (transcript.duration) {
|
||||
const minutes = Math.round(transcript.duration / 60)
|
||||
parts.push(`Duration: ${minutes} minutes`)
|
||||
}
|
||||
|
||||
if (transcript.host_email) {
|
||||
parts.push(`Host: ${transcript.host_email}`)
|
||||
}
|
||||
|
||||
if (transcript.participants && transcript.participants.length > 0) {
|
||||
parts.push(`Participants: ${transcript.participants.join(', ')}`)
|
||||
}
|
||||
|
||||
if (transcript.summary?.overview) {
|
||||
parts.push('')
|
||||
parts.push('--- Overview ---')
|
||||
parts.push(transcript.summary.overview)
|
||||
}
|
||||
|
||||
if (transcript.summary?.action_items) {
|
||||
parts.push('')
|
||||
parts.push('--- Action Items ---')
|
||||
parts.push(transcript.summary.action_items)
|
||||
}
|
||||
|
||||
if (transcript.summary?.keywords && transcript.summary.keywords.length > 0) {
|
||||
parts.push('')
|
||||
parts.push(`Keywords: ${transcript.summary.keywords.join(', ')}`)
|
||||
}
|
||||
|
||||
if (transcript.sentences && transcript.sentences.length > 0) {
|
||||
parts.push('')
|
||||
parts.push('--- Transcript ---')
|
||||
for (const sentence of transcript.sentences) {
|
||||
parts.push(`${sentence.speaker_name}: ${sentence.text}`)
|
||||
}
|
||||
}
|
||||
|
||||
return parts.join('\n')
|
||||
}
|
||||
|
||||
export const firefliesConnector: ConnectorConfig = {
|
||||
...firefliesConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const hostEmail = (sourceConfig.hostEmail as string) || ''
|
||||
const maxTranscripts = sourceConfig.maxTranscripts ? Number(sourceConfig.maxTranscripts) : 0
|
||||
|
||||
const skip = cursor ? Number(cursor) : 0
|
||||
|
||||
const variables: Record<string, unknown> = {
|
||||
limit: TRANSCRIPTS_PER_PAGE,
|
||||
skip,
|
||||
}
|
||||
|
||||
if (hostEmail.trim()) {
|
||||
variables.host_email = hostEmail.trim()
|
||||
}
|
||||
|
||||
logger.info('Listing Fireflies transcripts', { skip, limit: TRANSCRIPTS_PER_PAGE, hostEmail })
|
||||
|
||||
const data = await firefliesGraphQL(
|
||||
accessToken,
|
||||
`query Transcripts(
|
||||
$limit: Int
|
||||
$skip: Int
|
||||
$host_email: String
|
||||
) {
|
||||
transcripts(
|
||||
limit: $limit
|
||||
skip: $skip
|
||||
host_email: $host_email
|
||||
) {
|
||||
id
|
||||
title
|
||||
date
|
||||
duration
|
||||
host_email
|
||||
organizer_email
|
||||
participants
|
||||
transcript_url
|
||||
speakers {
|
||||
id
|
||||
name
|
||||
}
|
||||
}
|
||||
}`,
|
||||
variables
|
||||
)
|
||||
|
||||
const transcripts = (data.transcripts || []) as FirefliesTranscript[]
|
||||
|
||||
const documents: ExternalDocument[] = transcripts.map((transcript) => {
|
||||
const meetingDate = transcript.date ? new Date(transcript.date).toISOString() : undefined
|
||||
const speakerNames = transcript.speakers?.map((s) => s.name).filter(Boolean) ?? []
|
||||
|
||||
return {
|
||||
externalId: transcript.id,
|
||||
title: transcript.title || 'Untitled Meeting',
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain' as const,
|
||||
sourceUrl: transcript.transcript_url || undefined,
|
||||
contentHash: `fireflies:${transcript.id}:${transcript.date ?? ''}:${transcript.duration ?? ''}`,
|
||||
metadata: {
|
||||
hostEmail: transcript.host_email,
|
||||
duration: transcript.duration,
|
||||
meetingDate,
|
||||
participants: transcript.participants,
|
||||
speakers: speakerNames,
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
const hitLimit = maxTranscripts > 0 && totalFetched >= maxTranscripts
|
||||
|
||||
const hasMore = !hitLimit && transcripts.length === TRANSCRIPTS_PER_PAGE
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore ? String(skip + transcripts.length) : undefined,
|
||||
hasMore,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
_sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
try {
|
||||
const data = await firefliesGraphQL(
|
||||
accessToken,
|
||||
`query Transcript($id: String!) {
|
||||
transcript(id: $id) {
|
||||
id
|
||||
title
|
||||
date
|
||||
duration
|
||||
host_email
|
||||
organizer_email
|
||||
participants
|
||||
transcript_url
|
||||
speakers {
|
||||
id
|
||||
name
|
||||
}
|
||||
sentences {
|
||||
index
|
||||
speaker_name
|
||||
text
|
||||
}
|
||||
summary {
|
||||
keywords
|
||||
action_items
|
||||
overview
|
||||
short_summary
|
||||
}
|
||||
}
|
||||
}`,
|
||||
{ id: externalId }
|
||||
)
|
||||
|
||||
const transcript = data.transcript as FirefliesTranscript | null
|
||||
if (!transcript) return null
|
||||
|
||||
const content = formatTranscriptContent(transcript)
|
||||
const contentHash = `fireflies:${transcript.id}:${transcript.date ?? ''}:${transcript.duration ?? ''}`
|
||||
|
||||
const meetingDate = transcript.date ? new Date(transcript.date).toISOString() : undefined
|
||||
const speakerNames = transcript.speakers?.map((s) => s.name).filter(Boolean) ?? []
|
||||
|
||||
return {
|
||||
externalId: transcript.id,
|
||||
title: transcript.title || 'Untitled Meeting',
|
||||
content,
|
||||
contentDeferred: false,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: transcript.transcript_url || undefined,
|
||||
contentHash,
|
||||
metadata: {
|
||||
hostEmail: transcript.host_email,
|
||||
duration: transcript.duration,
|
||||
meetingDate,
|
||||
participants: transcript.participants,
|
||||
speakers: speakerNames,
|
||||
keywords: transcript.summary?.keywords,
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Failed to get Fireflies transcript', {
|
||||
externalId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const maxTranscripts = sourceConfig.maxTranscripts as string | undefined
|
||||
if (maxTranscripts && (Number.isNaN(Number(maxTranscripts)) || Number(maxTranscripts) < 0)) {
|
||||
return { valid: false, error: 'Max transcripts must be a non-negative number' }
|
||||
}
|
||||
|
||||
try {
|
||||
await firefliesGraphQL(
|
||||
accessToken,
|
||||
`query User {
|
||||
user {
|
||||
user_id
|
||||
name
|
||||
email
|
||||
}
|
||||
}`,
|
||||
{},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.hostEmail === 'string') {
|
||||
result.hostEmail = metadata.hostEmail
|
||||
}
|
||||
|
||||
const speakers = Array.isArray(metadata.speakers) ? (metadata.speakers as string[]) : []
|
||||
if (speakers.length > 0) {
|
||||
result.speakers = speakers.join(', ')
|
||||
}
|
||||
|
||||
if (metadata.duration != null) {
|
||||
const num = Number(metadata.duration)
|
||||
if (!Number.isNaN(num)) result.duration = num
|
||||
}
|
||||
|
||||
const meetingDate = parseTagDate(metadata.meetingDate)
|
||||
if (meetingDate) result.meetingDate = meetingDate
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { firefliesConnector } from '@/connectors/fireflies/fireflies'
|
||||
@@ -0,0 +1,41 @@
|
||||
import { FirefliesIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const firefliesConnectorMeta: ConnectorMeta = {
|
||||
id: 'fireflies',
|
||||
name: 'Fireflies',
|
||||
description: 'Sync meeting transcripts from Fireflies.ai',
|
||||
version: '1.0.0',
|
||||
icon: FirefliesIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'apiKey',
|
||||
label: 'API Key',
|
||||
placeholder: 'Enter your Fireflies API key',
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'hostEmail',
|
||||
title: 'Filter by Host Email',
|
||||
type: 'short-input',
|
||||
placeholder: 'e.g. john@example.com',
|
||||
required: false,
|
||||
description: 'Only sync transcripts hosted by this email',
|
||||
},
|
||||
{
|
||||
id: 'maxTranscripts',
|
||||
title: 'Max Transcripts',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 100 (default: unlimited)',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'hostEmail', displayName: 'Host Email', fieldType: 'text' },
|
||||
{ id: 'speakers', displayName: 'Speakers', fieldType: 'text' },
|
||||
{ id: 'duration', displayName: 'Duration (seconds)', fieldType: 'number' },
|
||||
{ id: 'meetingDate', displayName: 'Meeting Date', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,448 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { githubConnectorMeta } from '@/connectors/github/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import {
|
||||
CONNECTOR_MAX_FILE_BYTES,
|
||||
markSkipped,
|
||||
parseTagDate,
|
||||
sizeLimitSkipReason,
|
||||
stubOrSkipBySize,
|
||||
takeIndexableWithinCap,
|
||||
} from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('GitHubConnector')
|
||||
|
||||
const GITHUB_API_URL = 'https://api.github.com'
|
||||
const BATCH_SIZE = 30
|
||||
const GIT_SHA_PREFIX = 'git-sha:'
|
||||
const MAX_FILE_SIZE = CONNECTOR_MAX_FILE_BYTES
|
||||
const BINARY_SNIFF_BYTES = 8000
|
||||
|
||||
/**
|
||||
* Heuristic binary detection: Git treats files containing a NUL byte in the
|
||||
* first 8000 bytes as binary. Matches `git diff` / `git grep` semantics.
|
||||
*/
|
||||
function isBinaryBuffer(buf: Buffer): boolean {
|
||||
const len = Math.min(buf.length, BINARY_SNIFF_BYTES)
|
||||
for (let i = 0; i < len; i++) {
|
||||
if (buf[i] === 0) return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the repository string into owner and repo.
|
||||
*/
|
||||
function parseRepo(repository: string): { owner: string; repo: string } {
|
||||
const cleaned = repository.replace(/^https?:\/\/github\.com\//, '').replace(/\.git$/, '')
|
||||
const parts = cleaned.split('/')
|
||||
if (parts.length < 2 || !parts[0] || !parts[1]) {
|
||||
throw new Error(`Invalid repository format: "${repository}". Use "owner/repo".`)
|
||||
}
|
||||
return { owner: parts[0], repo: parts[1] }
|
||||
}
|
||||
|
||||
/**
|
||||
* File extension filter set from user config. Returns null if no filter (accept all).
|
||||
*/
|
||||
function parseExtensions(extensions: string): Set<string> | null {
|
||||
const trimmed = extensions.trim()
|
||||
if (!trimmed) return null
|
||||
const exts = trimmed
|
||||
.split(',')
|
||||
.map((e) => e.trim().toLowerCase())
|
||||
.filter(Boolean)
|
||||
.map((e) => (e.startsWith('.') ? e : `.${e}`))
|
||||
return exts.length > 0 ? new Set(exts) : null
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether a file path matches the extension filter.
|
||||
*/
|
||||
function matchesExtension(filePath: string, extSet: Set<string> | null): boolean {
|
||||
if (!extSet) return true
|
||||
const lastDot = filePath.lastIndexOf('.')
|
||||
if (lastDot === -1) return false
|
||||
return extSet.has(filePath.slice(lastDot).toLowerCase())
|
||||
}
|
||||
|
||||
interface TreeItem {
|
||||
path: string
|
||||
mode: string
|
||||
type: string
|
||||
sha: string
|
||||
size?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the full recursive tree for a branch.
|
||||
*/
|
||||
async function fetchTree(
|
||||
accessToken: string,
|
||||
owner: string,
|
||||
repo: string,
|
||||
branch: string
|
||||
): Promise<TreeItem[]> {
|
||||
const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/git/trees/${encodeURIComponent(branch)}?recursive=1`
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Accept: 'application/vnd.github+json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'X-GitHub-Api-Version': '2022-11-28',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error('Failed to fetch GitHub tree', { status: response.status, error: errorText })
|
||||
throw new Error(`Failed to fetch repository tree: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
|
||||
if (data.truncated) {
|
||||
logger.warn('GitHub tree was truncated — some files may be missing', { owner, repo, branch })
|
||||
}
|
||||
|
||||
return (data.tree || []).filter((item: TreeItem) => item.type === 'blob')
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches blob content via the Git Blobs API. Used as a fallback when the
|
||||
* `/contents/` endpoint cannot return the file body (files larger than 1 MB
|
||||
* return `content: ""` and `encoding: "none"`). Supports blobs up to 100 MB.
|
||||
*/
|
||||
async function fetchBlobContent(
|
||||
accessToken: string,
|
||||
owner: string,
|
||||
repo: string,
|
||||
sha: string
|
||||
): Promise<string | null> {
|
||||
const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/git/blobs/${encodeURIComponent(sha)}`
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Accept: 'application/vnd.github+json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'X-GitHub-Api-Version': '2022-11-28',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch git blob ${sha}: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const content = (data.content as string) || ''
|
||||
const encoding = data.encoding as string | undefined
|
||||
|
||||
if (encoding === 'base64') {
|
||||
const buf = Buffer.from(content, 'base64')
|
||||
if (isBinaryBuffer(buf)) return null
|
||||
return buf.toString('utf8')
|
||||
}
|
||||
/**
|
||||
* Per https://docs.github.com/en/rest/git/blobs the Blobs API only ever
|
||||
* returns base64. Refuse to silently persist empty content for an
|
||||
* unexpected encoding so a sync surfaces the error instead.
|
||||
*/
|
||||
throw new Error(`Unexpected git blob encoding for ${sha}: ${encoding ?? 'undefined'}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a lightweight stub ExternalDocument from a tree item.
|
||||
* Uses the Git blob SHA as contentHash for change detection, avoiding
|
||||
* the need to fetch blob content for every file during listing.
|
||||
* Content is deferred and only fetched for new/changed documents.
|
||||
*/
|
||||
function treeItemToStub(
|
||||
owner: string,
|
||||
repo: string,
|
||||
branch: string,
|
||||
item: TreeItem
|
||||
): ExternalDocument {
|
||||
return {
|
||||
externalId: item.path,
|
||||
title: item.path.split('/').pop() || item.path,
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: `https://github.com/${owner}/${repo}/blob/${branch.split('/').map(encodeURIComponent).join('/')}/${item.path.split('/').map(encodeURIComponent).join('/')}`,
|
||||
contentHash: `${GIT_SHA_PREFIX}${item.sha}`,
|
||||
metadata: {
|
||||
path: item.path,
|
||||
sha: item.sha,
|
||||
size: item.size,
|
||||
branch,
|
||||
repository: `${owner}/${repo}`,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export const githubConnector: ConnectorConfig = {
|
||||
...githubConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const { owner, repo } = parseRepo(sourceConfig.repository as string)
|
||||
const branch = ((sourceConfig.branch as string) || 'main').trim()
|
||||
const pathPrefix = ((sourceConfig.pathPrefix as string) || '').trim()
|
||||
const extSet = parseExtensions((sourceConfig.extensions as string) || '')
|
||||
const maxFiles = sourceConfig.maxFiles ? Number(sourceConfig.maxFiles) : 0
|
||||
|
||||
let capped: TreeItem[]
|
||||
if (syncContext?.filteredTree) {
|
||||
capped = syncContext.filteredTree as TreeItem[]
|
||||
} else {
|
||||
const tree = await fetchTree(accessToken, owner, repo, branch)
|
||||
|
||||
// Filter by path prefix and extensions. Oversized files are kept here and
|
||||
// surfaced as skipped (failed) documents at stub time so they stay visible.
|
||||
const filtered = tree.filter((item) => {
|
||||
if (pathPrefix && !item.path.startsWith(pathPrefix)) return false
|
||||
if (!matchesExtension(item.path, extSet)) return false
|
||||
return true
|
||||
})
|
||||
|
||||
// Apply the max-files limit to indexable files only; oversized files within
|
||||
// the capped window are kept (and surfaced as skipped) but never consume the cap.
|
||||
capped =
|
||||
maxFiles > 0
|
||||
? takeIndexableWithinCap(
|
||||
filtered,
|
||||
(item) => Boolean(item.size && item.size > MAX_FILE_SIZE),
|
||||
maxFiles,
|
||||
0
|
||||
).documents
|
||||
: filtered
|
||||
if (syncContext) syncContext.filteredTree = capped
|
||||
}
|
||||
|
||||
// Paginate using offset cursor
|
||||
const offset = cursor ? Number(cursor) : 0
|
||||
const batch = capped.slice(offset, offset + BATCH_SIZE)
|
||||
|
||||
logger.info('Listing GitHub files', {
|
||||
owner,
|
||||
repo,
|
||||
branch,
|
||||
totalFiltered: capped.length,
|
||||
offset,
|
||||
batchSize: batch.length,
|
||||
})
|
||||
|
||||
const documents = batch.map((item) =>
|
||||
stubOrSkipBySize(treeItemToStub(owner, repo, branch, item), item.size, MAX_FILE_SIZE)
|
||||
)
|
||||
|
||||
const nextOffset = offset + BATCH_SIZE
|
||||
const hasMore = nextOffset < capped.length
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore ? String(nextOffset) : undefined,
|
||||
hasMore,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
externalId: string,
|
||||
_syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocument | null> => {
|
||||
const { owner, repo } = parseRepo(sourceConfig.repository as string)
|
||||
const branch = ((sourceConfig.branch as string) || 'main').trim()
|
||||
|
||||
// externalId is the file path
|
||||
const path = externalId
|
||||
|
||||
try {
|
||||
const encodedPath = path.split('/').map(encodeURIComponent).join('/')
|
||||
const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/contents/${encodedPath}?ref=${encodeURIComponent(branch)}`
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Accept: 'application/vnd.github+json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'X-GitHub-Api-Version': '2022-11-28',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) return null
|
||||
if (response.status === 403) {
|
||||
logger.info('Skipping GitHub file rejected by Contents API', {
|
||||
path,
|
||||
status: response.status,
|
||||
})
|
||||
return null
|
||||
}
|
||||
throw new Error(`Failed to fetch file ${path}: ${response.status}`)
|
||||
}
|
||||
|
||||
const lastModifiedHeader = response.headers.get('last-modified') || undefined
|
||||
const data = await response.json()
|
||||
|
||||
const size = typeof data.size === 'number' ? data.size : 0
|
||||
if (size > MAX_FILE_SIZE) {
|
||||
logger.info('Skipping GitHub file exceeding size limit', {
|
||||
path,
|
||||
size,
|
||||
limit: MAX_FILE_SIZE,
|
||||
})
|
||||
return markSkipped(
|
||||
{
|
||||
externalId,
|
||||
title: path.split('/').pop() || path,
|
||||
content: '',
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: `https://github.com/${owner}/${repo}/blob/${branch.split('/').map(encodeURIComponent).join('/')}/${path.split('/').map(encodeURIComponent).join('/')}`,
|
||||
contentHash: `${GIT_SHA_PREFIX}${data.sha as string}`,
|
||||
metadata: {
|
||||
path,
|
||||
sha: data.sha as string,
|
||||
size,
|
||||
branch,
|
||||
repository: `${owner}/${repo}`,
|
||||
},
|
||||
},
|
||||
sizeLimitSkipReason(MAX_FILE_SIZE)
|
||||
)
|
||||
}
|
||||
|
||||
const rawContent = (data.content as string) || ''
|
||||
const encoding = data.encoding as string | undefined
|
||||
let content: string
|
||||
if (encoding === 'base64' && rawContent.length > 0) {
|
||||
const buf = Buffer.from(rawContent, 'base64')
|
||||
if (isBinaryBuffer(buf)) {
|
||||
logger.info('Skipping binary GitHub file', { path, size })
|
||||
return null
|
||||
}
|
||||
content = buf.toString('utf8')
|
||||
} else if (encoding === 'none' && data.sha && size > 0) {
|
||||
const blobContent = await fetchBlobContent(accessToken, owner, repo, data.sha as string)
|
||||
if (blobContent === null) {
|
||||
logger.info('Skipping binary GitHub file', { path, size })
|
||||
return null
|
||||
}
|
||||
content = blobContent
|
||||
} else {
|
||||
content = ''
|
||||
}
|
||||
|
||||
return {
|
||||
externalId,
|
||||
title: path.split('/').pop() || path,
|
||||
content,
|
||||
contentDeferred: false,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: `https://github.com/${owner}/${repo}/blob/${branch.split('/').map(encodeURIComponent).join('/')}/${path.split('/').map(encodeURIComponent).join('/')}`,
|
||||
contentHash: `${GIT_SHA_PREFIX}${data.sha as string}`,
|
||||
metadata: {
|
||||
path,
|
||||
sha: data.sha as string,
|
||||
size: data.size as number,
|
||||
branch,
|
||||
repository: `${owner}/${repo}`,
|
||||
lastModified: lastModifiedHeader,
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn(`Failed to fetch GitHub document ${externalId}`, {
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const repository = (sourceConfig.repository as string)?.trim()
|
||||
if (!repository) {
|
||||
return { valid: false, error: 'Repository is required' }
|
||||
}
|
||||
|
||||
let owner: string
|
||||
let repo: string
|
||||
try {
|
||||
const parsed = parseRepo(repository)
|
||||
owner = parsed.owner
|
||||
repo = parsed.repo
|
||||
} catch (error) {
|
||||
return {
|
||||
valid: false,
|
||||
error: getErrorMessage(error, 'Invalid repository format'),
|
||||
}
|
||||
}
|
||||
|
||||
const maxFiles = sourceConfig.maxFiles as string | undefined
|
||||
if (maxFiles && (Number.isNaN(Number(maxFiles)) || Number(maxFiles) <= 0)) {
|
||||
return { valid: false, error: 'Max files must be a positive number' }
|
||||
}
|
||||
|
||||
const branch = ((sourceConfig.branch as string) || 'main').trim()
|
||||
|
||||
try {
|
||||
// Verify repo and branch are accessible
|
||||
const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/branches/${encodeURIComponent(branch)}`
|
||||
const response = await fetchWithRetry(
|
||||
url,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Accept: 'application/vnd.github+json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'X-GitHub-Api-Version': '2022-11-28',
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (response.status === 404) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `Repository "${owner}/${repo}" or branch "${branch}" not found`,
|
||||
}
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
return { valid: false, error: `Cannot access repository: ${response.status}` }
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.path === 'string') result.path = metadata.path
|
||||
if (typeof metadata.repository === 'string') result.repository = metadata.repository
|
||||
if (typeof metadata.branch === 'string') result.branch = metadata.branch
|
||||
|
||||
if (metadata.size != null) {
|
||||
const num = Number(metadata.size)
|
||||
if (!Number.isNaN(num)) result.size = num
|
||||
}
|
||||
|
||||
const lastModified = parseTagDate(metadata.lastModified)
|
||||
if (lastModified) result.lastModified = lastModified
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { githubConnector } from '@/connectors/github/github'
|
||||
@@ -0,0 +1,62 @@
|
||||
import { GithubIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const githubConnectorMeta: ConnectorMeta = {
|
||||
id: 'github',
|
||||
name: 'GitHub',
|
||||
description: 'Sync files from a GitHub repository',
|
||||
version: '1.0.0',
|
||||
icon: GithubIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'apiKey',
|
||||
label: 'Personal Access Token',
|
||||
placeholder: 'ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx',
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'repository',
|
||||
title: 'Repository',
|
||||
type: 'short-input',
|
||||
placeholder: 'owner/repo',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'branch',
|
||||
title: 'Branch',
|
||||
type: 'short-input',
|
||||
placeholder: 'main (default)',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'pathPrefix',
|
||||
title: 'Path Filter',
|
||||
type: 'short-input',
|
||||
placeholder: 'e.g. docs/, src/components/',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'extensions',
|
||||
title: 'File Extensions',
|
||||
type: 'short-input',
|
||||
placeholder: 'e.g. .md, .txt, .mdx',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'maxFiles',
|
||||
title: 'Max Files',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 500 (default: unlimited)',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'path', displayName: 'File Path', fieldType: 'text' },
|
||||
{ id: 'repository', displayName: 'Repository', fieldType: 'text' },
|
||||
{ id: 'branch', displayName: 'Branch', fieldType: 'text' },
|
||||
{ id: 'size', displayName: 'File Size', fieldType: 'number' },
|
||||
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
||||
export { gitlabConnector } from '@/connectors/gitlab/gitlab'
|
||||
@@ -0,0 +1,140 @@
|
||||
import { GitLabIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const gitlabConnectorMeta: ConnectorMeta = {
|
||||
id: 'gitlab',
|
||||
name: 'GitLab',
|
||||
description:
|
||||
'Sync repository files, wiki pages, and issues from a GitLab project into your knowledge base',
|
||||
version: '1.0.0',
|
||||
icon: GitLabIcon,
|
||||
|
||||
/**
|
||||
* Incremental sync applies to issues only (via the `updated_after` filter
|
||||
* derived from lastSyncAt). Wikis and repository files lack a change timestamp
|
||||
* on listing, so they are always re-listed in full and reconciled by content
|
||||
* hash (wiki: content digest, file: git blob SHA) — unchanged docs are skipped.
|
||||
*/
|
||||
supportsIncrementalSync: true,
|
||||
|
||||
auth: {
|
||||
mode: 'apiKey',
|
||||
label: 'Personal Access Token',
|
||||
placeholder: 'Enter your GitLab PAT',
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'host',
|
||||
title: 'Host',
|
||||
type: 'short-input',
|
||||
placeholder: 'gitlab.com',
|
||||
required: false,
|
||||
description: 'Self-managed GitLab host. Leave blank for gitlab.com.',
|
||||
},
|
||||
{
|
||||
id: 'project',
|
||||
title: 'Project',
|
||||
type: 'short-input',
|
||||
placeholder: 'group/project or numeric ID',
|
||||
required: true,
|
||||
description: 'Project path (e.g. my-group/my-repo) or numeric project ID.',
|
||||
},
|
||||
{
|
||||
id: 'contentTypes',
|
||||
title: 'Content',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
options: [
|
||||
{ label: 'Code, Wiki & Issues', id: 'all' },
|
||||
{ label: 'Code (repository files) only', id: 'repo' },
|
||||
{ label: 'Wiki only', id: 'wiki' },
|
||||
{ label: 'Issues only', id: 'issues' },
|
||||
{ label: 'Wiki & Issues', id: 'both' },
|
||||
],
|
||||
description: 'Which content to index. "Code" syncs repository files (READMEs, docs, source).',
|
||||
},
|
||||
{
|
||||
id: 'ref',
|
||||
title: 'Branch',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'Default branch',
|
||||
description: 'Branch or tag to sync repository files from. Applies only when syncing Code.',
|
||||
},
|
||||
{
|
||||
id: 'pathPrefix',
|
||||
title: 'Path Filter',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. docs/',
|
||||
description:
|
||||
'Only sync repository files under this path prefix. Applies only when syncing Code.',
|
||||
},
|
||||
{
|
||||
id: 'fileExtensions',
|
||||
title: 'File Extensions',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. .md, .txt, .mdx',
|
||||
description:
|
||||
'Only sync repository files with these extensions (comma-separated). Leave blank for all text files. Applies only when syncing Code.',
|
||||
},
|
||||
{
|
||||
id: 'issueState',
|
||||
title: 'Issue State',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
options: [
|
||||
{ label: 'All', id: 'all' },
|
||||
{ label: 'Open only', id: 'opened' },
|
||||
{ label: 'Closed only', id: 'closed' },
|
||||
],
|
||||
description: 'Which issues to sync by state. Applies only when syncing issues.',
|
||||
},
|
||||
{
|
||||
id: 'issueLabels',
|
||||
title: 'Issue Labels',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. bug,docs (comma-separated)',
|
||||
description:
|
||||
'Only sync issues with all of these labels (comma-separated). Applies only when syncing issues.',
|
||||
},
|
||||
{
|
||||
id: 'issueMilestone',
|
||||
title: 'Issue Milestone',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. v1.0 (milestone title)',
|
||||
description:
|
||||
'Only sync issues assigned to this milestone (exact title). Applies only when syncing issues.',
|
||||
},
|
||||
{
|
||||
id: 'maxItems',
|
||||
title: 'Max Items',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 500 (default: unlimited)',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'contentType', displayName: 'Content Type', fieldType: 'text' },
|
||||
{ id: 'title', displayName: 'Title', fieldType: 'text' },
|
||||
{ id: 'state', displayName: 'State', fieldType: 'text' },
|
||||
{ id: 'author', displayName: 'Author', fieldType: 'text' },
|
||||
{ id: 'labels', displayName: 'Labels', fieldType: 'text' },
|
||||
{ id: 'milestone', displayName: 'Milestone', fieldType: 'text' },
|
||||
{ id: 'path', displayName: 'File Path', fieldType: 'text' },
|
||||
{ id: 'size', displayName: 'File Size (bytes)', fieldType: 'number' },
|
||||
{ id: 'createdAt', displayName: 'Created At', fieldType: 'date' },
|
||||
{ id: 'updatedAt', displayName: 'Updated At', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,575 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { DEFAULT_MAX_THREADS, gmailConnectorMeta } from '@/connectors/gmail/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { htmlToPlainText, joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('GmailConnector')
|
||||
|
||||
const GMAIL_API_BASE = 'https://gmail.googleapis.com/gmail/v1/users/me'
|
||||
const THREADS_PER_PAGE = 100
|
||||
|
||||
interface GmailHeader {
|
||||
name: string
|
||||
value: string
|
||||
}
|
||||
|
||||
interface GmailMessagePart {
|
||||
mimeType?: string
|
||||
body?: { data?: string; size?: number }
|
||||
parts?: GmailMessagePart[]
|
||||
headers?: GmailHeader[]
|
||||
}
|
||||
|
||||
interface GmailMessage {
|
||||
id: string
|
||||
threadId: string
|
||||
internalDate?: string
|
||||
payload?: GmailMessagePart
|
||||
labelIds?: string[]
|
||||
snippet?: string
|
||||
}
|
||||
|
||||
interface GmailThread {
|
||||
id: string
|
||||
historyId?: string
|
||||
messages?: GmailMessage[]
|
||||
snippet?: string
|
||||
}
|
||||
|
||||
interface GmailLabel {
|
||||
id: string
|
||||
name: string
|
||||
type?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a single Gmail label name for use in a `label:` operator.
|
||||
* Gmail search syntax accepts quoted strings for labels containing spaces;
|
||||
* unquoted label tokens have spaces replaced with hyphens.
|
||||
*/
|
||||
function formatLabelToken(name: string): string {
|
||||
const trimmed = name.trim()
|
||||
if (!trimmed) return ''
|
||||
if (/\s/.test(trimmed)) {
|
||||
const escaped = trimmed.replace(/\\/g, '\\\\').replace(/"/g, '\\"')
|
||||
return `label:"${escaped}"`
|
||||
}
|
||||
return `label:${trimmed}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a Gmail search query string from the source config.
|
||||
* Combines the user's custom query with the label and date range filters.
|
||||
* When multiple labels are provided, they are OR-joined: `(label:A OR label:B)`.
|
||||
*/
|
||||
function buildSearchQuery(sourceConfig: Record<string, unknown>): string {
|
||||
const parts: string[] = []
|
||||
|
||||
const labelNames = parseMultiValue(sourceConfig.label)
|
||||
if (labelNames.length === 1) {
|
||||
const token = formatLabelToken(labelNames[0])
|
||||
if (token) parts.push(token)
|
||||
} else if (labelNames.length > 1) {
|
||||
const tokens = labelNames.map(formatLabelToken).filter(Boolean)
|
||||
if (tokens.length === 1) {
|
||||
parts.push(tokens[0])
|
||||
} else if (tokens.length > 1) {
|
||||
parts.push(`(${tokens.join(' OR ')})`)
|
||||
}
|
||||
}
|
||||
|
||||
const dateRange = (sourceConfig.dateRange as string) || 'all'
|
||||
const now = new Date()
|
||||
switch (dateRange) {
|
||||
case '7d':
|
||||
parts.push(`after:${formatGmailDate(daysAgo(now, 7))}`)
|
||||
break
|
||||
case '30d':
|
||||
parts.push(`after:${formatGmailDate(daysAgo(now, 30))}`)
|
||||
break
|
||||
case '90d':
|
||||
parts.push(`after:${formatGmailDate(daysAgo(now, 90))}`)
|
||||
break
|
||||
case '6m':
|
||||
parts.push(`after:${formatGmailDate(daysAgo(now, 180))}`)
|
||||
break
|
||||
case '1y':
|
||||
parts.push(`after:${formatGmailDate(daysAgo(now, 365))}`)
|
||||
break
|
||||
}
|
||||
|
||||
const excludePromotions = sourceConfig.excludePromotions !== 'false'
|
||||
if (excludePromotions) {
|
||||
parts.push('-category:promotions')
|
||||
}
|
||||
|
||||
const excludeSocial = sourceConfig.excludeSocial !== 'false'
|
||||
if (excludeSocial) {
|
||||
parts.push('-category:social')
|
||||
}
|
||||
|
||||
const customQuery = sourceConfig.query as string | undefined
|
||||
const trimmedCustom = customQuery?.trim()
|
||||
if (trimmedCustom) {
|
||||
/**
|
||||
* Wrap the user-supplied query in parentheses whenever it contains an OR
|
||||
* so it's AND-joined as a single clause with the preceding label / category
|
||||
* / date filters. Always wrap (rather than try to detect existing outer
|
||||
* parens) because a regex like /^\(.*\)$/ misclassifies inputs such as
|
||||
* `(from:alice) OR (from:bob)` where the parens don't bracket the whole
|
||||
* expression. Double-wrapping is a no-op in Gmail search syntax.
|
||||
*/
|
||||
const needsGroup = /\bOR\b/i.test(trimmedCustom)
|
||||
parts.push(needsGroup ? `(${trimmedCustom})` : trimmedCustom)
|
||||
}
|
||||
|
||||
return parts.join(' ')
|
||||
}
|
||||
|
||||
function daysAgo(now: Date, days: number): Date {
|
||||
return new Date(now.getTime() - days * 24 * 60 * 60 * 1000)
|
||||
}
|
||||
|
||||
function formatGmailDate(date: Date): string {
|
||||
const y = date.getFullYear()
|
||||
const m = String(date.getMonth() + 1).padStart(2, '0')
|
||||
const d = String(date.getDate()).padStart(2, '0')
|
||||
return `${y}/${m}/${d}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Decodes base64url-encoded content from the Gmail API.
|
||||
* Uses Buffer to correctly handle multi-byte UTF-8 characters.
|
||||
*/
|
||||
function decodeBase64Url(data: string): string {
|
||||
return Buffer.from(data, 'base64url').toString('utf-8')
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the plain text body from a Gmail message payload.
|
||||
* Prefers text/plain, falls back to text/html with tag stripping.
|
||||
*/
|
||||
function extractBody(part: GmailMessagePart): string {
|
||||
if (part.mimeType === 'text/plain' && part.body?.data) {
|
||||
return decodeBase64Url(part.body.data)
|
||||
}
|
||||
|
||||
if (part.parts) {
|
||||
// Prefer text/plain from multipart
|
||||
for (const child of part.parts) {
|
||||
if (child.mimeType === 'text/plain' && child.body?.data) {
|
||||
return decodeBase64Url(child.body.data)
|
||||
}
|
||||
}
|
||||
// Fall back to text/html
|
||||
for (const child of part.parts) {
|
||||
if (child.mimeType === 'text/html' && child.body?.data) {
|
||||
return htmlToPlainText(decodeBase64Url(child.body.data))
|
||||
}
|
||||
}
|
||||
// Recurse into nested multipart
|
||||
for (const child of part.parts) {
|
||||
const result = extractBody(child)
|
||||
if (result) return result
|
||||
}
|
||||
}
|
||||
|
||||
if (part.mimeType === 'text/html' && part.body?.data) {
|
||||
return htmlToPlainText(decodeBase64Url(part.body.data))
|
||||
}
|
||||
|
||||
return ''
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets a header value from a Gmail message payload.
|
||||
*/
|
||||
function getHeader(payload: GmailMessagePart | undefined, name: string): string | undefined {
|
||||
if (!payload?.headers) return undefined
|
||||
const header = payload.headers.find((h) => h.name.toLowerCase() === name.toLowerCase())
|
||||
return header?.value
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a thread's messages into a single document string.
|
||||
*/
|
||||
function formatThread(thread: GmailThread): {
|
||||
content: string
|
||||
subject: string
|
||||
metadata: Record<string, unknown>
|
||||
} {
|
||||
const messages = thread.messages || []
|
||||
if (messages.length === 0) {
|
||||
return { content: '', subject: 'Untitled Thread', metadata: {} }
|
||||
}
|
||||
|
||||
const firstMessage = messages[0]
|
||||
const lastMessage = messages[messages.length - 1]
|
||||
const subject = getHeader(firstMessage.payload, 'Subject') || 'No Subject'
|
||||
const from = getHeader(firstMessage.payload, 'From') || 'Unknown'
|
||||
const to = getHeader(firstMessage.payload, 'To') || ''
|
||||
const labelIds = firstMessage.labelIds || []
|
||||
|
||||
const lines: string[] = []
|
||||
lines.push(`Subject: ${subject}`)
|
||||
lines.push(`From: ${from}`)
|
||||
if (to) lines.push(`To: ${to}`)
|
||||
lines.push(`Messages: ${messages.length}`)
|
||||
lines.push('')
|
||||
|
||||
for (const msg of messages) {
|
||||
const msgFrom = getHeader(msg.payload, 'From') || 'Unknown'
|
||||
const msgDate = getHeader(msg.payload, 'Date') || ''
|
||||
const body = msg.payload ? extractBody(msg.payload) : ''
|
||||
|
||||
lines.push(`--- ${msgFrom} (${msgDate}) ---`)
|
||||
lines.push(body.trim())
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
const firstDate = firstMessage.internalDate
|
||||
? new Date(Number(firstMessage.internalDate)).toISOString()
|
||||
: undefined
|
||||
const lastDate = lastMessage.internalDate
|
||||
? new Date(Number(lastMessage.internalDate)).toISOString()
|
||||
: undefined
|
||||
|
||||
return {
|
||||
content: lines.join('\n').trim(),
|
||||
subject,
|
||||
metadata: {
|
||||
from,
|
||||
to,
|
||||
subject,
|
||||
messageCount: messages.length,
|
||||
labelIds,
|
||||
firstMessageDate: firstDate,
|
||||
lastMessageDate: lastDate,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches a full thread with all its messages.
|
||||
*/
|
||||
async function fetchThread(accessToken: string, threadId: string): Promise<GmailThread | null> {
|
||||
const url = `${GMAIL_API_BASE}/threads/${threadId}?format=FULL`
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) return null
|
||||
throw new Error(`Failed to fetch thread ${threadId}: ${response.status}`)
|
||||
}
|
||||
|
||||
return (await response.json()) as GmailThread
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves label IDs to human-readable label names using a cache.
|
||||
*/
|
||||
async function resolveLabelNames(
|
||||
accessToken: string,
|
||||
labelIds: string[],
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<string[]> {
|
||||
const cacheKey = '_gmailLabelCache'
|
||||
|
||||
if (syncContext && !syncContext[cacheKey]) {
|
||||
try {
|
||||
const url = `${GMAIL_API_BASE}/labels`
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (response.ok) {
|
||||
const data = await response.json()
|
||||
const labels = (data.labels || []) as GmailLabel[]
|
||||
const labelMap: Record<string, string> = {}
|
||||
for (const label of labels) {
|
||||
labelMap[label.id] = label.name
|
||||
}
|
||||
syncContext[cacheKey] = labelMap
|
||||
}
|
||||
} catch {
|
||||
syncContext[cacheKey] = {}
|
||||
}
|
||||
}
|
||||
|
||||
const cache = (syncContext?.[cacheKey] as Record<string, string>) ?? {}
|
||||
return labelIds
|
||||
.map((id) => cache[id] || id)
|
||||
.filter((name) => !name.startsWith('CATEGORY_') && name !== 'UNREAD')
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a lightweight document stub from a thread list entry.
|
||||
* Uses metadata-based contentHash for change detection without downloading content.
|
||||
*/
|
||||
function threadToStub(thread: {
|
||||
id: string
|
||||
snippet?: string
|
||||
historyId?: string
|
||||
}): ExternalDocument {
|
||||
return {
|
||||
externalId: thread.id,
|
||||
title: thread.snippet || 'Untitled Thread',
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: `https://mail.google.com/mail/u/0/#inbox/${thread.id}`,
|
||||
contentHash: `gmail:${thread.id}:${thread.historyId ?? ''}`,
|
||||
metadata: {},
|
||||
}
|
||||
}
|
||||
|
||||
export const gmailConnector: ConnectorConfig = {
|
||||
...gmailConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const searchQuery = buildSearchQuery(sourceConfig)
|
||||
const maxThreads = sourceConfig.maxThreads
|
||||
? Number(sourceConfig.maxThreads)
|
||||
: DEFAULT_MAX_THREADS
|
||||
|
||||
const totalFetched = (syncContext?.totalThreadsFetched as number) ?? 0
|
||||
if (totalFetched >= maxThreads) {
|
||||
return { documents: [], hasMore: false }
|
||||
}
|
||||
|
||||
const remaining = maxThreads - totalFetched
|
||||
const pageSize = Math.min(THREADS_PER_PAGE, remaining)
|
||||
|
||||
const queryParams = new URLSearchParams({
|
||||
maxResults: String(pageSize),
|
||||
})
|
||||
|
||||
if (searchQuery) {
|
||||
queryParams.set('q', searchQuery)
|
||||
}
|
||||
|
||||
if (cursor) {
|
||||
queryParams.set('pageToken', cursor)
|
||||
}
|
||||
|
||||
const url = `${GMAIL_API_BASE}/threads?${queryParams.toString()}`
|
||||
|
||||
logger.info('Listing Gmail threads', {
|
||||
query: searchQuery,
|
||||
cursor: cursor ?? 'initial',
|
||||
maxThreads,
|
||||
})
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error('Failed to list Gmail threads', { status: response.status, error: errorText })
|
||||
throw new Error(`Failed to list Gmail threads: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const threads = (data.threads || []) as { id: string; snippet?: string; historyId?: string }[]
|
||||
|
||||
if (threads.length === 0) {
|
||||
return { documents: [], hasMore: false }
|
||||
}
|
||||
|
||||
const documents = threads.map(threadToStub)
|
||||
|
||||
const newTotal = totalFetched + documents.length
|
||||
if (syncContext) syncContext.totalThreadsFetched = newTotal
|
||||
|
||||
const nextPageToken = data.nextPageToken as string | undefined
|
||||
const hitLimit = newTotal >= maxThreads
|
||||
if (hitLimit && syncContext) syncContext.listingCapped = true
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hitLimit ? undefined : nextPageToken,
|
||||
hasMore: hitLimit ? false : Boolean(nextPageToken),
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
_sourceConfig: Record<string, unknown>,
|
||||
externalId: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocument | null> => {
|
||||
try {
|
||||
const thread = await fetchThread(accessToken, externalId)
|
||||
if (!thread) return null
|
||||
|
||||
const { content, subject, metadata } = formatThread(thread)
|
||||
if (!content.trim()) return null
|
||||
|
||||
const labelIds = (metadata.labelIds as string[]) || []
|
||||
const labelNames = await resolveLabelNames(accessToken, labelIds, syncContext)
|
||||
metadata.labels = labelNames
|
||||
|
||||
return {
|
||||
externalId: thread.id,
|
||||
title: subject,
|
||||
content,
|
||||
contentDeferred: false,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: `https://mail.google.com/mail/u/0/#inbox/${thread.id}`,
|
||||
contentHash: `gmail:${thread.id}:${thread.historyId ?? ''}`,
|
||||
metadata,
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Failed to get Gmail thread', {
|
||||
externalId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const maxThreads = sourceConfig.maxThreads as string | undefined
|
||||
|
||||
if (maxThreads && (Number.isNaN(Number(maxThreads)) || Number(maxThreads) <= 0)) {
|
||||
return { valid: false, error: 'Max threads must be a positive number' }
|
||||
}
|
||||
|
||||
try {
|
||||
// Verify Gmail API access by fetching profile
|
||||
const profileUrl = `${GMAIL_API_BASE}/profile`
|
||||
const profileResponse = await fetchWithRetry(
|
||||
profileUrl,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!profileResponse.ok) {
|
||||
return { valid: false, error: `Failed to access Gmail: ${profileResponse.status}` }
|
||||
}
|
||||
|
||||
// If labels are specified, verify each one exists
|
||||
const labelNames = parseMultiValue(sourceConfig.label)
|
||||
if (labelNames.length > 0) {
|
||||
const labelsUrl = `${GMAIL_API_BASE}/labels`
|
||||
const labelsResponse = await fetchWithRetry(
|
||||
labelsUrl,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!labelsResponse.ok) {
|
||||
return { valid: false, error: 'Failed to fetch labels' }
|
||||
}
|
||||
|
||||
const labelsData = await labelsResponse.json()
|
||||
const labels = (labelsData.labels || []) as GmailLabel[]
|
||||
const labelNameSet = new Set(labels.map((l) => l.name.toLowerCase()))
|
||||
const missing = labelNames.filter((name) => !labelNameSet.has(name.toLowerCase()))
|
||||
|
||||
if (missing.length > 0) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `Label(s) not found: ${missing.join(', ')}. Available labels: ${labels
|
||||
.filter(
|
||||
(l) =>
|
||||
l.type !== 'system' ||
|
||||
['INBOX', 'IMPORTANT', 'STARRED', 'SENT', 'DRAFT'].includes(l.id)
|
||||
)
|
||||
.map((l) => l.name)
|
||||
.slice(0, 15)
|
||||
.join(', ')}`,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If a custom query is specified, verify it's valid by doing a dry-run
|
||||
const query = sourceConfig.query as string | undefined
|
||||
if (query?.trim()) {
|
||||
const searchQuery = buildSearchQuery(sourceConfig)
|
||||
const testUrl = `${GMAIL_API_BASE}/threads?q=${encodeURIComponent(searchQuery)}&maxResults=1`
|
||||
const testResponse = await fetchWithRetry(
|
||||
testUrl,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!testResponse.ok) {
|
||||
return { valid: false, error: 'Invalid search query. Check Gmail search syntax.' }
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.from === 'string') {
|
||||
result.from = metadata.from
|
||||
}
|
||||
|
||||
const labels = joinTagArray(metadata.labels)
|
||||
if (labels) {
|
||||
result.labels = labels
|
||||
}
|
||||
|
||||
if (typeof metadata.messageCount === 'number') {
|
||||
result.messageCount = metadata.messageCount
|
||||
}
|
||||
|
||||
const lastMessageDate = parseTagDate(metadata.lastMessageDate)
|
||||
if (lastMessageDate) {
|
||||
result.lastMessageDate = lastMessageDate
|
||||
}
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { gmailConnector } from '@/connectors/gmail/gmail'
|
||||
@@ -0,0 +1,100 @@
|
||||
import { GmailIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const DEFAULT_MAX_THREADS = 500
|
||||
|
||||
export const gmailConnectorMeta: ConnectorMeta = {
|
||||
id: 'gmail',
|
||||
name: 'Gmail',
|
||||
description: 'Sync email threads from Gmail',
|
||||
version: '1.0.0',
|
||||
icon: GmailIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'oauth',
|
||||
provider: 'google-email',
|
||||
requiredScopes: ['https://www.googleapis.com/auth/gmail.modify'],
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'labelSelector',
|
||||
title: 'Labels',
|
||||
type: 'selector',
|
||||
selectorKey: 'gmail.labels',
|
||||
canonicalParamId: 'label',
|
||||
mode: 'basic',
|
||||
multi: true,
|
||||
placeholder: 'Select one or more labels',
|
||||
required: false,
|
||||
description: 'Only sync emails matching any of these labels. Leave empty for all mail.',
|
||||
},
|
||||
{
|
||||
id: 'label',
|
||||
title: 'Labels',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'label',
|
||||
mode: 'advanced',
|
||||
multi: true,
|
||||
placeholder: 'e.g. INBOX, IMPORTANT (comma-separated; commas in label names not supported)',
|
||||
required: false,
|
||||
description: 'Only sync emails matching any of these labels. Leave empty for all mail.',
|
||||
},
|
||||
{
|
||||
id: 'dateRange',
|
||||
title: 'Date Range',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
options: [
|
||||
{ label: 'Last 7 days', id: '7d' },
|
||||
{ label: 'Last 30 days', id: '30d' },
|
||||
{ label: 'Last 90 days', id: '90d' },
|
||||
{ label: 'Last 6 months', id: '6m' },
|
||||
{ label: 'Last year', id: '1y' },
|
||||
{ label: 'All time', id: 'all' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'excludePromotions',
|
||||
title: 'Exclude Promotions',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
options: [
|
||||
{ label: 'Yes (recommended)', id: 'true' },
|
||||
{ label: 'No', id: 'false' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'excludeSocial',
|
||||
title: 'Exclude Social',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
options: [
|
||||
{ label: 'Yes (recommended)', id: 'true' },
|
||||
{ label: 'No', id: 'false' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'query',
|
||||
title: 'Search Filter',
|
||||
type: 'short-input',
|
||||
placeholder: 'e.g. from:boss@company.com subject:report has:attachment',
|
||||
required: false,
|
||||
description: 'Additional Gmail search filter. Uses the same syntax as the Gmail search bar.',
|
||||
},
|
||||
{
|
||||
id: 'maxThreads',
|
||||
title: 'Max Threads',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: `e.g. 200 (default: ${DEFAULT_MAX_THREADS})`,
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'from', displayName: 'From', fieldType: 'text' },
|
||||
{ id: 'labels', displayName: 'Labels', fieldType: 'text' },
|
||||
{ id: 'messageCount', displayName: 'Messages in Thread', fieldType: 'number' },
|
||||
{ id: 'lastMessageDate', displayName: 'Last Message', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,540 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { gongConnectorMeta } from '@/connectors/gong/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('GongConnector')
|
||||
|
||||
const GONG_API_BASE = 'https://api.gong.io/v2'
|
||||
const DEFAULT_LOOKBACK_DAYS = 90
|
||||
const MAX_LOOKBACK_DAYS = 180
|
||||
/**
|
||||
* Days of overlap added when computing the incremental sync window. Gong call data
|
||||
* (parties, transcript) can finish processing minutes to hours — occasionally a
|
||||
* day or two — after a call ends. The sync engine re-attempts calls whose
|
||||
* transcript was not yet ready (a null getDocument result is never persisted, so
|
||||
* the call is re-listed and re-fetched on the next sync), but only while the call
|
||||
* stays inside the incremental window. A two-week overlap keeps recently-ended
|
||||
* calls in that window long enough for late transcripts to be picked up, at the
|
||||
* cost of re-listing already-synced calls (skipped downstream by content hash).
|
||||
*/
|
||||
const INCREMENTAL_OVERLAP_DAYS = 14
|
||||
const MS_PER_DAY = 24 * 60 * 60 * 1000
|
||||
|
||||
/**
|
||||
* Metadata for a single call participant. `speakerId` cross-references the
|
||||
* `speakerId` field on transcript monologues, letting the connector attribute
|
||||
* each spoken line to a named participant.
|
||||
*/
|
||||
interface GongParty {
|
||||
id?: string
|
||||
name?: string
|
||||
emailAddress?: string
|
||||
speakerId?: string
|
||||
affiliation?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Core call metadata returned by the extensive calls endpoint. Mirrors Gong's
|
||||
* `CallBasicData` (the `metaData` object) — every field here is present on the
|
||||
* `/v2/calls/extensive` stub response and never requires the transcript fetch.
|
||||
*/
|
||||
interface GongCallMetaData {
|
||||
id?: string
|
||||
title?: string
|
||||
scheduled?: string
|
||||
started?: string
|
||||
duration?: number
|
||||
url?: string
|
||||
workspaceId?: string
|
||||
primaryUserId?: string
|
||||
direction?: string
|
||||
scope?: string
|
||||
system?: string
|
||||
language?: string
|
||||
purpose?: string
|
||||
isPrivate?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* A single call object from POST /v2/calls/extensive.
|
||||
*/
|
||||
interface GongExtensiveCall {
|
||||
metaData?: GongCallMetaData
|
||||
parties?: GongParty[]
|
||||
}
|
||||
|
||||
interface GongRecords {
|
||||
cursor?: string
|
||||
totalRecords?: number
|
||||
currentPageSize?: number
|
||||
}
|
||||
|
||||
interface GongExtensiveCallsResponse {
|
||||
calls?: GongExtensiveCall[]
|
||||
records?: GongRecords
|
||||
}
|
||||
|
||||
/**
|
||||
* A single sentence within a transcript monologue. Gong returns timing in
|
||||
* `startMs`/`endMs`; only `text` is used for the formatted transcript.
|
||||
*/
|
||||
interface GongTranscriptSentence {
|
||||
text?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* A monologue (one speaker turn) within a call transcript.
|
||||
*/
|
||||
interface GongMonologue {
|
||||
speakerId?: string
|
||||
topic?: string
|
||||
sentences?: GongTranscriptSentence[]
|
||||
}
|
||||
|
||||
interface GongCallTranscript {
|
||||
callId?: string
|
||||
transcript?: GongMonologue[]
|
||||
}
|
||||
|
||||
interface GongTranscriptResponse {
|
||||
callTranscripts?: GongCallTranscript[]
|
||||
records?: GongRecords
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the Authorization header value for Gong's Basic auth scheme.
|
||||
*
|
||||
* Gong authenticates with `Basic base64(accessKey:accessKeySecret)`. The sync
|
||||
* engine passes the user's stored key as `accessToken`. To support both raw
|
||||
* `accessKey:accessKeySecret` pairs and pre-encoded credentials, the raw form
|
||||
* (containing a colon) is base64-encoded here; an already-encoded value is sent
|
||||
* as-is.
|
||||
*/
|
||||
function buildAuthHeader(accessToken: string): string {
|
||||
const token = accessToken.includes(':')
|
||||
? Buffer.from(accessToken, 'utf8').toString('base64')
|
||||
: accessToken
|
||||
return `Basic ${token}`
|
||||
}
|
||||
|
||||
function buildHeaders(accessToken: string): Record<string, string> {
|
||||
return {
|
||||
'Content-Type': 'application/json',
|
||||
Accept: 'application/json',
|
||||
Authorization: buildAuthHeader(accessToken),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a comma- or newline-separated list of Gong IDs into a trimmed,
|
||||
* de-duplicated, non-empty array. Returns `undefined` when nothing usable
|
||||
* remains so the caller can omit the filter key entirely.
|
||||
*/
|
||||
function parseIdList(raw: unknown): string[] | undefined {
|
||||
if (typeof raw !== 'string') return undefined
|
||||
const ids = Array.from(
|
||||
new Set(
|
||||
raw
|
||||
.split(/[\n,]/)
|
||||
.map((value) => value.trim())
|
||||
.filter((value) => value.length > 0)
|
||||
)
|
||||
)
|
||||
return ids.length > 0 ? ids : undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Metadata-based content hash shared by `listDocuments` stubs and `getDocument`
|
||||
* results. Derived purely from call identity and its start time so the value is
|
||||
* identical across both paths — guaranteeing the sync engine only re-fetches a
|
||||
* transcript when the call's metadata actually changes.
|
||||
*/
|
||||
function buildContentHash(callId: string, started: string | undefined): string {
|
||||
return `gong:${callId}:${started ?? ''}`
|
||||
}
|
||||
|
||||
function buildCallTitle(metaData: GongCallMetaData | undefined): string {
|
||||
return metaData?.title?.trim() || 'Untitled Gong Call'
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts named participant labels from a call's parties for tag mapping and
|
||||
* the transcript header.
|
||||
*/
|
||||
function buildParticipantNames(parties: GongParty[] | undefined): string[] {
|
||||
if (!parties) return []
|
||||
const names: string[] = []
|
||||
for (const party of parties) {
|
||||
const label = party.name?.trim() || party.emailAddress?.trim()
|
||||
if (label) names.push(label)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a `speakerId` → display-name map from a call's parties so transcript
|
||||
* monologues (keyed by `speakerId`) can be attributed to a named speaker.
|
||||
*/
|
||||
function buildSpeakerMap(parties: GongParty[] | undefined): Record<string, string> {
|
||||
const map: Record<string, string> = {}
|
||||
if (!parties) return map
|
||||
for (const party of parties) {
|
||||
if (!party.speakerId) continue
|
||||
const label = party.name?.trim() || party.emailAddress?.trim()
|
||||
if (label) map[party.speakerId] = label
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
function buildMetadata(
|
||||
metaData: GongCallMetaData | undefined,
|
||||
participants: string[]
|
||||
): Record<string, unknown> {
|
||||
return {
|
||||
callId: metaData?.id,
|
||||
callTitle: metaData?.title,
|
||||
callDate: metaData?.started,
|
||||
scheduledDate: metaData?.scheduled,
|
||||
duration: metaData?.duration,
|
||||
workspaceId: metaData?.workspaceId,
|
||||
primaryUserId: metaData?.primaryUserId,
|
||||
direction: metaData?.direction,
|
||||
scope: metaData?.scope,
|
||||
system: metaData?.system,
|
||||
language: metaData?.language,
|
||||
purpose: metaData?.purpose,
|
||||
isPrivate: metaData?.isPrivate,
|
||||
participants,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a call's transcript into speaker-attributed plain text with a header
|
||||
* describing the call (title, date, duration, participants).
|
||||
*/
|
||||
function formatTranscriptContent(
|
||||
metaData: GongCallMetaData | undefined,
|
||||
participants: string[],
|
||||
speakerMap: Record<string, string>,
|
||||
monologues: GongMonologue[]
|
||||
): string {
|
||||
const parts: string[] = []
|
||||
|
||||
parts.push(`Call: ${buildCallTitle(metaData)}`)
|
||||
if (metaData?.started) parts.push(`Date: ${metaData.started}`)
|
||||
if (metaData?.duration != null) {
|
||||
const minutes = Math.round(metaData.duration / 60)
|
||||
parts.push(`Duration: ${minutes} minutes`)
|
||||
}
|
||||
if (participants.length > 0) parts.push(`Participants: ${participants.join(', ')}`)
|
||||
|
||||
parts.push('')
|
||||
parts.push('--- Transcript ---')
|
||||
|
||||
for (const monologue of monologues) {
|
||||
const speaker = (monologue.speakerId && speakerMap[monologue.speakerId]) || 'Unknown Speaker'
|
||||
const text = (monologue.sentences ?? [])
|
||||
.map((sentence) => sentence.text?.trim())
|
||||
.filter((value): value is string => Boolean(value))
|
||||
.join(' ')
|
||||
if (text) parts.push(`${speaker}: ${text}`)
|
||||
}
|
||||
|
||||
return parts.join('\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the effective lookback window in days, narrowing to the time since
|
||||
* the last successful sync (plus an overlap to catch transcripts that finished
|
||||
* processing late) when incremental sync is active.
|
||||
*/
|
||||
function computeLookbackDays(
|
||||
sourceConfig: Record<string, unknown>,
|
||||
lastSyncAt: Date | undefined
|
||||
): number {
|
||||
const raw = sourceConfig.lookback as string | undefined
|
||||
const configured = Number(raw)
|
||||
const baseline =
|
||||
Number.isFinite(configured) && configured > 0
|
||||
? Math.min(Math.floor(configured), MAX_LOOKBACK_DAYS)
|
||||
: DEFAULT_LOOKBACK_DAYS
|
||||
|
||||
if (!lastSyncAt) return baseline
|
||||
|
||||
const sinceLastSync = Math.ceil((Date.now() - lastSyncAt.getTime()) / MS_PER_DAY)
|
||||
const incremental = Math.max(sinceLastSync + INCREMENTAL_OVERLAP_DAYS, INCREMENTAL_OVERLAP_DAYS)
|
||||
return Math.min(incremental, baseline)
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches a single page of calls from POST /v2/calls/extensive with parties
|
||||
* exposed (needed to resolve transcript speaker IDs to names).
|
||||
*/
|
||||
async function fetchExtensiveCalls(
|
||||
accessToken: string,
|
||||
filter: Record<string, unknown>,
|
||||
cursor: string | undefined,
|
||||
retryOptions?: Parameters<typeof fetchWithRetry>[2]
|
||||
): Promise<GongExtensiveCallsResponse> {
|
||||
const body: Record<string, unknown> = {
|
||||
filter,
|
||||
contentSelector: { exposedFields: { parties: true } },
|
||||
}
|
||||
if (cursor) body.cursor = cursor
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
`${GONG_API_BASE}/calls/extensive`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: buildHeaders(accessToken),
|
||||
body: JSON.stringify(body),
|
||||
},
|
||||
retryOptions
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
throw new Error(
|
||||
`Failed to list Gong calls: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}`
|
||||
)
|
||||
}
|
||||
|
||||
return (await response.json()) as GongExtensiveCallsResponse
|
||||
}
|
||||
|
||||
export const gongConnector: ConnectorConfig = {
|
||||
...gongConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>,
|
||||
lastSyncAt?: Date
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const lookbackDays = computeLookbackDays(sourceConfig, lastSyncAt)
|
||||
const maxCalls = sourceConfig.maxCalls ? Number(sourceConfig.maxCalls) : 0
|
||||
const workspaceId = (sourceConfig.workspaceId as string | undefined)?.trim()
|
||||
const primaryUserIds = parseIdList(sourceConfig.primaryUserIds)
|
||||
|
||||
const cachedWindow = syncContext?.gongDateWindow as
|
||||
| { fromDateTime: string; toDateTime: string }
|
||||
| undefined
|
||||
const now = new Date()
|
||||
const window = cachedWindow ?? {
|
||||
fromDateTime: new Date(now.getTime() - lookbackDays * MS_PER_DAY).toISOString(),
|
||||
toDateTime: now.toISOString(),
|
||||
}
|
||||
if (syncContext && !cachedWindow) syncContext.gongDateWindow = window
|
||||
const { fromDateTime, toDateTime } = window
|
||||
|
||||
const filter: Record<string, unknown> = { fromDateTime, toDateTime }
|
||||
if (workspaceId) filter.workspaceId = workspaceId
|
||||
if (primaryUserIds) filter.primaryUserIds = primaryUserIds
|
||||
|
||||
logger.info('Listing Gong calls', {
|
||||
fromDateTime,
|
||||
toDateTime,
|
||||
hasCursor: Boolean(cursor),
|
||||
incremental: Boolean(lastSyncAt),
|
||||
})
|
||||
|
||||
const data = await fetchExtensiveCalls(accessToken, filter, cursor)
|
||||
const calls = data.calls ?? []
|
||||
const nextPageCursor = data.records?.cursor?.trim() || undefined
|
||||
|
||||
const allDocuments: ExternalDocument[] = []
|
||||
for (const call of calls) {
|
||||
const callId = call.metaData?.id
|
||||
if (!callId) continue
|
||||
const participants = buildParticipantNames(call.parties)
|
||||
allDocuments.push({
|
||||
externalId: callId,
|
||||
title: buildCallTitle(call.metaData),
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: call.metaData?.url || undefined,
|
||||
contentHash: buildContentHash(callId, call.metaData?.started),
|
||||
metadata: buildMetadata(call.metaData, participants),
|
||||
})
|
||||
}
|
||||
|
||||
const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
|
||||
let documents = allDocuments
|
||||
let capDroppedDocs = false
|
||||
if (maxCalls > 0) {
|
||||
const remaining = Math.max(0, maxCalls - prevFetched)
|
||||
if (allDocuments.length > remaining) {
|
||||
documents = allDocuments.slice(0, remaining)
|
||||
capDroppedDocs = true
|
||||
}
|
||||
}
|
||||
|
||||
const totalFetched = prevFetched + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
const hitLimit = maxCalls > 0 && totalFetched >= maxCalls
|
||||
const hasMore = !hitLimit && Boolean(nextPageCursor)
|
||||
|
||||
/**
|
||||
* Only flag the listing as capped when the `maxCalls` limit actually
|
||||
* truncated calls that still exist in the source — either by dropping calls
|
||||
* from the current page or by stopping while another page remains. Reaching
|
||||
* the limit exactly at source exhaustion (no dropped calls, no further
|
||||
* cursor) yields a complete listing, so deletion reconciliation must still
|
||||
* run for calls removed in Gong.
|
||||
*/
|
||||
if (syncContext && (capDroppedDocs || (hitLimit && Boolean(nextPageCursor)))) {
|
||||
syncContext.listingCapped = true
|
||||
}
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore ? nextPageCursor : undefined,
|
||||
hasMore,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
try {
|
||||
if (!externalId) return null
|
||||
|
||||
const workspaceId = (sourceConfig.workspaceId as string | undefined)?.trim()
|
||||
const filter: Record<string, unknown> = { callIds: [externalId] }
|
||||
if (workspaceId) filter.workspaceId = workspaceId
|
||||
|
||||
const callData = await fetchExtensiveCalls(accessToken, filter, undefined)
|
||||
const call = callData.calls?.[0]
|
||||
if (!call?.metaData?.id) {
|
||||
logger.warn('Gong call not found', { externalId })
|
||||
return null
|
||||
}
|
||||
|
||||
const metaData = call.metaData
|
||||
const participants = buildParticipantNames(call.parties)
|
||||
const speakerMap = buildSpeakerMap(call.parties)
|
||||
|
||||
const transcriptResponse = await fetchWithRetry(`${GONG_API_BASE}/calls/transcript`, {
|
||||
method: 'POST',
|
||||
headers: buildHeaders(accessToken),
|
||||
body: JSON.stringify({ filter: { callIds: [externalId] } }),
|
||||
})
|
||||
|
||||
if (!transcriptResponse.ok) {
|
||||
if (transcriptResponse.status === 404) return null
|
||||
throw new Error(`Failed to fetch Gong transcript: ${transcriptResponse.status}`)
|
||||
}
|
||||
|
||||
const transcriptData = (await transcriptResponse.json()) as GongTranscriptResponse
|
||||
const callTranscript = transcriptData.callTranscripts?.find(
|
||||
(entry) => entry.callId === externalId
|
||||
)
|
||||
const monologues = callTranscript?.transcript ?? []
|
||||
if (monologues.length === 0) {
|
||||
logger.info('Transcript not available for Gong call', { externalId })
|
||||
return null
|
||||
}
|
||||
|
||||
const hasSpokenText = monologues.some((monologue) =>
|
||||
(monologue.sentences ?? []).some((sentence) => Boolean(sentence.text?.trim()))
|
||||
)
|
||||
if (!hasSpokenText) return null
|
||||
|
||||
const content = formatTranscriptContent(metaData, participants, speakerMap, monologues)
|
||||
|
||||
return {
|
||||
externalId: metaData.id ?? externalId,
|
||||
title: buildCallTitle(metaData),
|
||||
content,
|
||||
contentDeferred: false,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: metaData.url || undefined,
|
||||
contentHash: buildContentHash(metaData.id ?? externalId, metaData.started),
|
||||
metadata: buildMetadata(metaData, participants),
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Failed to get Gong call transcript', {
|
||||
externalId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const maxCalls = sourceConfig.maxCalls as string | undefined
|
||||
if (maxCalls && (Number.isNaN(Number(maxCalls)) || Number(maxCalls) < 0)) {
|
||||
return { valid: false, error: 'Max calls must be a non-negative number' }
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetchWithRetry(
|
||||
`${GONG_API_BASE}/users`,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: buildHeaders(accessToken),
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
return {
|
||||
valid: false,
|
||||
error: `Gong access failed: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}`,
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.callTitle === 'string' && metadata.callTitle.trim()) {
|
||||
result.callTitle = metadata.callTitle
|
||||
}
|
||||
|
||||
const participants = Array.isArray(metadata.participants)
|
||||
? (metadata.participants as string[])
|
||||
: []
|
||||
if (participants.length > 0) {
|
||||
result.participants = participants.join(', ')
|
||||
}
|
||||
|
||||
if (metadata.duration != null) {
|
||||
const num = Number(metadata.duration)
|
||||
if (!Number.isNaN(num)) result.duration = num
|
||||
}
|
||||
|
||||
const callDate = parseTagDate(metadata.callDate)
|
||||
if (callDate) result.callDate = callDate
|
||||
|
||||
const scheduledDate = parseTagDate(metadata.scheduledDate)
|
||||
if (scheduledDate) result.scheduledDate = scheduledDate
|
||||
|
||||
const textTags = ['direction', 'scope', 'system', 'language', 'purpose'] as const
|
||||
for (const key of textTags) {
|
||||
const value = metadata[key]
|
||||
if (typeof value === 'string' && value.trim()) result[key] = value.trim()
|
||||
}
|
||||
|
||||
if (typeof metadata.isPrivate === 'boolean') result.isPrivate = metadata.isPrivate
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { gongConnector } from '@/connectors/gong/gong'
|
||||
@@ -0,0 +1,72 @@
|
||||
import { GongIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const gongConnectorMeta: ConnectorMeta = {
|
||||
id: 'gong',
|
||||
name: 'Gong',
|
||||
description: 'Sync call transcripts from Gong revenue intelligence',
|
||||
version: '1.0.0',
|
||||
icon: GongIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'apiKey',
|
||||
label: 'Access Key & Secret',
|
||||
placeholder: 'accessKey:accessKeySecret',
|
||||
},
|
||||
|
||||
supportsIncrementalSync: true,
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'lookback',
|
||||
title: 'Date Range',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
options: [
|
||||
{ label: 'Last 30 days', id: '30' },
|
||||
{ label: 'Last 90 days (recommended)', id: '90' },
|
||||
{ label: 'Last 6 months', id: '180' },
|
||||
],
|
||||
description:
|
||||
'On initial sync only. Controls how far back to look for calls with transcripts.',
|
||||
},
|
||||
{
|
||||
id: 'maxCalls',
|
||||
title: 'Max Calls',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 200 (default: unlimited)',
|
||||
},
|
||||
{
|
||||
id: 'workspaceId',
|
||||
title: 'Workspace ID',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'Optional — limit to a single Gong workspace',
|
||||
},
|
||||
{
|
||||
id: 'primaryUserIds',
|
||||
title: 'Host User IDs',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'Optional — comma-separated Gong user IDs (call hosts)',
|
||||
description:
|
||||
'Only sync calls hosted by these users. Find IDs in Gong under Company Settings → Users, or via the API.',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'callTitle', displayName: 'Call Title', fieldType: 'text' },
|
||||
{ id: 'participants', displayName: 'Participants', fieldType: 'text' },
|
||||
{ id: 'duration', displayName: 'Duration (seconds)', fieldType: 'number' },
|
||||
{ id: 'callDate', displayName: 'Call Date', fieldType: 'date' },
|
||||
{ id: 'scheduledDate', displayName: 'Scheduled Date', fieldType: 'date' },
|
||||
{ id: 'direction', displayName: 'Direction', fieldType: 'text' },
|
||||
{ id: 'scope', displayName: 'Scope', fieldType: 'text' },
|
||||
{ id: 'system', displayName: 'System', fieldType: 'text' },
|
||||
{ id: 'language', displayName: 'Language', fieldType: 'text' },
|
||||
{ id: 'purpose', displayName: 'Purpose', fieldType: 'text' },
|
||||
{ id: 'isPrivate', displayName: 'Private', fieldType: 'boolean' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,510 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { DEFAULT_MAX_EVENTS, googleCalendarConnectorMeta } from '@/connectors/google-calendar/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { parseMultiValue, parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('GoogleCalendarConnector')
|
||||
|
||||
const CALENDAR_API_BASE = 'https://www.googleapis.com/calendar/v3'
|
||||
const DEFAULT_RANGE_DAYS = 30
|
||||
const PAGE_SIZE = 250
|
||||
|
||||
interface CalendarEventTime {
|
||||
date?: string
|
||||
dateTime?: string
|
||||
timeZone?: string
|
||||
}
|
||||
|
||||
interface CalendarAttendee {
|
||||
email?: string
|
||||
displayName?: string
|
||||
responseStatus?: string
|
||||
self?: boolean
|
||||
resource?: boolean
|
||||
optional?: boolean
|
||||
}
|
||||
|
||||
interface CalendarEvent {
|
||||
id: string
|
||||
status?: string
|
||||
htmlLink?: string
|
||||
created?: string
|
||||
updated?: string
|
||||
summary?: string
|
||||
description?: string
|
||||
location?: string
|
||||
creator?: { email?: string; displayName?: string }
|
||||
organizer?: { email?: string; displayName?: string; self?: boolean }
|
||||
start?: CalendarEventTime
|
||||
end?: CalendarEventTime
|
||||
attendees?: CalendarAttendee[]
|
||||
recurringEventId?: string
|
||||
eventType?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a CalendarEventTime into a human-readable string.
|
||||
* All-day events use the date field; timed events use dateTime.
|
||||
*/
|
||||
function formatEventTime(eventTime?: CalendarEventTime): string {
|
||||
if (!eventTime) return 'Unknown'
|
||||
if (eventTime.dateTime) {
|
||||
const date = new Date(eventTime.dateTime)
|
||||
return date.toLocaleString('en-US', {
|
||||
weekday: 'long',
|
||||
year: 'numeric',
|
||||
month: 'long',
|
||||
day: 'numeric',
|
||||
hour: 'numeric',
|
||||
minute: '2-digit',
|
||||
timeZoneName: 'short',
|
||||
timeZone: eventTime.timeZone || undefined,
|
||||
})
|
||||
}
|
||||
if (eventTime.date) {
|
||||
const date = new Date(`${eventTime.date}T00:00:00`)
|
||||
return date.toLocaleDateString('en-US', {
|
||||
weekday: 'long',
|
||||
year: 'numeric',
|
||||
month: 'long',
|
||||
day: 'numeric',
|
||||
})
|
||||
}
|
||||
return 'Unknown'
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines whether the event is all-day based on whether `date` (not `dateTime`) is used.
|
||||
*/
|
||||
function isAllDayEvent(event: CalendarEvent): boolean {
|
||||
return Boolean(event.start?.date && !event.start?.dateTime)
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats attendees into a comma-separated list of names/emails.
|
||||
*/
|
||||
function formatAttendees(attendees?: CalendarAttendee[]): string {
|
||||
if (!attendees || attendees.length === 0) return ''
|
||||
return attendees
|
||||
.filter((a) => !a.resource)
|
||||
.map((a) => a.displayName || a.email || 'Unknown')
|
||||
.join(', ')
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats an organizer into a display string.
|
||||
*/
|
||||
function formatOrganizer(organizer?: { email?: string; displayName?: string }): string {
|
||||
if (!organizer) return ''
|
||||
if (organizer.displayName && organizer.email) {
|
||||
return `${organizer.displayName} (${organizer.email})`
|
||||
}
|
||||
return organizer.displayName || organizer.email || ''
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a readable content string from a calendar event.
|
||||
*/
|
||||
function eventToContent(event: CalendarEvent): string {
|
||||
const parts: string[] = []
|
||||
|
||||
parts.push(`Event: ${event.summary || 'Untitled Event'}`)
|
||||
|
||||
if (isAllDayEvent(event)) {
|
||||
parts.push(`Date: ${formatEventTime(event.start)} (All Day)`)
|
||||
} else {
|
||||
parts.push(`Date: ${formatEventTime(event.start)} - ${formatEventTime(event.end)}`)
|
||||
}
|
||||
|
||||
if (event.location) {
|
||||
parts.push(`Location: ${event.location}`)
|
||||
}
|
||||
|
||||
const organizer = formatOrganizer(event.organizer)
|
||||
if (organizer) {
|
||||
parts.push(`Organizer: ${organizer}`)
|
||||
}
|
||||
|
||||
const attendees = formatAttendees(event.attendees)
|
||||
if (attendees) {
|
||||
parts.push(`Attendees: ${attendees}`)
|
||||
}
|
||||
|
||||
if (event.description) {
|
||||
parts.push('')
|
||||
parts.push('Description:')
|
||||
parts.push(
|
||||
event.description
|
||||
.replace(/<[^>]*>/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
)
|
||||
}
|
||||
|
||||
return parts.join('\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the default time range boundaries: 30 days in the past to 30 days in the future.
|
||||
*/
|
||||
function getDefaultTimeRange(): { timeMin: string; timeMax: string } {
|
||||
const now = new Date()
|
||||
const past = new Date(now)
|
||||
past.setDate(past.getDate() - DEFAULT_RANGE_DAYS)
|
||||
const future = new Date(now)
|
||||
future.setDate(future.getDate() + DEFAULT_RANGE_DAYS)
|
||||
return {
|
||||
timeMin: past.toISOString(),
|
||||
timeMax: future.toISOString(),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the date range config into timeMin/timeMax values.
|
||||
*/
|
||||
function getTimeRange(sourceConfig: Record<string, unknown>): { timeMin: string; timeMax: string } {
|
||||
const dateRange = (sourceConfig.dateRange as string) || 'default'
|
||||
|
||||
const now = new Date()
|
||||
|
||||
switch (dateRange) {
|
||||
case 'past_only': {
|
||||
const past = new Date(now)
|
||||
past.setDate(past.getDate() - DEFAULT_RANGE_DAYS)
|
||||
return { timeMin: past.toISOString(), timeMax: now.toISOString() }
|
||||
}
|
||||
case 'future_only': {
|
||||
const future = new Date(now)
|
||||
future.setDate(future.getDate() + DEFAULT_RANGE_DAYS)
|
||||
return { timeMin: now.toISOString(), timeMax: future.toISOString() }
|
||||
}
|
||||
case 'past_90': {
|
||||
const past = new Date(now)
|
||||
past.setDate(past.getDate() - 90)
|
||||
const future = new Date(now)
|
||||
future.setDate(future.getDate() + 90)
|
||||
return { timeMin: past.toISOString(), timeMax: future.toISOString() }
|
||||
}
|
||||
default:
|
||||
return getDefaultTimeRange()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a CalendarEvent to an ExternalDocument.
|
||||
*
|
||||
* Backward compatibility: when only a single calendar is configured (the only
|
||||
* code path that existed before multi-calendar support), externalId and
|
||||
* contentHash use the legacy non-namespaced format so existing connectors see
|
||||
* zero churn on re-sync. When 2+ calendars are configured, we namespace by
|
||||
* calendarId because Google Calendar event IDs are only unique within a
|
||||
* single calendar.
|
||||
*/
|
||||
function eventToDocument(
|
||||
event: CalendarEvent,
|
||||
calendarId: string,
|
||||
isMultiCalendar: boolean
|
||||
): ExternalDocument | null {
|
||||
if (event.status === 'cancelled') return null
|
||||
|
||||
const content = eventToContent(event)
|
||||
if (!content.trim()) return null
|
||||
|
||||
const startTime = event.start?.dateTime || event.start?.date || ''
|
||||
const attendeeCount = event.attendees?.filter((a) => !a.resource).length || 0
|
||||
|
||||
const externalId = isMultiCalendar ? `${calendarId}:${event.id}` : event.id
|
||||
const contentHash = isMultiCalendar
|
||||
? `gcal:${calendarId}:${event.id}:${event.updated ?? ''}`
|
||||
: `gcal:${event.id}:${event.updated ?? ''}`
|
||||
|
||||
return {
|
||||
externalId,
|
||||
title: event.summary || 'Untitled Event',
|
||||
content,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: event.htmlLink || `https://calendar.google.com/calendar/event?eid=${event.id}`,
|
||||
contentHash,
|
||||
metadata: {
|
||||
calendarId,
|
||||
startTime,
|
||||
endTime: event.end?.dateTime || event.end?.date || '',
|
||||
location: event.location || '',
|
||||
organizer: formatOrganizer(event.organizer),
|
||||
attendeeCount,
|
||||
isAllDay: isAllDayEvent(event),
|
||||
eventDate: startTime,
|
||||
updatedTime: event.updated,
|
||||
createdTime: event.created,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export const googleCalendarConnector: ConnectorConfig = {
|
||||
...googleCalendarConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const parsedCalendarIds = parseMultiValue(sourceConfig.calendarId)
|
||||
const calendarIds = parsedCalendarIds.length > 0 ? parsedCalendarIds : ['primary']
|
||||
const { timeMin, timeMax } = getTimeRange(sourceConfig)
|
||||
const searchQuery = (sourceConfig.searchQuery as string) || ''
|
||||
|
||||
/**
|
||||
* Cursor format:
|
||||
* - For a single calendar with legacy cursors: the raw pageToken string
|
||||
* - For multi-calendar walking: JSON-encoded { calendarIndex, pageToken }
|
||||
*/
|
||||
let calendarIndex = 0
|
||||
let pageToken: string | undefined
|
||||
|
||||
if (cursor) {
|
||||
try {
|
||||
const parsed = JSON.parse(cursor) as { calendarIndex: number; pageToken?: string }
|
||||
if (typeof parsed.calendarIndex === 'number') {
|
||||
calendarIndex = parsed.calendarIndex
|
||||
pageToken = parsed.pageToken
|
||||
} else {
|
||||
pageToken = cursor
|
||||
}
|
||||
} catch {
|
||||
pageToken = cursor
|
||||
}
|
||||
}
|
||||
|
||||
if (calendarIndex >= calendarIds.length) {
|
||||
return { documents: [], hasMore: false }
|
||||
}
|
||||
|
||||
const calendarId = calendarIds[calendarIndex]
|
||||
|
||||
const queryParams = new URLSearchParams({
|
||||
singleEvents: 'true',
|
||||
orderBy: 'startTime',
|
||||
maxResults: String(PAGE_SIZE),
|
||||
timeMin,
|
||||
timeMax,
|
||||
})
|
||||
|
||||
if (searchQuery.trim()) {
|
||||
queryParams.set('q', searchQuery.trim())
|
||||
}
|
||||
|
||||
if (pageToken) {
|
||||
queryParams.set('pageToken', pageToken)
|
||||
}
|
||||
|
||||
const url = `${CALENDAR_API_BASE}/calendars/${encodeURIComponent(calendarId)}/events?${queryParams.toString()}`
|
||||
|
||||
logger.info('Listing Google Calendar events', {
|
||||
calendarId,
|
||||
calendarIndex,
|
||||
calendarCount: calendarIds.length,
|
||||
timeMin,
|
||||
timeMax,
|
||||
hasPageToken: Boolean(pageToken),
|
||||
})
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error('Failed to list Google Calendar events', {
|
||||
status: response.status,
|
||||
calendarId,
|
||||
error: errorText,
|
||||
})
|
||||
throw new Error(`Failed to list Google Calendar events: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const events = (data.items || []) as CalendarEvent[]
|
||||
|
||||
const isMultiCalendar = calendarIds.length > 1
|
||||
const documents: ExternalDocument[] = []
|
||||
for (const event of events) {
|
||||
const doc = eventToDocument(event, calendarId, isMultiCalendar)
|
||||
if (doc) documents.push(doc)
|
||||
}
|
||||
|
||||
const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
|
||||
const maxEvents = sourceConfig.maxEvents ? Number(sourceConfig.maxEvents) : DEFAULT_MAX_EVENTS
|
||||
const hitLimit = maxEvents > 0 && totalFetched >= maxEvents
|
||||
|
||||
const nextPageToken = data.nextPageToken as string | undefined
|
||||
|
||||
if (hitLimit) {
|
||||
return { documents, hasMore: false }
|
||||
}
|
||||
|
||||
if (nextPageToken) {
|
||||
return {
|
||||
documents,
|
||||
nextCursor: JSON.stringify({ calendarIndex, pageToken: nextPageToken }),
|
||||
hasMore: true,
|
||||
}
|
||||
}
|
||||
|
||||
const nextCalendarIndex = calendarIndex + 1
|
||||
if (nextCalendarIndex < calendarIds.length) {
|
||||
return {
|
||||
documents,
|
||||
nextCursor: JSON.stringify({ calendarIndex: nextCalendarIndex }),
|
||||
hasMore: true,
|
||||
}
|
||||
}
|
||||
|
||||
return { documents, hasMore: false }
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
/**
|
||||
* externalId format depends on connector configuration:
|
||||
* - Single-calendar (1 calendar configured): externalId = eventId (legacy
|
||||
* and current single-calendar format).
|
||||
* - Multi-calendar (2+ calendars configured): externalId =
|
||||
* `calendarId:eventId`. The first `:` is the separator — event IDs never
|
||||
* contain `:` while calendar IDs (e.g. `user@group.calendar.google.com`)
|
||||
* may include URL-safe chars but not `:`.
|
||||
*
|
||||
* Legacy in-flight rows that lack a separator fall back to the configured
|
||||
* calendar (or `primary`).
|
||||
*/
|
||||
const parsedCalendarIds = parseMultiValue(sourceConfig.calendarId)
|
||||
const calendarIds = parsedCalendarIds.length > 0 ? parsedCalendarIds : ['primary']
|
||||
|
||||
/**
|
||||
* Derive `isMultiCalendar` from the externalId itself, not from the current
|
||||
* config. If a row was synced under a multi-calendar config and the user
|
||||
* later removed calendars, the row's externalId still has the prefix —
|
||||
* returning a doc without the prefix would mint a duplicate via the sync
|
||||
* engine's externalId-keyed matching.
|
||||
*/
|
||||
const separatorIndex = externalId.indexOf(':')
|
||||
const isMultiCalendar = separatorIndex !== -1
|
||||
let calendarId: string
|
||||
let eventId: string
|
||||
if (separatorIndex === -1) {
|
||||
calendarId = calendarIds[0] ?? 'primary'
|
||||
eventId = externalId
|
||||
} else {
|
||||
calendarId = externalId.slice(0, separatorIndex)
|
||||
eventId = externalId.slice(separatorIndex + 1)
|
||||
}
|
||||
|
||||
const url = `${CALENDAR_API_BASE}/calendars/${encodeURIComponent(calendarId)}/events/${encodeURIComponent(eventId)}`
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) return null
|
||||
throw new Error(`Failed to get Google Calendar event: ${response.status}`)
|
||||
}
|
||||
|
||||
const event = (await response.json()) as CalendarEvent
|
||||
|
||||
if (event.status === 'cancelled') return null
|
||||
|
||||
return eventToDocument(event, calendarId, isMultiCalendar) ?? null
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const maxEvents = sourceConfig.maxEvents as string | undefined
|
||||
if (maxEvents && (Number.isNaN(Number(maxEvents)) || Number(maxEvents) <= 0)) {
|
||||
return { valid: false, error: 'Max events must be a positive number' }
|
||||
}
|
||||
|
||||
const parsedCalendarIds = parseMultiValue(sourceConfig.calendarId)
|
||||
const calendarIds = parsedCalendarIds.length > 0 ? parsedCalendarIds : ['primary']
|
||||
|
||||
try {
|
||||
for (const calendarId of calendarIds) {
|
||||
const url = `${CALENDAR_API_BASE}/calendars/${encodeURIComponent(calendarId)}/events?maxResults=1&singleEvents=true&orderBy=startTime&timeMin=${encodeURIComponent(new Date().toISOString())}`
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
url,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `Calendar not found: ${calendarId}. Check the calendar ID.`,
|
||||
}
|
||||
}
|
||||
return {
|
||||
valid: false,
|
||||
error: `Failed to access Google Calendar "${calendarId}": ${response.status}`,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.organizer === 'string' && metadata.organizer) {
|
||||
result.organizer = metadata.organizer
|
||||
}
|
||||
|
||||
if (typeof metadata.attendeeCount === 'number') {
|
||||
result.attendeeCount = metadata.attendeeCount
|
||||
}
|
||||
|
||||
if (typeof metadata.location === 'string' && metadata.location) {
|
||||
result.location = metadata.location
|
||||
}
|
||||
|
||||
const eventDate = parseTagDate(metadata.eventDate)
|
||||
if (eventDate) result.eventDate = eventDate
|
||||
|
||||
const lastModified = parseTagDate(metadata.updatedTime)
|
||||
if (lastModified) result.lastModified = lastModified
|
||||
|
||||
const createdAt = parseTagDate(metadata.createdTime)
|
||||
if (createdAt) result.createdAt = createdAt
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { googleCalendarConnector } from '@/connectors/google-calendar/google-calendar'
|
||||
@@ -0,0 +1,81 @@
|
||||
import { GoogleCalendarIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const DEFAULT_MAX_EVENTS = 500
|
||||
|
||||
export const googleCalendarConnectorMeta: ConnectorMeta = {
|
||||
id: 'google_calendar',
|
||||
name: 'Google Calendar',
|
||||
description: 'Sync calendar events from Google Calendar',
|
||||
version: '1.0.0',
|
||||
icon: GoogleCalendarIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'oauth',
|
||||
provider: 'google-calendar',
|
||||
requiredScopes: ['https://www.googleapis.com/auth/calendar'],
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'calendarSelector',
|
||||
title: 'Calendars',
|
||||
type: 'selector',
|
||||
selectorKey: 'google.calendar',
|
||||
canonicalParamId: 'calendarId',
|
||||
mode: 'basic',
|
||||
multi: true,
|
||||
placeholder: 'Select one or more calendars',
|
||||
required: false,
|
||||
description: 'Calendars to sync from. Defaults to your primary calendar.',
|
||||
},
|
||||
{
|
||||
id: 'calendarId',
|
||||
title: 'Calendar IDs',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'calendarId',
|
||||
mode: 'advanced',
|
||||
multi: true,
|
||||
placeholder: 'e.g. primary, team@group.calendar.google.com (comma-separated for multiple)',
|
||||
required: false,
|
||||
description:
|
||||
'Calendars to sync from. Use "primary" for your main calendar. Defaults to "primary".',
|
||||
},
|
||||
{
|
||||
id: 'dateRange',
|
||||
title: 'Date Range',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
options: [
|
||||
{ label: 'Last 30 days + next 30 days (default)', id: 'default' },
|
||||
{ label: 'Past events only (last 30 days)', id: 'past_only' },
|
||||
{ label: 'Future events only (next 30 days)', id: 'future_only' },
|
||||
{ label: 'Extended range (90 days each way)', id: 'past_90' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'searchQuery',
|
||||
title: 'Search Query',
|
||||
type: 'short-input',
|
||||
placeholder: 'e.g. standup, sprint review (optional)',
|
||||
required: false,
|
||||
description: 'Filter events by text search across all fields.',
|
||||
},
|
||||
{
|
||||
id: 'maxEvents',
|
||||
title: 'Max Events',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: `e.g. 500 (default: ${DEFAULT_MAX_EVENTS})`,
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'organizer', displayName: 'Organizer', fieldType: 'text' },
|
||||
{ id: 'attendeeCount', displayName: 'Attendee Count', fieldType: 'number' },
|
||||
{ id: 'location', displayName: 'Location', fieldType: 'text' },
|
||||
{ id: 'eventDate', displayName: 'Event Date', fieldType: 'date' },
|
||||
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
|
||||
{ id: 'createdAt', displayName: 'Created', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,382 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { googleDocsConnectorMeta } from '@/connectors/google-docs/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import {
|
||||
buildDriveParentsClause,
|
||||
joinTagArray,
|
||||
parseMultiValue,
|
||||
parseTagDate,
|
||||
} from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('GoogleDocsConnector')
|
||||
|
||||
/**
|
||||
* Represents a Google Drive file entry returned by the Drive API.
|
||||
*/
|
||||
interface DriveFile {
|
||||
id: string
|
||||
name: string
|
||||
mimeType: string
|
||||
modifiedTime?: string
|
||||
createdTime?: string
|
||||
webViewLink?: string
|
||||
owners?: { displayName?: string; emailAddress?: string }[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a structural element within a Google Docs document body.
|
||||
*/
|
||||
interface DocsStructuralElement {
|
||||
paragraph?: {
|
||||
paragraphStyle?: {
|
||||
namedStyleType?: string
|
||||
}
|
||||
elements?: {
|
||||
textRun?: {
|
||||
content?: string
|
||||
}
|
||||
}[]
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents the response from the Google Docs API for a single document.
|
||||
*/
|
||||
interface DocsDocument {
|
||||
documentId: string
|
||||
title: string
|
||||
body?: {
|
||||
content?: DocsStructuralElement[]
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps a Google Docs heading style to a Markdown heading prefix.
|
||||
*/
|
||||
function headingPrefix(namedStyleType?: string): string {
|
||||
switch (namedStyleType) {
|
||||
case 'HEADING_1':
|
||||
return '# '
|
||||
case 'HEADING_2':
|
||||
return '## '
|
||||
case 'HEADING_3':
|
||||
return '### '
|
||||
case 'HEADING_4':
|
||||
return '#### '
|
||||
case 'HEADING_5':
|
||||
return '##### '
|
||||
case 'HEADING_6':
|
||||
return '###### '
|
||||
default:
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts plain text from a Google Docs API structured document response.
|
||||
* Headings are prefixed with Markdown-style `#` markers.
|
||||
*/
|
||||
function extractTextFromDocsBody(doc: DocsDocument): string {
|
||||
const elements = doc.body?.content
|
||||
if (!elements) return ''
|
||||
|
||||
const parts: string[] = []
|
||||
|
||||
for (const element of elements) {
|
||||
const paragraph = element.paragraph
|
||||
if (!paragraph?.elements) continue
|
||||
|
||||
const prefix = headingPrefix(paragraph.paragraphStyle?.namedStyleType)
|
||||
/**
|
||||
* Each paragraph's final `textRun.content` already ends with `\n`. Strip
|
||||
* it before joining with `\n` so a heading followed by a body paragraph
|
||||
* is separated by a single newline, not two.
|
||||
*/
|
||||
const text = paragraph.elements
|
||||
.map((el) => el.textRun?.content ?? '')
|
||||
.join('')
|
||||
.replace(/\n+$/, '')
|
||||
|
||||
if (text.trim()) {
|
||||
parts.push(`${prefix}${text}`)
|
||||
}
|
||||
}
|
||||
|
||||
return parts.join('\n').trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the structured content of a Google Doc via the Docs API and
|
||||
* extracts it as plain text.
|
||||
*/
|
||||
async function fetchDocContent(accessToken: string, documentId: string): Promise<string> {
|
||||
const url = `https://docs.googleapis.com/v1/documents/${documentId}?fields=body.content`
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch Google Doc content ${documentId}: ${response.status}`)
|
||||
}
|
||||
|
||||
const doc = (await response.json()) as DocsDocument
|
||||
return extractTextFromDocsBody(doc)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a lightweight stub from a Drive file entry. Content is deferred
|
||||
* and only fetched via getDocument for new or changed documents.
|
||||
*/
|
||||
function fileToStub(file: DriveFile): ExternalDocument {
|
||||
return {
|
||||
externalId: file.id,
|
||||
title: file.name || 'Untitled',
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: file.webViewLink || `https://docs.google.com/document/d/${file.id}/edit`,
|
||||
contentHash: `gdocs:${file.id}:${file.modifiedTime ?? ''}`,
|
||||
metadata: {
|
||||
modifiedTime: file.modifiedTime,
|
||||
createdTime: file.createdTime,
|
||||
owners: file.owners?.map((o) => o.displayName || o.emailAddress).filter(Boolean),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the Drive API query string for listing Google Docs.
|
||||
*/
|
||||
function buildQuery(sourceConfig: Record<string, unknown>): string {
|
||||
const parts: string[] = ['trashed = false', "mimeType = 'application/vnd.google-apps.document'"]
|
||||
|
||||
const parentsClause = buildDriveParentsClause(parseMultiValue(sourceConfig.folderId))
|
||||
if (parentsClause) parts.push(parentsClause)
|
||||
|
||||
return parts.join(' and ')
|
||||
}
|
||||
|
||||
export const googleDocsConnector: ConnectorConfig = {
|
||||
...googleDocsConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const query = buildQuery(sourceConfig)
|
||||
const pageSize = 100
|
||||
|
||||
const queryParams = new URLSearchParams({
|
||||
q: query,
|
||||
pageSize: String(pageSize),
|
||||
fields: 'nextPageToken,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners)',
|
||||
supportsAllDrives: 'true',
|
||||
includeItemsFromAllDrives: 'true',
|
||||
})
|
||||
|
||||
if (cursor) {
|
||||
queryParams.set('pageToken', cursor)
|
||||
}
|
||||
|
||||
const url = `https://www.googleapis.com/drive/v3/files?${queryParams.toString()}`
|
||||
|
||||
logger.info('Listing Google Docs', { query, cursor: cursor ?? 'initial' })
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error('Failed to list Google Docs', {
|
||||
status: response.status,
|
||||
error: errorText,
|
||||
})
|
||||
throw new Error(`Failed to list Google Docs: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const files = (data.files || []) as DriveFile[]
|
||||
|
||||
/**
|
||||
* Drive sets `incompleteSearch` when it could not search every corpus (it
|
||||
* arises with the `allDrives` scope enabled by `includeItemsFromAllDrives`).
|
||||
* A partial listing drops still-existing docs, so reconciliation must be
|
||||
* suppressed to avoid hard-deleting valid documents.
|
||||
*/
|
||||
const incompleteSearch = data.incompleteSearch === true
|
||||
|
||||
const maxDocs = sourceConfig.maxDocs ? Number(sourceConfig.maxDocs) : 0
|
||||
const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
|
||||
|
||||
let documents = files.map(fileToStub)
|
||||
let slicedSome = false
|
||||
if (maxDocs > 0) {
|
||||
const remaining = maxDocs - previouslyFetched
|
||||
if (documents.length > remaining) {
|
||||
slicedSome = true
|
||||
documents = documents.slice(0, remaining)
|
||||
}
|
||||
}
|
||||
|
||||
const totalFetched = previouslyFetched + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
const hitLimit = maxDocs > 0 && totalFetched >= maxDocs
|
||||
|
||||
const nextPageToken = data.nextPageToken as string | undefined
|
||||
|
||||
/**
|
||||
* Mark the listing as incomplete so the sync engine skips deletion
|
||||
* reconciliation when this page does not represent the full source set:
|
||||
* - `slicedSome`: the page held more docs than the `maxDocs` cap allowed.
|
||||
* - `hitLimit` with a next page: the cap was reached while more pages remain.
|
||||
* - `incompleteSearch`: Drive could not search every corpus, so the page is
|
||||
* partial and may omit still-existing docs.
|
||||
* Reconciliation against any of these would hard-delete valid documents.
|
||||
*/
|
||||
if (syncContext && (slicedSome || (hitLimit && Boolean(nextPageToken)) || incompleteSearch)) {
|
||||
syncContext.listingCapped = true
|
||||
}
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hitLimit ? undefined : nextPageToken,
|
||||
hasMore: hitLimit ? false : Boolean(nextPageToken),
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
_sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
const fields = 'id,name,mimeType,modifiedTime,createdTime,webViewLink,owners,trashed'
|
||||
const url = `https://www.googleapis.com/drive/v3/files/${externalId}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true`
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) return null
|
||||
throw new Error(`Failed to get Google Doc metadata: ${response.status}`)
|
||||
}
|
||||
|
||||
const file = (await response.json()) as DriveFile & { trashed?: boolean }
|
||||
|
||||
if (file.trashed) return null
|
||||
if (file.mimeType !== 'application/vnd.google-apps.document') return null
|
||||
|
||||
try {
|
||||
const content = await fetchDocContent(accessToken, file.id)
|
||||
if (!content.trim()) return null
|
||||
|
||||
return { ...fileToStub(file), content, contentDeferred: false }
|
||||
} catch (error) {
|
||||
logger.warn(`Failed to extract content from document: ${file.name} (${file.id})`, {
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const folderIds = parseMultiValue(sourceConfig.folderId)
|
||||
const maxDocs = sourceConfig.maxDocs as string | undefined
|
||||
|
||||
if (maxDocs && (Number.isNaN(Number(maxDocs)) || Number(maxDocs) <= 0)) {
|
||||
return { valid: false, error: 'Max documents must be a positive number' }
|
||||
}
|
||||
|
||||
try {
|
||||
if (folderIds.length > 0) {
|
||||
for (const folderId of folderIds) {
|
||||
const url = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(folderId)}?fields=id,name,mimeType&supportsAllDrives=true`
|
||||
const response = await fetchWithRetry(
|
||||
url,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `Folder "${folderId}" not found. Check the folder ID and permissions.`,
|
||||
}
|
||||
}
|
||||
return {
|
||||
valid: false,
|
||||
error: `Failed to access folder "${folderId}": ${response.status}`,
|
||||
}
|
||||
}
|
||||
|
||||
const folder = await response.json()
|
||||
if (folder.mimeType !== 'application/vnd.google-apps.folder') {
|
||||
return { valid: false, error: `"${folderId}" is not a folder` }
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const url =
|
||||
"https://www.googleapis.com/drive/v3/files?pageSize=1&q=mimeType%3D'application%2Fvnd.google-apps.document'&fields=files(id)"
|
||||
const response = await fetchWithRetry(
|
||||
url,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
return { valid: false, error: `Failed to access Google Docs: ${response.status}` }
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
return { valid: false, error: toError(error).message || 'Failed to validate configuration' }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
const owners = joinTagArray(metadata.owners)
|
||||
if (owners) result.owners = owners
|
||||
|
||||
const lastModified = parseTagDate(metadata.modifiedTime)
|
||||
if (lastModified) result.lastModified = lastModified
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { googleDocsConnector } from '@/connectors/google-docs/google-docs'
|
||||
@@ -0,0 +1,53 @@
|
||||
import { GoogleDocsIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const googleDocsConnectorMeta: ConnectorMeta = {
|
||||
id: 'google_docs',
|
||||
name: 'Google Docs',
|
||||
description: 'Sync Google Docs documents',
|
||||
version: '1.0.0',
|
||||
icon: GoogleDocsIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'oauth',
|
||||
provider: 'google-docs',
|
||||
requiredScopes: ['https://www.googleapis.com/auth/drive'],
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'folderSelector',
|
||||
title: 'Folders',
|
||||
type: 'selector',
|
||||
selectorKey: 'google.drive',
|
||||
mimeType: 'application/vnd.google-apps.folder',
|
||||
canonicalParamId: 'folderId',
|
||||
mode: 'basic',
|
||||
multi: true,
|
||||
placeholder: 'Select one or more folders (optional)',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'folderId',
|
||||
title: 'Folder IDs',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'folderId',
|
||||
mode: 'advanced',
|
||||
multi: true,
|
||||
placeholder: 'e.g. 1aBcDeFg…, 2cDeFgHi… (comma-separated for multiple)',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'maxDocs',
|
||||
title: 'Max Documents',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 500 (default: unlimited)',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'owners', displayName: 'Owner', fieldType: 'text' },
|
||||
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,421 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { googleDriveConnectorMeta } from '@/connectors/google-drive/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import {
|
||||
buildDriveParentsClause,
|
||||
CONNECTOR_MAX_FILE_BYTES,
|
||||
ConnectorFileTooLargeError,
|
||||
htmlToPlainText,
|
||||
joinTagArray,
|
||||
markSkipped,
|
||||
parseMultiValue,
|
||||
parseTagDate,
|
||||
readBodyWithLimit,
|
||||
sizeLimitSkipReason,
|
||||
} from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('GoogleDriveConnector')
|
||||
|
||||
const GOOGLE_WORKSPACE_MIME_TYPES: Record<string, string> = {
|
||||
'application/vnd.google-apps.document': 'text/plain',
|
||||
'application/vnd.google-apps.spreadsheet': 'text/csv',
|
||||
'application/vnd.google-apps.presentation': 'text/plain',
|
||||
}
|
||||
|
||||
const SUPPORTED_TEXT_MIME_TYPES = [
|
||||
'text/plain',
|
||||
'text/csv',
|
||||
'text/html',
|
||||
'text/markdown',
|
||||
'application/json',
|
||||
'application/xml',
|
||||
]
|
||||
|
||||
// Google Drive's `files.export` API rejects exports over 10 MB (exportSizeLimitExceeded),
|
||||
// so this is a hard external limit for Google Workspace docs — not the connector cap.
|
||||
const MAX_EXPORT_SIZE = 10 * 1024 * 1024
|
||||
|
||||
function isGoogleWorkspaceFile(mimeType: string): boolean {
|
||||
return mimeType in GOOGLE_WORKSPACE_MIME_TYPES
|
||||
}
|
||||
|
||||
function isSupportedTextFile(mimeType: string): boolean {
|
||||
return SUPPORTED_TEXT_MIME_TYPES.some((t) => mimeType.startsWith(t))
|
||||
}
|
||||
|
||||
async function exportGoogleWorkspaceFile(
|
||||
accessToken: string,
|
||||
fileId: string,
|
||||
sourceMimeType: string
|
||||
): Promise<string> {
|
||||
const exportMimeType = GOOGLE_WORKSPACE_MIME_TYPES[sourceMimeType]
|
||||
if (!exportMimeType) {
|
||||
throw new Error(`Unsupported Google Workspace MIME type: ${sourceMimeType}`)
|
||||
}
|
||||
|
||||
const url = `https://www.googleapis.com/drive/v3/files/${fileId}/export?mimeType=${encodeURIComponent(exportMimeType)}`
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: { Authorization: `Bearer ${accessToken}` },
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
// Google rejects exports over its 10 MB limit with a 403 exportSizeLimitExceeded
|
||||
// before streaming any bytes — surface that as an oversize skip, not a hard error.
|
||||
if (response.status === 403) {
|
||||
const body = await response.text().catch(() => '')
|
||||
if (body.includes('exportSizeLimitExceeded')) {
|
||||
throw new ConnectorFileTooLargeError(MAX_EXPORT_SIZE)
|
||||
}
|
||||
}
|
||||
throw new Error(`Failed to export file ${fileId}: ${response.status}`)
|
||||
}
|
||||
|
||||
const buffer = await readBodyWithLimit(response, MAX_EXPORT_SIZE)
|
||||
if (!buffer) {
|
||||
throw new ConnectorFileTooLargeError(MAX_EXPORT_SIZE)
|
||||
}
|
||||
return buffer.toString('utf8')
|
||||
}
|
||||
|
||||
async function downloadTextFile(accessToken: string, fileId: string): Promise<string> {
|
||||
const url = `https://www.googleapis.com/drive/v3/files/${fileId}?alt=media`
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: { Authorization: `Bearer ${accessToken}` },
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to download file ${fileId}: ${response.status}`)
|
||||
}
|
||||
|
||||
// Stream with a hard byte cap so a file with missing/under-reported listing
|
||||
// size metadata is never fully buffered into memory. Oversized files raise
|
||||
// DriveFileTooLargeError so getDocument can surface them as skipped (failed) rows.
|
||||
const buffer = await readBodyWithLimit(response, CONNECTOR_MAX_FILE_BYTES)
|
||||
if (!buffer) {
|
||||
throw new ConnectorFileTooLargeError(CONNECTOR_MAX_FILE_BYTES)
|
||||
}
|
||||
return buffer.toString('utf8')
|
||||
}
|
||||
|
||||
async function fetchFileContent(
|
||||
accessToken: string,
|
||||
fileId: string,
|
||||
mimeType: string
|
||||
): Promise<string> {
|
||||
if (isGoogleWorkspaceFile(mimeType)) {
|
||||
return exportGoogleWorkspaceFile(accessToken, fileId, mimeType)
|
||||
}
|
||||
if (mimeType === 'text/html') {
|
||||
const html = await downloadTextFile(accessToken, fileId)
|
||||
return htmlToPlainText(html)
|
||||
}
|
||||
if (isSupportedTextFile(mimeType)) {
|
||||
return downloadTextFile(accessToken, fileId)
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported MIME type for content extraction: ${mimeType}`)
|
||||
}
|
||||
|
||||
interface DriveFile {
|
||||
id: string
|
||||
name: string
|
||||
mimeType: string
|
||||
modifiedTime?: string
|
||||
createdTime?: string
|
||||
webViewLink?: string
|
||||
parents?: string[]
|
||||
owners?: { displayName?: string; emailAddress?: string }[]
|
||||
size?: string
|
||||
starred?: boolean
|
||||
trashed?: boolean
|
||||
}
|
||||
|
||||
function buildQuery(sourceConfig: Record<string, unknown>): string {
|
||||
const parts: string[] = ['trashed = false']
|
||||
|
||||
const parentsClause = buildDriveParentsClause(parseMultiValue(sourceConfig.folderId))
|
||||
if (parentsClause) parts.push(parentsClause)
|
||||
|
||||
const fileType = (sourceConfig.fileType as string) || 'all'
|
||||
switch (fileType) {
|
||||
case 'documents':
|
||||
parts.push("mimeType = 'application/vnd.google-apps.document'")
|
||||
break
|
||||
case 'spreadsheets':
|
||||
parts.push("mimeType = 'application/vnd.google-apps.spreadsheet'")
|
||||
break
|
||||
case 'presentations':
|
||||
parts.push("mimeType = 'application/vnd.google-apps.presentation'")
|
||||
break
|
||||
case 'text':
|
||||
parts.push(`(${SUPPORTED_TEXT_MIME_TYPES.map((t) => `mimeType = '${t}'`).join(' or ')})`)
|
||||
break
|
||||
default: {
|
||||
// Include Google Workspace files + plain text files, exclude folders
|
||||
const allMimeTypes = [
|
||||
...Object.keys(GOOGLE_WORKSPACE_MIME_TYPES),
|
||||
...SUPPORTED_TEXT_MIME_TYPES,
|
||||
]
|
||||
parts.push(`(${allMimeTypes.map((t) => `mimeType = '${t}'`).join(' or ')})`)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return parts.join(' and ')
|
||||
}
|
||||
|
||||
function fileToStub(file: DriveFile): ExternalDocument {
|
||||
return {
|
||||
externalId: file.id,
|
||||
title: file.name || 'Untitled',
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: file.webViewLink || `https://drive.google.com/file/d/${file.id}/view`,
|
||||
contentHash: `gdrive:${file.id}:${file.modifiedTime ?? ''}`,
|
||||
metadata: {
|
||||
originalMimeType: file.mimeType,
|
||||
modifiedTime: file.modifiedTime,
|
||||
createdTime: file.createdTime,
|
||||
owners: file.owners?.map((o) => o.displayName || o.emailAddress).filter(Boolean),
|
||||
starred: file.starred,
|
||||
fileSize: file.size ? Number(file.size) : undefined,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export const googleDriveConnector: ConnectorConfig = {
|
||||
...googleDriveConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const query = buildQuery(sourceConfig)
|
||||
const pageSize = 100
|
||||
|
||||
const maxFiles = sourceConfig.maxFiles ? Number(sourceConfig.maxFiles) : 0
|
||||
const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
|
||||
|
||||
if (maxFiles > 0 && previouslyFetched >= maxFiles) {
|
||||
return { documents: [], hasMore: false }
|
||||
}
|
||||
|
||||
const remaining = maxFiles > 0 ? maxFiles - previouslyFetched : 0
|
||||
const effectivePageSize = maxFiles > 0 ? Math.min(pageSize, remaining) : pageSize
|
||||
|
||||
const queryParams = new URLSearchParams({
|
||||
q: query,
|
||||
pageSize: String(effectivePageSize),
|
||||
orderBy: 'modifiedTime desc',
|
||||
fields:
|
||||
'nextPageToken,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,parents,owners,size,starred)',
|
||||
supportsAllDrives: 'true',
|
||||
includeItemsFromAllDrives: 'true',
|
||||
})
|
||||
|
||||
if (cursor) {
|
||||
queryParams.set('pageToken', cursor)
|
||||
}
|
||||
|
||||
const url = `https://www.googleapis.com/drive/v3/files?${queryParams.toString()}`
|
||||
|
||||
logger.info('Listing Google Drive files', { query, cursor: cursor ?? 'initial' })
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error('Failed to list Google Drive files', {
|
||||
status: response.status,
|
||||
error: errorText,
|
||||
})
|
||||
throw new Error(`Failed to list Google Drive files: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const files = (data.files || []) as DriveFile[]
|
||||
|
||||
/**
|
||||
* Drive sets `incompleteSearch` when it could not search every corpus (it
|
||||
* arises with the `allDrives` scope enabled by `includeItemsFromAllDrives`).
|
||||
* A partial listing drops still-existing files, so reconciliation must be
|
||||
* suppressed to avoid hard-deleting valid documents.
|
||||
*/
|
||||
const incompleteSearch = data.incompleteSearch === true
|
||||
|
||||
const documents = files
|
||||
.filter((f) => isGoogleWorkspaceFile(f.mimeType) || isSupportedTextFile(f.mimeType))
|
||||
.map(fileToStub)
|
||||
|
||||
const totalFetched = previouslyFetched + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
const hitLimit = maxFiles > 0 && totalFetched >= maxFiles
|
||||
if (syncContext && (hitLimit || incompleteSearch)) syncContext.listingCapped = true
|
||||
|
||||
const nextPageToken = data.nextPageToken as string | undefined
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hitLimit ? undefined : nextPageToken,
|
||||
hasMore: hitLimit ? false : Boolean(nextPageToken),
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
const fields =
|
||||
'id,name,mimeType,modifiedTime,createdTime,webViewLink,parents,owners,size,starred,trashed'
|
||||
const url = `https://www.googleapis.com/drive/v3/files/${externalId}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true`
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) return null
|
||||
throw new Error(`Failed to get Google Drive file: ${response.status}`)
|
||||
}
|
||||
|
||||
const file = (await response.json()) as DriveFile
|
||||
|
||||
if (file.trashed) return null
|
||||
|
||||
try {
|
||||
const content = await fetchFileContent(accessToken, file.id, file.mimeType)
|
||||
if (!content.trim()) return null
|
||||
|
||||
const stub = fileToStub(file)
|
||||
return { ...stub, content, contentDeferred: false }
|
||||
} catch (error) {
|
||||
if (error instanceof ConnectorFileTooLargeError) {
|
||||
logger.info('Skipping oversized Google Drive file', { fileId: file.id, name: file.name })
|
||||
return markSkipped(fileToStub(file), sizeLimitSkipReason(error.limitBytes))
|
||||
}
|
||||
logger.warn(`Failed to fetch content for file: ${file.name} (${file.id})`, {
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const folderIds = parseMultiValue(sourceConfig.folderId)
|
||||
const maxFiles = sourceConfig.maxFiles as string | undefined
|
||||
|
||||
if (maxFiles && (Number.isNaN(Number(maxFiles)) || Number(maxFiles) <= 0)) {
|
||||
return { valid: false, error: 'Max files must be a positive number' }
|
||||
}
|
||||
|
||||
// Verify access to Drive API
|
||||
try {
|
||||
if (folderIds.length > 0) {
|
||||
// Verify each folder exists, is accessible, and is actually a folder
|
||||
for (const folderId of folderIds) {
|
||||
const url = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(folderId)}?fields=id,name,mimeType&supportsAllDrives=true`
|
||||
const response = await fetchWithRetry(
|
||||
url,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `Folder "${folderId}" not found. Check the folder ID and permissions.`,
|
||||
}
|
||||
}
|
||||
return {
|
||||
valid: false,
|
||||
error: `Failed to access folder "${folderId}": ${response.status}`,
|
||||
}
|
||||
}
|
||||
|
||||
const folder = await response.json()
|
||||
if (folder.mimeType !== 'application/vnd.google-apps.folder') {
|
||||
return { valid: false, error: `"${folderId}" is not a folder` }
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Verify basic Drive access by listing one file
|
||||
const url = 'https://www.googleapis.com/drive/v3/files?pageSize=1&fields=files(id)'
|
||||
const response = await fetchWithRetry(
|
||||
url,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
return { valid: false, error: `Failed to access Google Drive: ${response.status}` }
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
const owners = joinTagArray(metadata.owners)
|
||||
if (owners) result.owners = owners
|
||||
|
||||
if (typeof metadata.originalMimeType === 'string') {
|
||||
const mimeType = metadata.originalMimeType
|
||||
if (mimeType.includes('document')) result.fileType = 'Google Doc'
|
||||
else if (mimeType.includes('spreadsheet')) result.fileType = 'Google Sheet'
|
||||
else if (mimeType.includes('presentation')) result.fileType = 'Google Slides'
|
||||
else if (mimeType.startsWith('text/')) result.fileType = 'Text File'
|
||||
else result.fileType = mimeType
|
||||
}
|
||||
|
||||
const lastModified = parseTagDate(metadata.modifiedTime)
|
||||
if (lastModified) result.lastModified = lastModified
|
||||
|
||||
if (typeof metadata.starred === 'boolean') {
|
||||
result.starred = metadata.starred
|
||||
}
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { googleDriveConnector } from '@/connectors/google-drive/google-drive'
|
||||
@@ -0,0 +1,68 @@
|
||||
import { GoogleDriveIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const googleDriveConnectorMeta: ConnectorMeta = {
|
||||
id: 'google_drive',
|
||||
name: 'Google Drive',
|
||||
description: 'Sync documents from Google Drive',
|
||||
version: '1.0.0',
|
||||
icon: GoogleDriveIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'oauth',
|
||||
provider: 'google-drive',
|
||||
requiredScopes: ['https://www.googleapis.com/auth/drive'],
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'folderSelector',
|
||||
title: 'Folders',
|
||||
type: 'selector',
|
||||
selectorKey: 'google.drive',
|
||||
mimeType: 'application/vnd.google-apps.folder',
|
||||
canonicalParamId: 'folderId',
|
||||
mode: 'basic',
|
||||
multi: true,
|
||||
placeholder: 'Select one or more folders (optional)',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'folderId',
|
||||
title: 'Folder IDs',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'folderId',
|
||||
mode: 'advanced',
|
||||
multi: true,
|
||||
placeholder: 'e.g. 1aBcDeFg…, 2cDeFgHi… (comma-separated for multiple)',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'fileType',
|
||||
title: 'File Type',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
options: [
|
||||
{ label: 'All supported files', id: 'all' },
|
||||
{ label: 'Google Docs only', id: 'documents' },
|
||||
{ label: 'Google Sheets only', id: 'spreadsheets' },
|
||||
{ label: 'Google Slides only', id: 'presentations' },
|
||||
{ label: 'Plain text files only', id: 'text' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'maxFiles',
|
||||
title: 'Max Files',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 500 (default: unlimited)',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'owners', displayName: 'Owner', fieldType: 'text' },
|
||||
{ id: 'fileType', displayName: 'File Type', fieldType: 'text' },
|
||||
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
|
||||
{ id: 'starred', displayName: 'Starred', fieldType: 'boolean' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,765 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { googleFormsConnectorMeta, MAX_RESPONSES_PER_FORM } from '@/connectors/google-forms/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import {
|
||||
buildDriveParentsClause,
|
||||
joinTagArray,
|
||||
parseMultiValue,
|
||||
parseTagDate,
|
||||
} from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('GoogleFormsConnector')
|
||||
|
||||
const DRIVE_API_BASE = 'https://www.googleapis.com/drive/v3'
|
||||
const FORMS_API_BASE = 'https://forms.googleapis.com/v1'
|
||||
const FORM_MIME_TYPE = 'application/vnd.google-apps.form'
|
||||
const FOLDER_MIME_TYPE = 'application/vnd.google-apps.folder'
|
||||
|
||||
/**
|
||||
* Drive API page size when listing forms. The Drive API caps pageSize at 100.
|
||||
*/
|
||||
const DRIVE_PAGE_SIZE = 100
|
||||
|
||||
/**
|
||||
* Maximum responses returned per Forms API page (API caps and defaults to 5000).
|
||||
*/
|
||||
const RESPONSES_PAGE_SIZE = 5000
|
||||
|
||||
/**
|
||||
* Number of forms whose change indicators are fetched concurrently during
|
||||
* listing. Keeps the Forms API call volume bounded while still parallelizing.
|
||||
*/
|
||||
const LIST_CONCURRENCY = 4
|
||||
|
||||
/**
|
||||
* Content scope for a form document. `both` indexes the form's questions and its
|
||||
* submitted responses; `structure` indexes only the questions (no response reads,
|
||||
* so the responses scope is never exercised for that connector instance).
|
||||
*/
|
||||
type ContentScope = 'both' | 'structure'
|
||||
|
||||
/**
|
||||
* Resolves the content scope from sourceConfig, defaulting to `both`.
|
||||
*/
|
||||
function resolveContentScope(value: unknown): ContentScope {
|
||||
return value === 'structure' ? 'structure' : 'both'
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a Google Drive file entry for a form, returned by the Drive API.
|
||||
*/
|
||||
interface DriveFormFile {
|
||||
id: string
|
||||
name: string
|
||||
mimeType: string
|
||||
modifiedTime?: string
|
||||
createdTime?: string
|
||||
webViewLink?: string
|
||||
owners?: { displayName?: string; emailAddress?: string }[]
|
||||
trashed?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* A single answer entry inside a response answer container.
|
||||
*/
|
||||
interface FormTextAnswer {
|
||||
value?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* A single question's answers within a form response. The Forms API keys the
|
||||
* `answers` map by questionId and stores text values under
|
||||
* `textAnswers.answers[].value`.
|
||||
*/
|
||||
interface FormAnswer {
|
||||
questionId?: string
|
||||
textAnswers?: { answers?: FormTextAnswer[] }
|
||||
}
|
||||
|
||||
/**
|
||||
* A single submitted response to a form.
|
||||
*/
|
||||
interface FormResponse {
|
||||
responseId?: string
|
||||
createTime?: string
|
||||
lastSubmittedTime?: string
|
||||
respondentEmail?: string
|
||||
answers?: Record<string, FormAnswer>
|
||||
}
|
||||
|
||||
/**
|
||||
* Paginated response list from the Forms API.
|
||||
*/
|
||||
interface FormResponseList {
|
||||
responses?: FormResponse[]
|
||||
nextPageToken?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* A question item within a form's structure.
|
||||
*/
|
||||
interface FormQuestionItem {
|
||||
question?: {
|
||||
questionId?: string
|
||||
required?: boolean
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A single structural item within a form (question, section, image, etc.).
|
||||
*/
|
||||
interface FormItem {
|
||||
itemId?: string
|
||||
title?: string
|
||||
description?: string
|
||||
questionItem?: FormQuestionItem
|
||||
}
|
||||
|
||||
/**
|
||||
* The form structure returned by the Forms API `forms.get` endpoint.
|
||||
*/
|
||||
interface FormStructure {
|
||||
formId?: string
|
||||
info?: {
|
||||
title?: string
|
||||
description?: string
|
||||
documentTitle?: string
|
||||
}
|
||||
items?: FormItem[]
|
||||
revisionId?: string
|
||||
responderUri?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Lightweight metadata captured during listing, sufficient to build a stub
|
||||
* and detect changes without downloading the full form content.
|
||||
*/
|
||||
interface FormStubInput {
|
||||
file: DriveFormFile
|
||||
formTitle?: string
|
||||
revisionId?: string
|
||||
latestResponseTime?: string
|
||||
contentScope: ContentScope
|
||||
responseCap: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the effective per-form response cap applied when rendering content:
|
||||
* the user-configured `maxResponsesPerForm` clamped to the hard
|
||||
* `MAX_RESPONSES_PER_FORM` ceiling. Part of the content hash so changing the
|
||||
* cap re-syncs every form (the rendered content depends on it).
|
||||
*/
|
||||
function resolveResponseCap(sourceConfig: Record<string, unknown>): number {
|
||||
const configured = parsePositiveInt(sourceConfig.maxResponsesPerForm)
|
||||
return configured > 0 ? Math.min(configured, MAX_RESPONSES_PER_FORM) : MAX_RESPONSES_PER_FORM
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses an optional positive-integer config value, returning 0 when unset/invalid.
|
||||
*/
|
||||
function parsePositiveInt(value: unknown): number {
|
||||
if (value == null || value === '') return 0
|
||||
const num = Number(value)
|
||||
return Number.isNaN(num) || num <= 0 ? 0 : Math.floor(num)
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps a small array over an async worker with a bounded concurrency, preserving
|
||||
* input order in the returned results.
|
||||
*/
|
||||
async function mapWithConcurrency<T, R>(
|
||||
items: T[],
|
||||
limit: number,
|
||||
worker: (item: T, index: number) => Promise<R>
|
||||
): Promise<R[]> {
|
||||
const results = new Array<R>(items.length)
|
||||
let next = 0
|
||||
|
||||
async function run(): Promise<void> {
|
||||
while (next < items.length) {
|
||||
const current = next++
|
||||
results[current] = await worker(items[current], current)
|
||||
}
|
||||
}
|
||||
|
||||
const runners = Array.from({ length: Math.min(limit, items.length) }, run)
|
||||
await Promise.all(runners)
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the form structure via the Forms API. Returns null on 404 (form
|
||||
* deleted or inaccessible).
|
||||
*/
|
||||
async function fetchFormStructure(
|
||||
accessToken: string,
|
||||
formId: string
|
||||
): Promise<FormStructure | null> {
|
||||
const url = `${FORMS_API_BASE}/forms/${encodeURIComponent(formId)}`
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) return null
|
||||
throw new Error(`Failed to fetch form structure ${formId}: ${response.status}`)
|
||||
}
|
||||
|
||||
return (await response.json()) as FormStructure
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of fetching a form's responses: the collected responses (capped at
|
||||
* `MAX_RESPONSES_PER_FORM` for rendering) plus the greatest submission timestamp
|
||||
* across ALL response pages.
|
||||
*
|
||||
* `latestSubmittedTime` is tracked separately from the capped `responses` so the
|
||||
* content hash computed in getDocument stays identical to the one computed during
|
||||
* listing, which scans the same full set via `fetchLatestResponseTime`. If it
|
||||
* were derived from the capped slice alone, a form with more than
|
||||
* `MAX_RESPONSES_PER_FORM` responses could hash differently between the two paths
|
||||
* and re-sync on every run.
|
||||
*/
|
||||
interface FetchedResponses {
|
||||
responses: FormResponse[]
|
||||
latestSubmittedTime?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches form responses, retaining up to `MAX_RESPONSES_PER_FORM` for rendering.
|
||||
* Every page is scanned for the latest submission timestamp even after the
|
||||
* render cap is reached — the Forms API does not guarantee response order, so
|
||||
* the newest submission may sit on any page. `fetchLatestResponseTime` scans
|
||||
* the same full set during listing, keeping the content hash identical across
|
||||
* the listing and getDocument paths regardless of form size.
|
||||
*/
|
||||
async function fetchFormResponses(accessToken: string, formId: string): Promise<FetchedResponses> {
|
||||
const collected: FormResponse[] = []
|
||||
let latest = ''
|
||||
let pageToken: string | undefined
|
||||
|
||||
do {
|
||||
const url = new URL(`${FORMS_API_BASE}/forms/${encodeURIComponent(formId)}/responses`)
|
||||
url.searchParams.set('pageSize', String(RESPONSES_PAGE_SIZE))
|
||||
if (pageToken) url.searchParams.set('pageToken', pageToken)
|
||||
|
||||
const response = await fetchWithRetry(url.toString(), {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to list responses for form ${formId}: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as FormResponseList
|
||||
const responses = data.responses ?? []
|
||||
|
||||
const pageLatest = latestResponseTime(responses)
|
||||
if (pageLatest && pageLatest > latest) latest = pageLatest
|
||||
|
||||
for (const r of responses) {
|
||||
if (collected.length >= MAX_RESPONSES_PER_FORM) break
|
||||
collected.push(r)
|
||||
}
|
||||
|
||||
pageToken = data.nextPageToken
|
||||
} while (pageToken)
|
||||
|
||||
return { responses: collected, latestSubmittedTime: latest || undefined }
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the latest response submission time for change detection without
|
||||
* retaining responses. Scans every page — the Forms API does not guarantee
|
||||
* response order, so the newest submission may sit on any page. Returns the
|
||||
* greatest `lastSubmittedTime` (falling back to `createTime`), or undefined
|
||||
* when there are none. Throws on a failed read so the caller skips the form
|
||||
* for this run instead of computing a hash from incomplete data — a swallowed
|
||||
* error would poison the stub's content hash and re-process the form on every
|
||||
* sync, while throwing routes into the per-form catch that sets
|
||||
* `skippedOnError` → `listingCapped`.
|
||||
*/
|
||||
async function fetchLatestResponseTime(
|
||||
accessToken: string,
|
||||
formId: string
|
||||
): Promise<string | undefined> {
|
||||
let latest = ''
|
||||
let pageToken: string | undefined
|
||||
|
||||
do {
|
||||
const url = new URL(`${FORMS_API_BASE}/forms/${encodeURIComponent(formId)}/responses`)
|
||||
url.searchParams.set('pageSize', String(RESPONSES_PAGE_SIZE))
|
||||
if (pageToken) url.searchParams.set('pageToken', pageToken)
|
||||
|
||||
const response = await fetchWithRetry(url.toString(), {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`Failed to read responses for change detection on form ${formId}: ${response.status}`
|
||||
)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as FormResponseList
|
||||
const pageLatest = latestResponseTime(data.responses ?? [])
|
||||
if (pageLatest && pageLatest > latest) latest = pageLatest
|
||||
pageToken = data.nextPageToken
|
||||
} while (pageToken)
|
||||
|
||||
return latest || undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the greatest submission timestamp across the given responses, or
|
||||
* undefined when the list is empty.
|
||||
*/
|
||||
function latestResponseTime(responses: FormResponse[]): string | undefined {
|
||||
let latest = ''
|
||||
for (const r of responses) {
|
||||
const t = r.lastSubmittedTime || r.createTime || ''
|
||||
if (t > latest) latest = t
|
||||
}
|
||||
return latest || undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the content hash for a form. The hash must change when either the form
|
||||
* structure (revisionId) or, when responses are indexed, the set of responses
|
||||
* (latest submission time) changes. Drive `modifiedTime` alone is insufficient
|
||||
* because new response submissions do not update the form's Drive modifiedTime.
|
||||
* The content scope is part of the hash so that toggling response indexing
|
||||
* forces a re-sync of every document.
|
||||
*/
|
||||
function formContentHash(input: FormStubInput): string {
|
||||
const responsePart =
|
||||
input.contentScope === 'both'
|
||||
? `${input.latestResponseTime ?? ''}:${input.responseCap}`
|
||||
: 'none'
|
||||
return `gforms:${input.file.id}:${input.contentScope}:${input.revisionId ?? ''}:${responsePart}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a lightweight stub from a form's Drive file and change indicators.
|
||||
* Content is deferred and only fetched via getDocument for new/changed forms.
|
||||
*/
|
||||
function formToStub(input: FormStubInput): ExternalDocument {
|
||||
const { file } = input
|
||||
const title = input.formTitle?.trim() || file.name || 'Untitled Form'
|
||||
return {
|
||||
externalId: file.id,
|
||||
title,
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: file.webViewLink || `https://docs.google.com/forms/d/${file.id}/edit`,
|
||||
contentHash: formContentHash(input),
|
||||
metadata: {
|
||||
formTitle: title,
|
||||
modifiedTime: file.modifiedTime,
|
||||
createdTime: file.createdTime,
|
||||
latestResponseTime: input.contentScope === 'both' ? input.latestResponseTime : undefined,
|
||||
owners: file.owners?.map((o) => o.displayName || o.emailAddress).filter(Boolean),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the answer values for a single question from a response.
|
||||
*/
|
||||
function extractAnswerText(answer: FormAnswer | undefined): string {
|
||||
const values = answer?.textAnswers?.answers
|
||||
?.map((a) => a.value)
|
||||
.filter((v): v is string => typeof v === 'string' && v.trim().length > 0)
|
||||
return values && values.length > 0 ? values.join(', ') : ''
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a question-id → title map from the form structure, so responses can be
|
||||
* rendered with human-readable question labels instead of opaque IDs.
|
||||
*/
|
||||
function buildQuestionTitleMap(form: FormStructure): Map<string, string> {
|
||||
const map = new Map<string, string>()
|
||||
for (const item of form.items ?? []) {
|
||||
const questionId = item.questionItem?.question?.questionId
|
||||
if (questionId && item.title) {
|
||||
map.set(questionId, item.title)
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders the full form document: its structure (title, description, questions)
|
||||
* followed by each response's question/answer pairs when responses are included.
|
||||
*/
|
||||
function renderFormDocument(form: FormStructure, responses: FormResponse[]): string {
|
||||
const parts: string[] = []
|
||||
|
||||
const title = form.info?.title || form.info?.documentTitle
|
||||
if (title) parts.push(`# ${title}`)
|
||||
if (form.info?.description?.trim()) parts.push(form.info.description.trim())
|
||||
|
||||
const questionTitles = buildQuestionTitleMap(form)
|
||||
|
||||
const questionLines: string[] = []
|
||||
for (const item of form.items ?? []) {
|
||||
if (!item.title?.trim()) continue
|
||||
const required = item.questionItem?.question?.required ? ' (required)' : ''
|
||||
questionLines.push(`- ${item.title.trim()}${required}`)
|
||||
if (item.description?.trim()) questionLines.push(` ${item.description.trim()}`)
|
||||
}
|
||||
if (questionLines.length > 0) {
|
||||
parts.push('## Questions')
|
||||
parts.push(questionLines.join('\n'))
|
||||
}
|
||||
|
||||
if (responses.length > 0) {
|
||||
parts.push(`## Responses (${responses.length})`)
|
||||
responses.forEach((response, index) => {
|
||||
const responseLines: string[] = []
|
||||
const submitted = response.lastSubmittedTime || response.createTime
|
||||
const header = submitted
|
||||
? `### Response ${index + 1} — ${submitted}`
|
||||
: `### Response ${index + 1}`
|
||||
responseLines.push(header)
|
||||
if (response.respondentEmail) {
|
||||
responseLines.push(`Respondent: ${response.respondentEmail}`)
|
||||
}
|
||||
for (const [questionId, answer] of Object.entries(response.answers ?? {})) {
|
||||
const label = questionTitles.get(questionId) || questionId
|
||||
const value = extractAnswerText(answer)
|
||||
if (value) responseLines.push(`${label}: ${value}`)
|
||||
}
|
||||
parts.push(responseLines.join('\n'))
|
||||
})
|
||||
}
|
||||
|
||||
return parts.join('\n\n').trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the Drive `q` query that selects form files, optionally scoped to one
|
||||
* or more folders. Single quotes and backslashes in folder IDs are escaped to
|
||||
* prevent query injection.
|
||||
*/
|
||||
function buildDriveQuery(folderIds: string[]): string {
|
||||
const parts = ['trashed = false', `mimeType = '${FORM_MIME_TYPE}'`]
|
||||
const parentsClause = buildDriveParentsClause(folderIds)
|
||||
if (parentsClause) parts.push(parentsClause)
|
||||
return parts.join(' and ')
|
||||
}
|
||||
|
||||
export const googleFormsConnector: ConnectorConfig = {
|
||||
...googleFormsConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const maxForms = parsePositiveInt(sourceConfig.maxForms)
|
||||
const contentScope = resolveContentScope(sourceConfig.contentScope)
|
||||
const responseCap = resolveResponseCap(sourceConfig)
|
||||
const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
|
||||
|
||||
if (maxForms > 0 && previouslyFetched >= maxForms) {
|
||||
return { documents: [], hasMore: false }
|
||||
}
|
||||
|
||||
const folderIds = parseMultiValue(sourceConfig.folderId)
|
||||
const queryParams = new URLSearchParams({
|
||||
q: buildDriveQuery(folderIds),
|
||||
pageSize: String(DRIVE_PAGE_SIZE),
|
||||
orderBy: 'modifiedTime desc',
|
||||
fields: 'nextPageToken,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners)',
|
||||
supportsAllDrives: 'true',
|
||||
includeItemsFromAllDrives: 'true',
|
||||
})
|
||||
if (cursor) queryParams.set('pageToken', cursor)
|
||||
|
||||
const url = `${DRIVE_API_BASE}/files?${queryParams.toString()}`
|
||||
|
||||
logger.info('Listing Google Forms', {
|
||||
folderId: folderIds.length > 0 ? folderIds.join(',') : 'all',
|
||||
contentScope,
|
||||
cursor: cursor ?? 'initial',
|
||||
})
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error('Failed to list Google Forms', { status: response.status, error: errorText })
|
||||
throw new Error(`Failed to list Google Forms: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
let files = (data.files || []) as DriveFormFile[]
|
||||
|
||||
/**
|
||||
* Drive sets `incompleteSearch` when it could not search every corpus (it
|
||||
* arises with the `allDrives` scope enabled by `includeItemsFromAllDrives`).
|
||||
* A partial listing drops still-existing forms, so reconciliation must be
|
||||
* suppressed to avoid hard-deleting valid documents.
|
||||
*/
|
||||
const incompleteSearch = data.incompleteSearch === true
|
||||
|
||||
let slicedSome = false
|
||||
if (maxForms > 0) {
|
||||
const remaining = maxForms - previouslyFetched
|
||||
if (files.length > remaining) {
|
||||
slicedSome = true
|
||||
files = files.slice(0, remaining)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build stubs with metadata-based change indicators. Each form needs its
|
||||
* revisionId (structure changes) and, when responses are indexed, the latest
|
||||
* response time (new submissions) so the sync engine can detect changes
|
||||
* without downloading full content. Forms are processed with bounded
|
||||
* concurrency; a transient per-form failure is skipped rather than aborting
|
||||
* the whole page, but it is recorded so the listing is marked incomplete.
|
||||
*/
|
||||
let skippedOnError = false
|
||||
const stubs = await mapWithConcurrency(files, LIST_CONCURRENCY, async (file) => {
|
||||
try {
|
||||
const form = await fetchFormStructure(accessToken, file.id)
|
||||
if (!form) return null
|
||||
const latest =
|
||||
contentScope === 'both' ? await fetchLatestResponseTime(accessToken, file.id) : undefined
|
||||
return formToStub({
|
||||
file,
|
||||
formTitle: form.info?.title || form.info?.documentTitle,
|
||||
revisionId: form.revisionId,
|
||||
latestResponseTime: latest,
|
||||
contentScope,
|
||||
responseCap,
|
||||
})
|
||||
} catch (error) {
|
||||
skippedOnError = true
|
||||
logger.warn(`Skipping form during listing: ${file.name} (${file.id})`, {
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
})
|
||||
|
||||
const documents = stubs.filter((s): s is ExternalDocument => s !== null)
|
||||
|
||||
const totalFetched = previouslyFetched + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
const hitLimit = maxForms > 0 && totalFetched >= maxForms
|
||||
|
||||
const nextPageToken = data.nextPageToken as string | undefined
|
||||
|
||||
/**
|
||||
* Mark the listing as incomplete so the sync engine skips deletion
|
||||
* reconciliation. Three cases drop still-existing forms from the listing:
|
||||
* - `slicedSome`: this page held more forms than the `maxForms` cap allowed,
|
||||
* so forms beyond the slice were truncated. This is independent of
|
||||
* `hitLimit`, which counts successfully fetched stubs and can fall below
|
||||
* the cap when 404s or errors null out items even though real forms were
|
||||
* sliced off.
|
||||
* - `hitLimit` with a next page: the cap was reached while more pages of
|
||||
* forms remain in the source.
|
||||
* - `skippedOnError`: a transient error dropped a still-present form.
|
||||
* - `incompleteSearch`: Drive could not search every corpus, so the page
|
||||
* itself is partial and may omit still-existing forms.
|
||||
* Deleting any of those would wipe valid documents from the knowledge base.
|
||||
* When the cap merely coincides with source exhaustion (no slice, no next
|
||||
* page), reconciliation stays enabled so deleted forms are cleaned up.
|
||||
*/
|
||||
if (
|
||||
syncContext &&
|
||||
(slicedSome || (hitLimit && Boolean(nextPageToken)) || skippedOnError || incompleteSearch)
|
||||
) {
|
||||
syncContext.listingCapped = true
|
||||
}
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hitLimit ? undefined : nextPageToken,
|
||||
hasMore: hitLimit ? false : Boolean(nextPageToken),
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
const contentScope = resolveContentScope(sourceConfig.contentScope)
|
||||
const fields = 'id,name,mimeType,modifiedTime,createdTime,webViewLink,owners,trashed'
|
||||
const metadataUrl = `${DRIVE_API_BASE}/files/${encodeURIComponent(externalId)}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true`
|
||||
|
||||
const metadataResponse = await fetchWithRetry(metadataUrl, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!metadataResponse.ok) {
|
||||
if (metadataResponse.status === 404) return null
|
||||
throw new Error(`Failed to get form metadata: ${metadataResponse.status}`)
|
||||
}
|
||||
|
||||
const file = (await metadataResponse.json()) as DriveFormFile
|
||||
|
||||
if (file.trashed) return null
|
||||
if (file.mimeType !== FORM_MIME_TYPE) return null
|
||||
|
||||
try {
|
||||
const form = await fetchFormStructure(accessToken, file.id)
|
||||
if (!form) return null
|
||||
|
||||
const responseCap = resolveResponseCap(sourceConfig)
|
||||
const fetched =
|
||||
contentScope === 'both'
|
||||
? await fetchFormResponses(accessToken, file.id)
|
||||
: { responses: [], latestSubmittedTime: undefined }
|
||||
const responses = fetched.responses
|
||||
const cappedResponses =
|
||||
responses.length > responseCap ? responses.slice(0, responseCap) : responses
|
||||
|
||||
const content = renderFormDocument(form, cappedResponses)
|
||||
if (!content.trim()) return null
|
||||
|
||||
const stub = formToStub({
|
||||
file,
|
||||
formTitle: form.info?.title || form.info?.documentTitle,
|
||||
revisionId: form.revisionId,
|
||||
latestResponseTime: fetched.latestSubmittedTime,
|
||||
contentScope,
|
||||
responseCap,
|
||||
})
|
||||
return { ...stub, content, contentDeferred: false }
|
||||
} catch (error) {
|
||||
logger.warn(`Failed to fetch content for form: ${file.name} (${file.id})`, {
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const folderIds = parseMultiValue(sourceConfig.folderId)
|
||||
const maxForms = sourceConfig.maxForms as string | undefined
|
||||
const maxResponsesPerForm = sourceConfig.maxResponsesPerForm as string | undefined
|
||||
|
||||
if (maxForms && (Number.isNaN(Number(maxForms)) || Number(maxForms) <= 0)) {
|
||||
return { valid: false, error: 'Max forms must be a positive number' }
|
||||
}
|
||||
|
||||
if (
|
||||
maxResponsesPerForm &&
|
||||
(Number.isNaN(Number(maxResponsesPerForm)) || Number(maxResponsesPerForm) <= 0)
|
||||
) {
|
||||
return { valid: false, error: 'Max responses per form must be a positive number' }
|
||||
}
|
||||
|
||||
try {
|
||||
if (folderIds.length > 0) {
|
||||
for (const folderId of folderIds) {
|
||||
const url = `${DRIVE_API_BASE}/files/${encodeURIComponent(folderId)}?fields=id,name,mimeType&supportsAllDrives=true`
|
||||
const response = await fetchWithRetry(
|
||||
url,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `Folder "${folderId}" not found. Check the folder ID and permissions.`,
|
||||
}
|
||||
}
|
||||
return {
|
||||
valid: false,
|
||||
error: `Failed to access folder "${folderId}": ${response.status}`,
|
||||
}
|
||||
}
|
||||
|
||||
const folder = await response.json()
|
||||
if (folder.mimeType !== FOLDER_MIME_TYPE) {
|
||||
return { valid: false, error: `"${folderId}" is not a folder` }
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const url = `${DRIVE_API_BASE}/files?pageSize=1&q=${encodeURIComponent(`mimeType = '${FORM_MIME_TYPE}'`)}&fields=files(id)&supportsAllDrives=true&includeItemsFromAllDrives=true`
|
||||
const response = await fetchWithRetry(
|
||||
url,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
return { valid: false, error: `Failed to access Google Forms: ${response.status}` }
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
return { valid: false, error: getErrorMessage(error, 'Failed to validate configuration') }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.formTitle === 'string' && metadata.formTitle.trim()) {
|
||||
result.formTitle = metadata.formTitle.trim()
|
||||
}
|
||||
|
||||
const owners = joinTagArray(metadata.owners)
|
||||
if (owners) result.owners = owners
|
||||
|
||||
const lastModified = parseTagDate(metadata.modifiedTime)
|
||||
if (lastModified) result.lastModified = lastModified
|
||||
|
||||
const lastResponse = parseTagDate(metadata.latestResponseTime)
|
||||
if (lastResponse) result.lastResponse = lastResponse
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { googleFormsConnector } from '@/connectors/google-forms/google-forms'
|
||||
@@ -0,0 +1,87 @@
|
||||
import { GoogleFormsIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
/**
|
||||
* Hard cap on the number of responses appended to a single form document.
|
||||
* Keeps individual documents within a reasonable size for embedding/indexing.
|
||||
*/
|
||||
export const MAX_RESPONSES_PER_FORM = 500
|
||||
|
||||
export const googleFormsConnectorMeta: ConnectorMeta = {
|
||||
id: 'google_forms',
|
||||
name: 'Google Forms',
|
||||
description: 'Sync Google Forms questions and responses into your knowledge base',
|
||||
version: '1.0.0',
|
||||
icon: GoogleFormsIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'oauth',
|
||||
provider: 'google-forms',
|
||||
requiredScopes: [
|
||||
'https://www.googleapis.com/auth/drive',
|
||||
'https://www.googleapis.com/auth/forms.body',
|
||||
'https://www.googleapis.com/auth/forms.responses.readonly',
|
||||
],
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'folderSelector',
|
||||
title: 'Folders',
|
||||
type: 'selector',
|
||||
selectorKey: 'google.drive',
|
||||
mimeType: 'application/vnd.google-apps.folder',
|
||||
canonicalParamId: 'folderId',
|
||||
mode: 'basic',
|
||||
multi: true,
|
||||
placeholder: 'Select one or more folders (optional)',
|
||||
required: false,
|
||||
description: 'Only sync forms inside these Drive folders. Leave blank to sync all forms.',
|
||||
},
|
||||
{
|
||||
id: 'folderId',
|
||||
title: 'Folder IDs',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'folderId',
|
||||
mode: 'advanced',
|
||||
multi: true,
|
||||
placeholder: 'e.g. 1aBcDeFg…, 2cDeFgHi… (comma-separated for multiple)',
|
||||
required: false,
|
||||
description: 'Only sync forms inside these Drive folders. Leave blank to sync all forms.',
|
||||
},
|
||||
{
|
||||
id: 'contentScope',
|
||||
title: 'Content',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
options: [
|
||||
{ label: 'Questions & responses', id: 'both' },
|
||||
{ label: 'Questions only', id: 'structure' },
|
||||
],
|
||||
description: 'Whether to index submitted responses alongside each form’s questions.',
|
||||
},
|
||||
{
|
||||
id: 'maxForms',
|
||||
title: 'Max Forms',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 100 (default: unlimited)',
|
||||
},
|
||||
{
|
||||
id: 'maxResponsesPerForm',
|
||||
title: 'Max Responses Per Form',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: `e.g. 100 (default: ${MAX_RESPONSES_PER_FORM})`,
|
||||
description: 'Cap on responses indexed per form. Applies only when indexing responses.',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'formTitle', displayName: 'Form Title', fieldType: 'text' },
|
||||
{ id: 'owners', displayName: 'Owner', fieldType: 'text' },
|
||||
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
|
||||
{ id: 'lastResponse', displayName: 'Last Response', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,537 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { googleMeetConnectorMeta } from '@/connectors/google-meet/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { joinTagArray, parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('GoogleMeetConnector')
|
||||
|
||||
const MEET_API_BASE = 'https://meet.googleapis.com/v2'
|
||||
const MS_PER_DAY = 24 * 60 * 60 * 1000
|
||||
/** Conference records list page size (Meet API max is 100). */
|
||||
const RECORDS_PAGE_SIZE = 100
|
||||
/** Transcripts list page size (Meet API max is 100). */
|
||||
const TRANSCRIPTS_PAGE_SIZE = 100
|
||||
/** Transcript entries page size (Meet API max is 100). */
|
||||
const ENTRIES_PAGE_SIZE = 100
|
||||
/** Max concurrent participant-name lookups during a single getDocument call. */
|
||||
const PARTICIPANT_FETCH_CONCURRENCY = 5
|
||||
|
||||
/**
|
||||
* A conference record as returned by the Meet REST API v2. A conference record
|
||||
* represents a single meeting session and is immutable once it has ended. Only the
|
||||
* fields the connector reads are modeled.
|
||||
*/
|
||||
interface ConferenceRecord {
|
||||
name: string
|
||||
startTime?: string
|
||||
endTime?: string | null
|
||||
expireTime?: string
|
||||
space?: string
|
||||
}
|
||||
|
||||
interface ConferenceRecordsListResponse {
|
||||
conferenceRecords?: ConferenceRecord[]
|
||||
nextPageToken?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* The Google Doc a transcript is exported to once its `state` reaches
|
||||
* `FILE_GENERATED`. Used to link the synced document back to the source transcript.
|
||||
*/
|
||||
interface DocsDestination {
|
||||
document?: string
|
||||
exportUri?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* A transcript of a conference record. `state` progresses STARTED → ENDED →
|
||||
* FILE_GENERATED; entries are only complete once the session has ended.
|
||||
*/
|
||||
interface Transcript {
|
||||
name: string
|
||||
state?: 'STATE_UNSPECIFIED' | 'STARTED' | 'ENDED' | 'FILE_GENERATED'
|
||||
startTime?: string
|
||||
endTime?: string
|
||||
docsDestination?: DocsDestination
|
||||
}
|
||||
|
||||
interface TranscriptsListResponse {
|
||||
transcripts?: Transcript[]
|
||||
nextPageToken?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* A single speaker-attributed segment of a transcript. `participant` is the resource
|
||||
* name of the speaking participant (resolved to a display name separately).
|
||||
*/
|
||||
interface TranscriptEntry {
|
||||
name: string
|
||||
participant?: string
|
||||
text?: string
|
||||
languageCode?: string
|
||||
startTime?: string
|
||||
endTime?: string
|
||||
}
|
||||
|
||||
interface TranscriptEntriesListResponse {
|
||||
transcriptEntries?: TranscriptEntry[]
|
||||
nextPageToken?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* A meeting participant. The Meet API uses a oneof for the identity — exactly one of
|
||||
* `signedinUser`, `anonymousUser`, or `phoneUser` is populated, each carrying a
|
||||
* `displayName`.
|
||||
*/
|
||||
interface Participant {
|
||||
name: string
|
||||
signedinUser?: { user?: string; displayName?: string }
|
||||
anonymousUser?: { displayName?: string }
|
||||
phoneUser?: { displayName?: string }
|
||||
}
|
||||
|
||||
function meetHeaders(accessToken: string): Record<string, string> {
|
||||
return { Authorization: `Bearer ${accessToken}` }
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes a conference record identifier to its full resource name
|
||||
* (`conferenceRecords/{id}`), tolerating a bare id.
|
||||
*/
|
||||
function conferenceResourceName(externalId: string): string {
|
||||
const trimmed = externalId.trim()
|
||||
return trimmed.startsWith('conferenceRecords/') ? trimmed : `conferenceRecords/${trimmed}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Derives a stable, human-readable title for a meeting. Conference records carry no
|
||||
* title, so the meeting's start date is used.
|
||||
*/
|
||||
function recordTitle(record: ConferenceRecord): string {
|
||||
const date = record.startTime?.slice(0, 10)
|
||||
return date ? `Google Meet — ${date}` : 'Google Meet meeting'
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the meeting duration in whole minutes, or undefined when the meeting has
|
||||
* not ended or timestamps are missing.
|
||||
*/
|
||||
function recordDurationMinutes(record: ConferenceRecord): number | undefined {
|
||||
if (!record.startTime || !record.endTime) return undefined
|
||||
const start = new Date(record.startTime).getTime()
|
||||
const end = new Date(record.endTime).getTime()
|
||||
if (Number.isNaN(start) || Number.isNaN(end) || end <= start) return undefined
|
||||
return Math.round((end - start) / 60000)
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the metadata-based change-detection hash for a conference record. Records
|
||||
* are immutable once ended, so the end time fully captures the final state; an
|
||||
* in-progress meeting (no end time) re-syncs once it ends and the hash changes. The
|
||||
* identical formula is used for both the listing stub and the fetched document.
|
||||
*/
|
||||
function buildContentHash(record: ConferenceRecord): string {
|
||||
return `gmeet:${record.name}:${record.endTime ?? ''}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the deferred listing stub for a conference record. Transcript content is
|
||||
* fetched lazily in getDocument; only metadata and the change hash are computed here.
|
||||
*/
|
||||
function recordToStub(record: ConferenceRecord): ExternalDocument {
|
||||
return {
|
||||
externalId: record.name,
|
||||
title: recordTitle(record),
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
contentHash: buildContentHash(record),
|
||||
metadata: {
|
||||
meetingDate: record.startTime,
|
||||
duration: recordDurationMinutes(record),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a transcript entry's start time as epoch milliseconds for chronological
|
||||
* sorting. Entries without a parseable start time sort last (stably).
|
||||
*/
|
||||
function entryStartMs(entry: TranscriptEntry): number {
|
||||
if (!entry.startTime) return Number.POSITIVE_INFINITY
|
||||
const ms = new Date(entry.startTime).getTime()
|
||||
return Number.isNaN(ms) ? Number.POSITIVE_INFINITY : ms
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a participant's display name across the identity oneof, falling back to a
|
||||
* stable placeholder when no name is exposed (e.g. anonymous joins).
|
||||
*/
|
||||
function participantDisplayName(participant: Participant): string {
|
||||
return (
|
||||
participant.signedinUser?.displayName?.trim() ||
|
||||
participant.anonymousUser?.displayName?.trim() ||
|
||||
participant.phoneUser?.displayName?.trim() ||
|
||||
'Unknown'
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches a single conference record. Returns null on 404 (record expired/deleted).
|
||||
*/
|
||||
async function fetchConferenceRecord(
|
||||
accessToken: string,
|
||||
name: string
|
||||
): Promise<ConferenceRecord | null> {
|
||||
const response = await fetchWithRetry(`${MEET_API_BASE}/${name}`, {
|
||||
method: 'GET',
|
||||
headers: meetHeaders(accessToken),
|
||||
})
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) return null
|
||||
throw new Error(`Failed to fetch Google Meet conference record: ${response.status}`)
|
||||
}
|
||||
return (await response.json()) as ConferenceRecord
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists every transcript belonging to a conference record, following pagination.
|
||||
*/
|
||||
async function fetchTranscripts(accessToken: string, recordName: string): Promise<Transcript[]> {
|
||||
const transcripts: Transcript[] = []
|
||||
let pageToken: string | undefined
|
||||
do {
|
||||
const params = new URLSearchParams({ pageSize: String(TRANSCRIPTS_PAGE_SIZE) })
|
||||
if (pageToken) params.set('pageToken', pageToken)
|
||||
const response = await fetchWithRetry(
|
||||
`${MEET_API_BASE}/${recordName}/transcripts?${params.toString()}`,
|
||||
{ method: 'GET', headers: meetHeaders(accessToken) }
|
||||
)
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) break
|
||||
throw new Error(`Failed to list Google Meet transcripts: ${response.status}`)
|
||||
}
|
||||
const data = (await response.json()) as TranscriptsListResponse
|
||||
if (data.transcripts) transcripts.push(...data.transcripts)
|
||||
pageToken = data.nextPageToken
|
||||
} while (pageToken)
|
||||
return transcripts
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists every entry of a transcript, following pagination. Entries are returned in
|
||||
* chronological order by the API.
|
||||
*/
|
||||
async function fetchTranscriptEntries(
|
||||
accessToken: string,
|
||||
transcriptName: string
|
||||
): Promise<TranscriptEntry[]> {
|
||||
const entries: TranscriptEntry[] = []
|
||||
let pageToken: string | undefined
|
||||
do {
|
||||
const params = new URLSearchParams({ pageSize: String(ENTRIES_PAGE_SIZE) })
|
||||
if (pageToken) params.set('pageToken', pageToken)
|
||||
const response = await fetchWithRetry(
|
||||
`${MEET_API_BASE}/${transcriptName}/entries?${params.toString()}`,
|
||||
{ method: 'GET', headers: meetHeaders(accessToken) }
|
||||
)
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) break
|
||||
throw new Error(`Failed to list Google Meet transcript entries: ${response.status}`)
|
||||
}
|
||||
const data = (await response.json()) as TranscriptEntriesListResponse
|
||||
if (data.transcriptEntries) entries.push(...data.transcriptEntries)
|
||||
pageToken = data.nextPageToken
|
||||
} while (pageToken)
|
||||
return entries
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the display names for a set of participant resource names, returning a map
|
||||
* keyed by resource name. Participants that fail to resolve are omitted so the caller
|
||||
* falls back to a placeholder.
|
||||
*/
|
||||
async function resolveParticipantNames(
|
||||
accessToken: string,
|
||||
participantNames: string[]
|
||||
): Promise<Map<string, string>> {
|
||||
const map = new Map<string, string>()
|
||||
for (let i = 0; i < participantNames.length; i += PARTICIPANT_FETCH_CONCURRENCY) {
|
||||
const batch = participantNames.slice(i, i + PARTICIPANT_FETCH_CONCURRENCY)
|
||||
await Promise.all(
|
||||
batch.map(async (name) => {
|
||||
try {
|
||||
const response = await fetchWithRetry(`${MEET_API_BASE}/${name}`, {
|
||||
method: 'GET',
|
||||
headers: meetHeaders(accessToken),
|
||||
})
|
||||
if (!response.ok) return
|
||||
const participant = (await response.json()) as Participant
|
||||
map.set(name, participantDisplayName(participant))
|
||||
} catch (error) {
|
||||
logger.warn('Failed to resolve Google Meet participant', {
|
||||
participant: name,
|
||||
error: toError(error).message,
|
||||
})
|
||||
}
|
||||
})
|
||||
)
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a meeting header plus speaker-attributed transcript lines into plain text.
|
||||
*/
|
||||
function formatTranscriptContent(
|
||||
record: ConferenceRecord,
|
||||
entries: TranscriptEntry[],
|
||||
participantNames: Map<string, string>
|
||||
): string {
|
||||
const parts: string[] = []
|
||||
parts.push(`Meeting: ${recordTitle(record)}`)
|
||||
if (record.startTime) parts.push(`Date: ${record.startTime}`)
|
||||
const minutes = recordDurationMinutes(record)
|
||||
if (minutes != null) parts.push(`Duration: ${minutes} minutes`)
|
||||
|
||||
const speakers = Array.from(
|
||||
new Set(
|
||||
entries
|
||||
.map((entry) => (entry.participant ? participantNames.get(entry.participant) : undefined))
|
||||
.filter((name): name is string => Boolean(name))
|
||||
)
|
||||
)
|
||||
if (speakers.length > 0) parts.push(`Participants: ${speakers.join(', ')}`)
|
||||
|
||||
parts.push('')
|
||||
parts.push('--- Transcript ---')
|
||||
for (const entry of entries) {
|
||||
const text = entry.text?.trim()
|
||||
if (!text) continue
|
||||
const speaker = (entry.participant && participantNames.get(entry.participant)) || 'Unknown'
|
||||
parts.push(`${speaker}: ${text}`)
|
||||
}
|
||||
|
||||
return parts.join('\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the conference records list `filter` from the connector's scoping config.
|
||||
* Only the documented `start_time` filter is emitted, and only when a lookback window
|
||||
* is configured (full sync otherwise).
|
||||
*/
|
||||
function buildRecordsFilter(sourceConfig: Record<string, unknown>): string | undefined {
|
||||
const lookbackDays = sourceConfig.lookbackDays ? Number(sourceConfig.lookbackDays) : 0
|
||||
if (!Number.isFinite(lookbackDays) || lookbackDays <= 0) return undefined
|
||||
const since = new Date(Date.now() - lookbackDays * MS_PER_DAY).toISOString()
|
||||
return `start_time >= "${since}"`
|
||||
}
|
||||
|
||||
export const googleMeetConnector: ConnectorConfig = {
|
||||
...googleMeetConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const maxMeetings = sourceConfig.maxMeetings ? Number(sourceConfig.maxMeetings) : 0
|
||||
const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
|
||||
|
||||
const pageSize =
|
||||
maxMeetings > 0
|
||||
? Math.min(RECORDS_PAGE_SIZE, Math.max(1, maxMeetings - prevFetched))
|
||||
: RECORDS_PAGE_SIZE
|
||||
const params = new URLSearchParams({ pageSize: String(pageSize) })
|
||||
if (cursor) params.set('pageToken', cursor)
|
||||
const filter = buildRecordsFilter(sourceConfig)
|
||||
if (filter) params.set('filter', filter)
|
||||
|
||||
logger.info('Listing Google Meet conference records', {
|
||||
hasCursor: Boolean(cursor),
|
||||
hasFilter: Boolean(filter),
|
||||
})
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
`${MEET_API_BASE}/conferenceRecords?${params.toString()}`,
|
||||
{ method: 'GET', headers: meetHeaders(accessToken) }
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
logger.error('Failed to list Google Meet conference records', {
|
||||
status: response.status,
|
||||
error: errorText.slice(0, 500),
|
||||
})
|
||||
throw new Error(`Failed to list Google Meet conference records: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as ConferenceRecordsListResponse
|
||||
const records = data.conferenceRecords ?? []
|
||||
const nextPageToken = data.nextPageToken?.trim() || undefined
|
||||
|
||||
const allDocuments = records
|
||||
.filter((record) => Boolean(record.name))
|
||||
.map((record) => recordToStub(record))
|
||||
|
||||
let documents = allDocuments
|
||||
if (maxMeetings > 0) {
|
||||
const remaining = Math.max(0, maxMeetings - prevFetched)
|
||||
if (allDocuments.length > remaining) documents = allDocuments.slice(0, remaining)
|
||||
}
|
||||
|
||||
const totalFetched = prevFetched + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
const reachedCap = maxMeetings > 0 && totalFetched >= maxMeetings
|
||||
|
||||
// Only flag the listing as capped when the cap actually truncated a larger source —
|
||||
// either more pages remain, or records were dropped from this page. If the source
|
||||
// was fully listed and merely happens to equal the cap, leave it unflagged so the
|
||||
// sync engine still reconciles deletions of meetings that disappear upstream.
|
||||
const truncated =
|
||||
reachedCap && (Boolean(nextPageToken) || allDocuments.length > documents.length)
|
||||
if (truncated && syncContext) syncContext.listingCapped = true
|
||||
|
||||
const hasMore = !reachedCap && Boolean(nextPageToken)
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore ? nextPageToken : undefined,
|
||||
hasMore,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
_sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
try {
|
||||
if (!externalId) return null
|
||||
const recordName = conferenceResourceName(externalId)
|
||||
|
||||
const record = await fetchConferenceRecord(accessToken, recordName)
|
||||
if (!record) return null
|
||||
|
||||
const transcripts = await fetchTranscripts(accessToken, recordName)
|
||||
if (transcripts.length === 0) return null
|
||||
|
||||
// Only index once every transcript is fully generated. Before then the entry set
|
||||
// is still being populated, and because the content hash is keyed on the (now
|
||||
// fixed) conference endTime, a partial transcript stored here would never be
|
||||
// refreshed on later syncs. Waiting for FILE_GENERATED keeps indexed content final.
|
||||
if (transcripts.some((transcript) => transcript.state !== 'FILE_GENERATED')) {
|
||||
logger.info('Google Meet transcript not finalized yet', { externalId })
|
||||
return null
|
||||
}
|
||||
|
||||
const entryGroups = await Promise.all(
|
||||
transcripts.map((transcript) => fetchTranscriptEntries(accessToken, transcript.name))
|
||||
)
|
||||
// The API guarantees chronological order only within a single transcript, so sort
|
||||
// the merged entries by start time to keep speaker lines in sequence when a
|
||||
// conference has more than one transcript.
|
||||
const entries = entryGroups.flat().sort((a, b) => entryStartMs(a) - entryStartMs(b))
|
||||
|
||||
const hasText = entries.some((entry) => entry.text?.trim())
|
||||
if (!hasText) {
|
||||
logger.info('Transcript not yet available for Google Meet conference', { externalId })
|
||||
return null
|
||||
}
|
||||
|
||||
const participantNames = await resolveParticipantNames(
|
||||
accessToken,
|
||||
Array.from(
|
||||
new Set(
|
||||
entries
|
||||
.map((entry) => entry.participant)
|
||||
.filter((name): name is string => Boolean(name))
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
const content = formatTranscriptContent(record, entries, participantNames)
|
||||
const sourceUrl = transcripts.find((t) => t.docsDestination?.exportUri)?.docsDestination
|
||||
?.exportUri
|
||||
|
||||
const speakers = Array.from(new Set(Array.from(participantNames.values())))
|
||||
|
||||
return {
|
||||
externalId: record.name,
|
||||
title: recordTitle(record),
|
||||
content,
|
||||
contentDeferred: false,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: sourceUrl || undefined,
|
||||
contentHash: buildContentHash(record),
|
||||
metadata: {
|
||||
meetingDate: record.startTime,
|
||||
duration: recordDurationMinutes(record),
|
||||
participants: speakers,
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Failed to get Google Meet transcript', {
|
||||
externalId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const maxMeetings = sourceConfig.maxMeetings as string | undefined
|
||||
if (maxMeetings && (Number.isNaN(Number(maxMeetings)) || Number(maxMeetings) < 0)) {
|
||||
return { valid: false, error: 'Max meetings must be a non-negative number' }
|
||||
}
|
||||
|
||||
const lookbackDays = sourceConfig.lookbackDays as string | undefined
|
||||
if (lookbackDays && (Number.isNaN(Number(lookbackDays)) || Number(lookbackDays) < 0)) {
|
||||
return { valid: false, error: 'Lookback window must be a non-negative number of days' }
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetchWithRetry(
|
||||
`${MEET_API_BASE}/conferenceRecords?pageSize=1`,
|
||||
{ method: 'GET', headers: meetHeaders(accessToken) },
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
return {
|
||||
valid: false,
|
||||
error: `Google Meet access failed: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}`,
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
const participants = joinTagArray(metadata.participants)
|
||||
if (participants) result.participants = participants
|
||||
|
||||
if (metadata.duration != null) {
|
||||
const num = Number(metadata.duration)
|
||||
if (!Number.isNaN(num)) result.duration = num
|
||||
}
|
||||
|
||||
const meetingDate = parseTagDate(metadata.meetingDate)
|
||||
if (meetingDate) result.meetingDate = meetingDate
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { googleMeetConnector } from '@/connectors/google-meet/google-meet'
|
||||
@@ -0,0 +1,42 @@
|
||||
import { GoogleMeetIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const googleMeetConnectorMeta: ConnectorMeta = {
|
||||
id: 'google_meet',
|
||||
name: 'Google Meet',
|
||||
description: 'Sync meeting transcripts from Google Meet into your knowledge base',
|
||||
version: '1.0.0',
|
||||
icon: GoogleMeetIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'oauth',
|
||||
provider: 'google-meet',
|
||||
requiredScopes: ['https://www.googleapis.com/auth/meetings.space.readonly'],
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'maxMeetings',
|
||||
title: 'Max Meetings',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 200 (default: unlimited)',
|
||||
description: 'Cap the total number of meetings synced. Leave blank to sync all.',
|
||||
},
|
||||
{
|
||||
id: 'lookbackDays',
|
||||
title: 'Lookback Window (days)',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. 90 (default: all time)',
|
||||
description: 'Only sync meetings from the last N days. Leave blank to sync any age.',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'participants', displayName: 'Participants', fieldType: 'text' },
|
||||
{ id: 'duration', displayName: 'Duration (minutes)', fieldType: 'number' },
|
||||
{ id: 'meetingDate', displayName: 'Meeting Date', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,379 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { googleSheetsConnectorMeta } from '@/connectors/google-sheets/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('GoogleSheetsConnector')
|
||||
|
||||
const SHEETS_API_BASE = 'https://sheets.googleapis.com/v4/spreadsheets'
|
||||
const DRIVE_API_BASE = 'https://www.googleapis.com/drive/v3/files'
|
||||
const MAX_ROWS = 10000
|
||||
const CONCURRENCY = 3
|
||||
|
||||
interface SheetProperties {
|
||||
sheetId: number
|
||||
title: string
|
||||
index: number
|
||||
gridProperties?: {
|
||||
rowCount?: number
|
||||
columnCount?: number
|
||||
}
|
||||
}
|
||||
|
||||
interface SpreadsheetMetadata {
|
||||
spreadsheetId: string
|
||||
properties: {
|
||||
title: string
|
||||
locale?: string
|
||||
}
|
||||
sheets: { properties: SheetProperties }[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats sheet data into an LLM-friendly text representation.
|
||||
* Each row is labeled with its index and columns are identified by header names.
|
||||
*/
|
||||
function formatSheetContent(headers: string[], rows: string[][]): string {
|
||||
if (headers.length === 0) return ''
|
||||
|
||||
const lines: string[] = []
|
||||
|
||||
for (let i = 0; i < rows.length; i++) {
|
||||
const row = rows[i]
|
||||
lines.push(`Row ${i + 1}:`)
|
||||
for (let j = 0; j < headers.length; j++) {
|
||||
const value = j < row.length ? row[j] : ''
|
||||
lines.push(` ${headers[j]}: ${value}`)
|
||||
}
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
return lines.join('\n').trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches all values from a single sheet tab.
|
||||
*/
|
||||
async function fetchSheetValues(
|
||||
accessToken: string,
|
||||
spreadsheetId: string,
|
||||
sheetTitle: string
|
||||
): Promise<string[][]> {
|
||||
const range = `'${sheetTitle.replace(/'/g, "''")}'!A1:ZZ${MAX_ROWS}`
|
||||
const url = `${SHEETS_API_BASE}/${encodeURIComponent(spreadsheetId)}/values/${encodeURIComponent(range)}?majorDimension=ROWS&valueRenderOption=FORMATTED_VALUE`
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch sheet values for "${sheetTitle}": ${response.status}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
return (data.values || []) as string[][]
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches spreadsheet metadata (title, sheet names, grid properties).
|
||||
*/
|
||||
async function fetchSpreadsheetMetadata(
|
||||
accessToken: string,
|
||||
spreadsheetId: string
|
||||
): Promise<SpreadsheetMetadata> {
|
||||
const url = `${SHEETS_API_BASE}/${encodeURIComponent(spreadsheetId)}?fields=spreadsheetId,properties.title,properties.locale,sheets.properties`
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch spreadsheet metadata: ${response.status}`)
|
||||
}
|
||||
|
||||
return (await response.json()) as SpreadsheetMetadata
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the spreadsheet's modifiedTime from the Drive API.
|
||||
*/
|
||||
async function fetchSpreadsheetModifiedTime(
|
||||
accessToken: string,
|
||||
spreadsheetId: string
|
||||
): Promise<string | undefined> {
|
||||
try {
|
||||
const url = `${DRIVE_API_BASE}/${encodeURIComponent(spreadsheetId)}?fields=modifiedTime&supportsAllDrives=true`
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
logger.warn('Failed to fetch modifiedTime from Drive API', { status: response.status })
|
||||
return undefined
|
||||
}
|
||||
|
||||
const data = (await response.json()) as { modifiedTime?: string }
|
||||
return data.modifiedTime
|
||||
} catch (error) {
|
||||
logger.warn('Error fetching modifiedTime from Drive API', {
|
||||
error: toError(error).message,
|
||||
})
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a single sheet tab into an ExternalDocument.
|
||||
*/
|
||||
async function sheetToDocument(
|
||||
accessToken: string,
|
||||
spreadsheetId: string,
|
||||
spreadsheetTitle: string,
|
||||
sheet: SheetProperties,
|
||||
modifiedTime?: string
|
||||
): Promise<ExternalDocument | null> {
|
||||
try {
|
||||
const values = await fetchSheetValues(accessToken, spreadsheetId, sheet.title)
|
||||
|
||||
if (values.length === 0) {
|
||||
logger.info(`Skipping empty sheet: ${sheet.title}`)
|
||||
return null
|
||||
}
|
||||
|
||||
const headers = values[0].map((h, idx) =>
|
||||
typeof h === 'string' && h.trim() ? h.trim() : `Column ${idx + 1}`
|
||||
)
|
||||
const dataRows = values.slice(1)
|
||||
|
||||
if (dataRows.length === 0) {
|
||||
logger.info(`Skipping header-only sheet: ${sheet.title}`)
|
||||
return null
|
||||
}
|
||||
|
||||
const content = formatSheetContent(headers, dataRows)
|
||||
if (!content.trim()) {
|
||||
return null
|
||||
}
|
||||
|
||||
const rowCount = dataRows.length
|
||||
|
||||
return {
|
||||
externalId: `${spreadsheetId}__sheet__${sheet.sheetId}`,
|
||||
title: `${spreadsheetTitle} - ${sheet.title}`,
|
||||
content,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: `https://docs.google.com/spreadsheets/d/${spreadsheetId}/edit#gid=${sheet.sheetId}`,
|
||||
contentHash: `gsheets:${spreadsheetId}:${sheet.sheetId}:${modifiedTime ?? ''}`,
|
||||
metadata: {
|
||||
spreadsheetId,
|
||||
spreadsheetTitle,
|
||||
sheetTitle: sheet.title,
|
||||
sheetId: sheet.sheetId,
|
||||
rowCount,
|
||||
columnCount: headers.length,
|
||||
...(modifiedTime ? { modifiedTime } : {}),
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn(`Failed to extract content from sheet: ${sheet.title}`, {
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
export const googleSheetsConnector: ConnectorConfig = {
|
||||
...googleSheetsConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
_cursor?: string,
|
||||
_syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const spreadsheetId = (sourceConfig.spreadsheetId as string)?.trim()
|
||||
if (!spreadsheetId) {
|
||||
throw new Error('Spreadsheet ID is required')
|
||||
}
|
||||
|
||||
logger.info('Fetching spreadsheet metadata', { spreadsheetId })
|
||||
|
||||
const [metadata, modifiedTime] = await Promise.all([
|
||||
fetchSpreadsheetMetadata(accessToken, spreadsheetId),
|
||||
fetchSpreadsheetModifiedTime(accessToken, spreadsheetId),
|
||||
])
|
||||
const sheetFilter = (sourceConfig.sheetFilter as string) || 'all'
|
||||
|
||||
let sheets = metadata.sheets.map((s) => s.properties)
|
||||
if (sheetFilter === 'first' && sheets.length > 0) {
|
||||
sheets = [sheets[0]]
|
||||
}
|
||||
|
||||
logger.info('Processing sheets', {
|
||||
spreadsheetTitle: metadata.properties.title,
|
||||
sheetCount: sheets.length,
|
||||
})
|
||||
|
||||
const documents: ExternalDocument[] = sheets.map((sheet) => ({
|
||||
externalId: `${spreadsheetId}__sheet__${sheet.sheetId}`,
|
||||
title: `${metadata.properties.title} - ${sheet.title}`,
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: `https://docs.google.com/spreadsheets/d/${spreadsheetId}/edit#gid=${sheet.sheetId}`,
|
||||
contentHash: `gsheets:${spreadsheetId}:${sheet.sheetId}:${modifiedTime ?? ''}`,
|
||||
metadata: {
|
||||
spreadsheetId,
|
||||
spreadsheetTitle: metadata.properties.title,
|
||||
sheetTitle: sheet.title,
|
||||
sheetId: sheet.sheetId,
|
||||
rowCount: sheet.gridProperties?.rowCount,
|
||||
columnCount: sheet.gridProperties?.columnCount,
|
||||
...(modifiedTime ? { modifiedTime } : {}),
|
||||
},
|
||||
}))
|
||||
|
||||
return {
|
||||
documents,
|
||||
hasMore: false,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
const parts = externalId.split('__sheet__')
|
||||
if (parts.length !== 2) {
|
||||
logger.warn('Invalid external ID format', { externalId })
|
||||
return null
|
||||
}
|
||||
|
||||
const spreadsheetId = parts[0]
|
||||
const sheetId = Number(parts[1])
|
||||
|
||||
if (Number.isNaN(sheetId)) {
|
||||
logger.warn('Invalid sheet ID in external ID', { externalId })
|
||||
return null
|
||||
}
|
||||
|
||||
let metadata: SpreadsheetMetadata
|
||||
let modifiedTime: string | undefined
|
||||
try {
|
||||
;[metadata, modifiedTime] = await Promise.all([
|
||||
fetchSpreadsheetMetadata(accessToken, spreadsheetId),
|
||||
fetchSpreadsheetModifiedTime(accessToken, spreadsheetId),
|
||||
])
|
||||
} catch (error) {
|
||||
const message = toError(error).message
|
||||
if (message.includes('404')) {
|
||||
logger.info('Spreadsheet not found (possibly deleted)', { spreadsheetId })
|
||||
return null
|
||||
}
|
||||
throw error
|
||||
}
|
||||
|
||||
const sheetEntry = metadata.sheets.find((s) => s.properties.sheetId === sheetId)
|
||||
|
||||
if (!sheetEntry) {
|
||||
logger.info('Sheet not found in spreadsheet', { spreadsheetId, sheetId })
|
||||
return null
|
||||
}
|
||||
|
||||
const doc = await sheetToDocument(
|
||||
accessToken,
|
||||
spreadsheetId,
|
||||
metadata.properties.title,
|
||||
sheetEntry.properties,
|
||||
modifiedTime
|
||||
)
|
||||
if (!doc) return null
|
||||
return { ...doc, contentDeferred: false }
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const spreadsheetId = (sourceConfig.spreadsheetId as string)?.trim()
|
||||
|
||||
if (!spreadsheetId) {
|
||||
return { valid: false, error: 'Spreadsheet ID is required' }
|
||||
}
|
||||
|
||||
try {
|
||||
const url = `${SHEETS_API_BASE}/${encodeURIComponent(spreadsheetId)}?fields=spreadsheetId,properties.title`
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
url,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) {
|
||||
return {
|
||||
valid: false,
|
||||
error: 'Spreadsheet not found. Check the ID and ensure it is shared with your account.',
|
||||
}
|
||||
}
|
||||
if (response.status === 403) {
|
||||
return {
|
||||
valid: false,
|
||||
error: 'Access denied. Ensure the spreadsheet is shared with your Google account.',
|
||||
}
|
||||
}
|
||||
return { valid: false, error: `Failed to access spreadsheet: ${response.status}` }
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.sheetTitle === 'string') {
|
||||
result.sheetTitle = metadata.sheetTitle
|
||||
}
|
||||
|
||||
if (typeof metadata.rowCount === 'number') {
|
||||
result.rowCount = metadata.rowCount
|
||||
}
|
||||
|
||||
if (typeof metadata.columnCount === 'number') {
|
||||
result.columnCount = metadata.columnCount
|
||||
}
|
||||
|
||||
const lastModified = parseTagDate(metadata.modifiedTime)
|
||||
if (lastModified) {
|
||||
result.lastModified = lastModified
|
||||
}
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { googleSheetsConnector } from '@/connectors/google-sheets/google-sheets'
|
||||
@@ -0,0 +1,57 @@
|
||||
import { GoogleSheetsIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const googleSheetsConnectorMeta: ConnectorMeta = {
|
||||
id: 'google_sheets',
|
||||
name: 'Google Sheets',
|
||||
description: 'Sync spreadsheet data from Google Sheets',
|
||||
version: '1.0.0',
|
||||
icon: GoogleSheetsIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'oauth',
|
||||
provider: 'google-sheets',
|
||||
requiredScopes: ['https://www.googleapis.com/auth/drive'],
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'spreadsheetSelector',
|
||||
title: 'Spreadsheet',
|
||||
type: 'selector',
|
||||
selectorKey: 'google.drive',
|
||||
mimeType: 'application/vnd.google-apps.spreadsheet',
|
||||
canonicalParamId: 'spreadsheetId',
|
||||
mode: 'basic',
|
||||
placeholder: 'Select a spreadsheet',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'spreadsheetId',
|
||||
title: 'Spreadsheet ID',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'spreadsheetId',
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. 1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgVE2upms',
|
||||
required: true,
|
||||
description: 'The ID from the spreadsheet URL: docs.google.com/spreadsheets/d/{ID}/edit',
|
||||
},
|
||||
{
|
||||
id: 'sheetFilter',
|
||||
title: 'Sheets to Sync',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
options: [
|
||||
{ label: 'All sheets', id: 'all' },
|
||||
{ label: 'First sheet only', id: 'first' },
|
||||
],
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'sheetTitle', displayName: 'Sheet Name', fieldType: 'text' },
|
||||
{ id: 'rowCount', displayName: 'Row Count', fieldType: 'number' },
|
||||
{ id: 'columnCount', displayName: 'Column Count', fieldType: 'number' },
|
||||
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,504 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { grainConnectorMeta } from '@/connectors/grain/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { joinTagArray, parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('GrainConnector')
|
||||
|
||||
const GRAIN_API_BASE = 'https://api.grain.com/_/public-api/v2'
|
||||
/**
|
||||
* Grain's Public API requires a pinned date-based version header on every request.
|
||||
* Matches the version used by the in-repo Grain tools.
|
||||
*/
|
||||
const GRAIN_API_VERSION = '2025-10-31'
|
||||
|
||||
/**
|
||||
* A participant on a Grain recording. The list endpoint only populates this when
|
||||
* `include.participants` is requested in the body.
|
||||
*/
|
||||
interface GrainParticipant {
|
||||
id: string
|
||||
name: string
|
||||
email: string | null
|
||||
}
|
||||
|
||||
/**
|
||||
* A team a Grain recording belongs to. Always present on the recording object (may be
|
||||
* an empty array).
|
||||
*/
|
||||
interface GrainTeam {
|
||||
id: string
|
||||
name: string
|
||||
}
|
||||
|
||||
/**
|
||||
* The meeting type classification of a Grain recording. Always present on the recording
|
||||
* object but nullable.
|
||||
*/
|
||||
interface GrainMeetingType {
|
||||
id: string
|
||||
name: string
|
||||
scope?: 'internal' | 'external'
|
||||
}
|
||||
|
||||
/**
|
||||
* A Grain recording as returned by the v2 recordings endpoints. Only the fields the
|
||||
* connector reads are modeled; the API returns additional optional fields.
|
||||
*
|
||||
* The v2 Public API returns the recording identifier as `id` (confirmed against the
|
||||
* Grain Public API reference and the in-repo Grain tools).
|
||||
*
|
||||
* `source`, `tags`, `teams`, and `meeting_type` are returned by default on the
|
||||
* recording object. `participants` is populated only when requested via the
|
||||
* `include.participants` flag — the connector requests it (see {@link RECORDING_INCLUDE})
|
||||
* so participant names are available for tag mapping.
|
||||
*/
|
||||
interface GrainRecording {
|
||||
id?: string
|
||||
title?: string
|
||||
start_datetime?: string
|
||||
end_datetime?: string
|
||||
duration_ms?: number
|
||||
url?: string
|
||||
source?: string
|
||||
tags?: string[]
|
||||
teams?: GrainTeam[]
|
||||
meeting_type?: GrainMeetingType | null
|
||||
participants?: GrainParticipant[]
|
||||
}
|
||||
|
||||
interface GrainRecordingsListResponse {
|
||||
recordings?: GrainRecording[]
|
||||
cursor?: string | null
|
||||
}
|
||||
|
||||
/**
|
||||
* A single speaker-attributed segment of a Grain transcript. The transcript endpoint
|
||||
* returns a bare JSON array of these.
|
||||
*/
|
||||
interface GrainTranscriptSegment {
|
||||
participant_id: string | null
|
||||
speaker?: string
|
||||
start?: number
|
||||
end?: number
|
||||
text?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* The `include` flags requested on every recordings call. Grain returns `teams` and
|
||||
* `meeting_type` by default, but gates `participants` behind an include flag. Participant
|
||||
* names feed connector tag mapping, so the flag is always requested. Only documented
|
||||
* include flags are sent to avoid the API rejecting unknown keys.
|
||||
*/
|
||||
const RECORDING_INCLUDE = { participants: true } as const
|
||||
|
||||
/** Number of milliseconds in a day, used to convert the lookback window to a timestamp. */
|
||||
const MS_PER_DAY = 24 * 60 * 60 * 1000
|
||||
|
||||
/**
|
||||
* Valid values for the recordings list `participant_scope` filter (verified against the
|
||||
* Grain Public API recordings list request body, which the in-repo Grain tools also use).
|
||||
*/
|
||||
const PARTICIPANT_SCOPES = ['internal', 'external'] as const
|
||||
type ParticipantScope = (typeof PARTICIPANT_SCOPES)[number]
|
||||
|
||||
function isParticipantScope(value: unknown): value is ParticipantScope {
|
||||
return typeof value === 'string' && PARTICIPANT_SCOPES.includes(value as ParticipantScope)
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the recordings list `filter` object from the connector's scoping config. Only
|
||||
* documented Grain filter keys are emitted, and only when configured, so an empty config
|
||||
* produces no `filter` (full sync). Returns undefined when no scoping is configured.
|
||||
*
|
||||
* Supported keys (verified against the in-repo Grain list_recordings tool / Public API):
|
||||
* - `after_datetime` — derived from `lookbackDays`; recordings on/after the window start
|
||||
* - `participant_scope` — `internal` or `external`
|
||||
* - `title_search` — substring match against recording titles
|
||||
* - `team` — recordings belonging to the given team UUID
|
||||
* - `meeting_type` — recordings classified as the given meeting type UUID
|
||||
*/
|
||||
function buildRecordingFilter(
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Record<string, unknown> | undefined {
|
||||
const filter: Record<string, unknown> = {}
|
||||
|
||||
const lookbackDays = sourceConfig.lookbackDays ? Number(sourceConfig.lookbackDays) : 0
|
||||
if (Number.isFinite(lookbackDays) && lookbackDays > 0) {
|
||||
filter.after_datetime = new Date(Date.now() - lookbackDays * MS_PER_DAY).toISOString()
|
||||
}
|
||||
|
||||
if (isParticipantScope(sourceConfig.participantScope)) {
|
||||
filter.participant_scope = sourceConfig.participantScope
|
||||
}
|
||||
|
||||
const titleSearch =
|
||||
typeof sourceConfig.titleSearch === 'string' ? sourceConfig.titleSearch.trim() : ''
|
||||
if (titleSearch) filter.title_search = titleSearch
|
||||
|
||||
const team = typeof sourceConfig.teamId === 'string' ? sourceConfig.teamId.trim() : ''
|
||||
if (team) filter.team = team
|
||||
|
||||
const meetingType =
|
||||
typeof sourceConfig.meetingTypeId === 'string' ? sourceConfig.meetingTypeId.trim() : ''
|
||||
if (meetingType) filter.meeting_type = meetingType
|
||||
|
||||
return Object.keys(filter).length > 0 ? filter : undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the auth + version headers shared by every Grain API request.
|
||||
*/
|
||||
function grainHeaders(accessToken: string): Record<string, string> {
|
||||
return {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Public-Api-Version': GRAIN_API_VERSION,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the recording's unique identifier. The v2 Public API returns the recording
|
||||
* id as the `id` field. Returns an empty string when it is absent.
|
||||
*/
|
||||
function recordingId(recording: GrainRecording): string {
|
||||
return (recording.id ?? '').trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Derives the document title for a recording, falling back to a stable placeholder.
|
||||
*/
|
||||
function recordingTitle(recording: GrainRecording): string {
|
||||
return recording.title?.trim() || 'Untitled Grain Recording'
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts participant display names from a recording, dropping blanks.
|
||||
*/
|
||||
function participantNames(recording: GrainRecording): string[] {
|
||||
return (recording.participants ?? [])
|
||||
.map((p) => p.name?.trim())
|
||||
.filter((name): name is string => Boolean(name))
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts team names from a recording, dropping blanks.
|
||||
*/
|
||||
function teamNames(recording: GrainRecording): string[] {
|
||||
return (recording.teams ?? [])
|
||||
.map((t) => t.name?.trim())
|
||||
.filter((name): name is string => Boolean(name))
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts user-applied tag labels from a recording, dropping blanks.
|
||||
*/
|
||||
function recordingLabels(recording: GrainRecording): string[] {
|
||||
return (recording.tags ?? [])
|
||||
.map((tag) => tag?.trim())
|
||||
.filter((tag): tag is string => Boolean(tag))
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the metadata-based change-detection hash for a recording.
|
||||
*
|
||||
* Grain exposes no `updated_at`/`modified` field, so the hash combines the stable
|
||||
* recording id with `end_datetime` and `duration_ms` — the values that change when a
|
||||
* recording is re-processed or re-cut. The identical formula is used for both the
|
||||
* listing stub and the fully-fetched document so unchanged recordings are skipped.
|
||||
*/
|
||||
function buildContentHash(recording: GrainRecording): string {
|
||||
return `grain:${recordingId(recording)}:${recording.end_datetime ?? ''}:${recording.duration_ms ?? ''}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the metadata bag attached to both stubs and fetched documents. Keeping a
|
||||
* single source ensures the stub and getDocument agree on tag inputs.
|
||||
*/
|
||||
function buildMetadata(recording: GrainRecording): Record<string, unknown> {
|
||||
return {
|
||||
title: recordingTitle(recording),
|
||||
duration: recording.duration_ms,
|
||||
meetingDate: recording.start_datetime,
|
||||
participants: participantNames(recording),
|
||||
source: recording.source,
|
||||
labels: recordingLabels(recording),
|
||||
teams: teamNames(recording),
|
||||
meetingType: recording.meeting_type?.name,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the deferred listing stub for a recording. Content is fetched lazily in
|
||||
* getDocument; only metadata and the change hash are computed here.
|
||||
*/
|
||||
function recordingToStub(recording: GrainRecording): ExternalDocument {
|
||||
return {
|
||||
externalId: recordingId(recording),
|
||||
title: recordingTitle(recording),
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: recording.url || undefined,
|
||||
contentHash: buildContentHash(recording),
|
||||
metadata: buildMetadata(recording),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a recording header plus speaker-attributed transcript lines into plain text.
|
||||
*/
|
||||
function formatTranscriptContent(
|
||||
recording: GrainRecording,
|
||||
segments: GrainTranscriptSegment[]
|
||||
): string {
|
||||
const parts: string[] = []
|
||||
|
||||
parts.push(`Meeting: ${recordingTitle(recording)}`)
|
||||
if (recording.start_datetime) parts.push(`Date: ${recording.start_datetime}`)
|
||||
if (recording.duration_ms != null) {
|
||||
const minutes = Math.round(recording.duration_ms / 60000)
|
||||
parts.push(`Duration: ${minutes} minutes`)
|
||||
}
|
||||
const names = participantNames(recording)
|
||||
if (names.length > 0) parts.push(`Participants: ${names.join(', ')}`)
|
||||
|
||||
parts.push('')
|
||||
parts.push('--- Transcript ---')
|
||||
for (const segment of segments) {
|
||||
const text = segment.text?.trim()
|
||||
if (!text) continue
|
||||
const speaker = segment.speaker?.trim() || 'Unknown'
|
||||
parts.push(`${speaker}: ${text}`)
|
||||
}
|
||||
|
||||
return parts.join('\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches a single recording's metadata from the v2 recordings endpoint.
|
||||
* Returns null on 404 (recording deleted/inaccessible).
|
||||
*/
|
||||
async function fetchRecording(accessToken: string, id: string): Promise<GrainRecording | null> {
|
||||
const response = await fetchWithRetry(`${GRAIN_API_BASE}/recordings/${id}`, {
|
||||
method: 'POST',
|
||||
headers: grainHeaders(accessToken),
|
||||
body: JSON.stringify({ include: RECORDING_INCLUDE }),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) return null
|
||||
throw new Error(`Failed to fetch Grain recording: ${response.status}`)
|
||||
}
|
||||
|
||||
return (await response.json()) as GrainRecording
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the speaker-attributed transcript segments for a recording.
|
||||
* Returns null on 404, or an empty array when the recording has no transcript yet.
|
||||
*/
|
||||
async function fetchTranscript(
|
||||
accessToken: string,
|
||||
id: string
|
||||
): Promise<GrainTranscriptSegment[] | null> {
|
||||
const response = await fetchWithRetry(`${GRAIN_API_BASE}/recordings/${id}/transcript`, {
|
||||
method: 'GET',
|
||||
headers: grainHeaders(accessToken),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) return null
|
||||
throw new Error(`Failed to fetch Grain transcript: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
return Array.isArray(data) ? (data as GrainTranscriptSegment[]) : []
|
||||
}
|
||||
|
||||
export const grainConnector: ConnectorConfig = {
|
||||
...grainConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const maxRecordings = sourceConfig.maxRecordings ? Number(sourceConfig.maxRecordings) : 0
|
||||
|
||||
const body: Record<string, unknown> = { include: RECORDING_INCLUDE }
|
||||
if (cursor) body.cursor = cursor
|
||||
|
||||
const cachedFilter = syncContext?.grainFilter as Record<string, unknown> | undefined | null
|
||||
const filter = cachedFilter !== undefined ? cachedFilter : buildRecordingFilter(sourceConfig)
|
||||
if (syncContext && cachedFilter === undefined) syncContext.grainFilter = filter ?? null
|
||||
if (filter) body.filter = filter
|
||||
|
||||
logger.info('Listing Grain recordings', {
|
||||
hasCursor: Boolean(cursor),
|
||||
hasFilter: Boolean(filter),
|
||||
})
|
||||
|
||||
const response = await fetchWithRetry(`${GRAIN_API_BASE}/recordings`, {
|
||||
method: 'POST',
|
||||
headers: grainHeaders(accessToken),
|
||||
body: JSON.stringify(body),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
logger.error('Failed to list Grain recordings', {
|
||||
status: response.status,
|
||||
error: errorText.slice(0, 500),
|
||||
})
|
||||
throw new Error(`Failed to list Grain recordings: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as GrainRecordingsListResponse
|
||||
const recordings = data.recordings ?? []
|
||||
const nextCursor = data.cursor?.trim() || undefined
|
||||
|
||||
const allDocuments: ExternalDocument[] = []
|
||||
for (const recording of recordings) {
|
||||
if (!recordingId(recording)) continue
|
||||
allDocuments.push(recordingToStub(recording))
|
||||
}
|
||||
|
||||
const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
|
||||
let documents = allDocuments
|
||||
if (maxRecordings > 0) {
|
||||
const remaining = Math.max(0, maxRecordings - prevFetched)
|
||||
if (allDocuments.length > remaining) {
|
||||
documents = allDocuments.slice(0, remaining)
|
||||
}
|
||||
}
|
||||
|
||||
const totalFetched = prevFetched + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
const hitLimit = maxRecordings > 0 && totalFetched >= maxRecordings
|
||||
if (hitLimit && syncContext) syncContext.listingCapped = true
|
||||
|
||||
const hasMore = !hitLimit && Boolean(nextCursor)
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore ? nextCursor : undefined,
|
||||
hasMore,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
_sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
try {
|
||||
if (!externalId) return null
|
||||
|
||||
const [recording, segments] = await Promise.all([
|
||||
fetchRecording(accessToken, externalId),
|
||||
fetchTranscript(accessToken, externalId),
|
||||
])
|
||||
if (!recording) return null
|
||||
if (!segments) return null
|
||||
|
||||
const hasTranscript = segments.some((segment) => segment.text?.trim())
|
||||
if (!hasTranscript) {
|
||||
logger.info('Transcript not yet available for Grain recording', { externalId })
|
||||
return null
|
||||
}
|
||||
|
||||
const content = formatTranscriptContent(recording, segments)
|
||||
|
||||
return {
|
||||
externalId,
|
||||
title: recordingTitle(recording),
|
||||
content,
|
||||
contentDeferred: false,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: recording.url || undefined,
|
||||
contentHash: buildContentHash(recording),
|
||||
metadata: buildMetadata(recording),
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Failed to get Grain recording', {
|
||||
externalId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const maxRecordings = sourceConfig.maxRecordings as string | undefined
|
||||
if (maxRecordings && (Number.isNaN(Number(maxRecordings)) || Number(maxRecordings) < 0)) {
|
||||
return { valid: false, error: 'Max recordings must be a non-negative number' }
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetchWithRetry(
|
||||
`${GRAIN_API_BASE}/recordings`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: grainHeaders(accessToken),
|
||||
body: JSON.stringify({}),
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
return {
|
||||
valid: false,
|
||||
error: `Grain access failed: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}`,
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.title === 'string' && metadata.title.trim()) {
|
||||
result.title = metadata.title
|
||||
}
|
||||
|
||||
const participants = joinTagArray(metadata.participants)
|
||||
if (participants) result.participants = participants
|
||||
|
||||
if (typeof metadata.source === 'string' && metadata.source.trim()) {
|
||||
result.source = metadata.source.trim()
|
||||
}
|
||||
|
||||
const labels = joinTagArray(metadata.labels)
|
||||
if (labels) result.labels = labels
|
||||
|
||||
const teams = joinTagArray(metadata.teams)
|
||||
if (teams) result.teams = teams
|
||||
|
||||
if (typeof metadata.meetingType === 'string' && metadata.meetingType.trim()) {
|
||||
result.meetingType = metadata.meetingType.trim()
|
||||
}
|
||||
|
||||
if (metadata.duration != null) {
|
||||
const num = Number(metadata.duration)
|
||||
if (!Number.isNaN(num)) result.duration = num
|
||||
}
|
||||
|
||||
const meetingDate = parseTagDate(metadata.meetingDate)
|
||||
if (meetingDate) result.meetingDate = meetingDate
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { grainConnector } from '@/connectors/grain/grain'
|
||||
@@ -0,0 +1,90 @@
|
||||
import { GrainIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const grainConnectorMeta: ConnectorMeta = {
|
||||
id: 'grain',
|
||||
name: 'Grain',
|
||||
description: 'Sync meeting recording transcripts from Grain',
|
||||
version: '1.0.0',
|
||||
icon: GrainIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'apiKey',
|
||||
label: 'API Key',
|
||||
placeholder: 'Enter your Grain API key',
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'maxRecordings',
|
||||
title: 'Max Recordings',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 200 (default: unlimited)',
|
||||
description: 'Cap the total number of recordings synced. Leave blank to sync all.',
|
||||
},
|
||||
{
|
||||
id: 'lookbackDays',
|
||||
title: 'Lookback Window (days)',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. 90 (default: all time)',
|
||||
description: 'Only sync recordings from the last N days. Leave blank to sync any age.',
|
||||
},
|
||||
{
|
||||
id: 'participantScope',
|
||||
title: 'Participant Scope',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
description:
|
||||
'Limit to internal-only meetings or meetings that include an external participant. Leave as Any to sync both.',
|
||||
options: [
|
||||
{ label: 'Any', id: '' },
|
||||
{ label: 'Internal only', id: 'internal' },
|
||||
{ label: 'External (has external participant)', id: 'external' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'titleSearch',
|
||||
title: 'Title Search',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. weekly standup',
|
||||
description: 'Only sync recordings whose title matches this text. Leave blank to sync all.',
|
||||
},
|
||||
{
|
||||
id: 'teamId',
|
||||
title: 'Team ID',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. a1b2c3d4-e5f6-7890-abcd-ef1234567890',
|
||||
description:
|
||||
'Only sync recordings belonging to this team (Grain team UUID). Leave blank to sync all teams.',
|
||||
},
|
||||
{
|
||||
id: 'meetingTypeId',
|
||||
title: 'Meeting Type ID',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. a1b2c3d4-e5f6-7890-abcd-ef1234567890',
|
||||
description:
|
||||
'Only sync recordings of this meeting type (Grain meeting type UUID). Leave blank to sync all types.',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'title', displayName: 'Title', fieldType: 'text' },
|
||||
{ id: 'participants', displayName: 'Participants', fieldType: 'text' },
|
||||
{ id: 'source', displayName: 'Source', fieldType: 'text' },
|
||||
{ id: 'labels', displayName: 'Labels', fieldType: 'text' },
|
||||
{ id: 'teams', displayName: 'Teams', fieldType: 'text' },
|
||||
{ id: 'meetingType', displayName: 'Meeting Type', fieldType: 'text' },
|
||||
{ id: 'duration', displayName: 'Duration (ms)', fieldType: 'number' },
|
||||
{ id: 'meetingDate', displayName: 'Meeting Date', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,465 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { granolaConnectorMeta } from '@/connectors/granola/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { htmlToPlainText, joinTagArray, parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('GranolaConnector')
|
||||
|
||||
const GRANOLA_API_BASE = 'https://public-api.granola.ai/v1'
|
||||
/** Granola caps page_size at 30; request the maximum to minimize round trips. */
|
||||
const PAGE_SIZE = 30
|
||||
|
||||
/** Granola folder identifiers match `fol_` followed by 14 alphanumeric chars. */
|
||||
const FOLDER_ID_PATTERN = /^fol_[a-zA-Z0-9]{14}$/
|
||||
|
||||
/**
|
||||
* A note owner or attendee as returned by the Granola API.
|
||||
*/
|
||||
interface GranolaUser {
|
||||
name: string | null
|
||||
email: string
|
||||
}
|
||||
|
||||
/**
|
||||
* The lightweight note shape returned by the list endpoint. It contains only
|
||||
* metadata — no summary or transcript content — so content must be fetched per
|
||||
* note via the get endpoint (the deferred-content pattern).
|
||||
*/
|
||||
interface GranolaNoteSummary {
|
||||
id: string
|
||||
object?: string
|
||||
title: string | null
|
||||
owner?: GranolaUser
|
||||
created_at: string
|
||||
updated_at: string
|
||||
}
|
||||
|
||||
/**
|
||||
* A folder the note belongs to, as returned by the get endpoint.
|
||||
*/
|
||||
interface GranolaFolderMembership {
|
||||
id: string
|
||||
name: string
|
||||
parent_folder_id?: string | null
|
||||
}
|
||||
|
||||
/**
|
||||
* Calendar event details attached to a note, when available.
|
||||
* Field names match the Granola API's CalendarEvent schema.
|
||||
*/
|
||||
interface GranolaCalendarEvent {
|
||||
event_title?: string | null
|
||||
organiser?: string | null
|
||||
calendar_event_id?: string | null
|
||||
scheduled_start_time?: string | null
|
||||
scheduled_end_time?: string | null
|
||||
invitees?: { email: string }[]
|
||||
}
|
||||
|
||||
/**
|
||||
* The full note shape returned by the get endpoint, including summary content.
|
||||
*/
|
||||
interface GranolaNoteDetail extends GranolaNoteSummary {
|
||||
web_url?: string | null
|
||||
calendar_event?: GranolaCalendarEvent | null
|
||||
attendees?: GranolaUser[]
|
||||
folder_membership?: GranolaFolderMembership[]
|
||||
summary_text?: string | null
|
||||
summary_markdown?: string | null
|
||||
}
|
||||
|
||||
/**
|
||||
* The list endpoint response envelope.
|
||||
*/
|
||||
interface GranolaListNotesResponse {
|
||||
notes?: GranolaNoteSummary[]
|
||||
hasMore?: boolean
|
||||
cursor?: string | null
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the authorization headers for a Granola API request.
|
||||
*/
|
||||
function granolaHeaders(accessToken: string): Record<string, string> {
|
||||
return {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Produces the change-detection hash for a note from its stable identifiers.
|
||||
* Granola exposes `updated_at`, which advances whenever the note (or its summary)
|
||||
* changes, so a metadata-only hash is sufficient and stays identical between the
|
||||
* list stub and the fetched document — letting the sync engine skip re-fetching
|
||||
* unchanged notes.
|
||||
*/
|
||||
function buildContentHash(id: string, updatedAt: string): string {
|
||||
return `granola:${id}:${updatedAt}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the optional `maxNotes` cap from source config.
|
||||
* Returns 0 (unlimited) when unset or invalid.
|
||||
*/
|
||||
function parseMaxNotes(sourceConfig: Record<string, unknown>): number {
|
||||
const raw = sourceConfig.maxNotes
|
||||
if (raw == null || raw === '') return 0
|
||||
const num = Number(raw)
|
||||
return Number.isFinite(num) && num > 0 ? Math.floor(num) : 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the optional `folderId` scope from source config. Returns a trimmed
|
||||
* folder id only when it matches Granola's `fol_…` identifier shape; otherwise
|
||||
* returns undefined so the request is not scoped to an invalid folder.
|
||||
*/
|
||||
function parseFolderId(sourceConfig: Record<string, unknown>): string | undefined {
|
||||
const raw = sourceConfig.folderId
|
||||
if (typeof raw !== 'string') return undefined
|
||||
const trimmed = raw.trim()
|
||||
if (!trimmed) return undefined
|
||||
return FOLDER_ID_PATTERN.test(trimmed) ? trimmed : undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses an optional ISO 8601 date filter from a named source-config field.
|
||||
* Returns a normalized ISO 8601 string when the value is a valid date; otherwise
|
||||
* returns undefined so the request is not scoped to an invalid date.
|
||||
*/
|
||||
function parseDateFilter(sourceConfig: Record<string, unknown>, key: string): string | undefined {
|
||||
const raw = sourceConfig[key]
|
||||
if (typeof raw !== 'string') return undefined
|
||||
const trimmed = raw.trim()
|
||||
if (!trimmed) return undefined
|
||||
const parsed = new Date(trimmed)
|
||||
return Number.isNaN(parsed.getTime()) ? undefined : parsed.toISOString()
|
||||
}
|
||||
|
||||
/**
|
||||
* Detects whether a string contains HTML markup. Granola returns markdown for
|
||||
* `summary_markdown`, but this guard lets us defensively strip tags if the API
|
||||
* ever emits HTML, without mangling legitimate markdown.
|
||||
*/
|
||||
function looksLikeHtml(value: string): boolean {
|
||||
return /<\/?[a-z][\s\S]*?>/i.test(value)
|
||||
}
|
||||
|
||||
/**
|
||||
* Assembles the document content from a note's title and summary. Prefers the
|
||||
* markdown summary, falling back to plain-text summary. HTML is stripped only
|
||||
* when detected so markdown formatting is preserved.
|
||||
*/
|
||||
function buildContent(note: GranolaNoteDetail): string {
|
||||
const parts: string[] = []
|
||||
|
||||
const title = note.title?.trim()
|
||||
if (title) parts.push(`# ${title}`)
|
||||
|
||||
const rawSummary = note.summary_markdown?.trim() || note.summary_text?.trim() || ''
|
||||
if (rawSummary) {
|
||||
parts.push('')
|
||||
parts.push(looksLikeHtml(rawSummary) ? htmlToPlainText(rawSummary) : rawSummary)
|
||||
}
|
||||
|
||||
return parts.join('\n').trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves an owner's display name, falling back to email, for tag mapping.
|
||||
*/
|
||||
function ownerDisplay(owner?: GranolaUser): string | undefined {
|
||||
if (!owner) return undefined
|
||||
return owner.name?.trim() || owner.email?.trim() || undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Collects attendee display names (falling back to email) for tag mapping.
|
||||
*/
|
||||
function collectAttendees(note: GranolaNoteDetail): string[] {
|
||||
if (!Array.isArray(note.attendees)) return []
|
||||
return note.attendees
|
||||
.map((a) => a.name?.trim() || a.email?.trim() || '')
|
||||
.filter((name): name is string => name.length > 0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Collects folder names for tag mapping.
|
||||
*/
|
||||
function collectFolders(note: GranolaNoteDetail): string[] {
|
||||
if (!Array.isArray(note.folder_membership)) return []
|
||||
return note.folder_membership
|
||||
.map((f) => f.name?.trim() || '')
|
||||
.filter((name): name is string => name.length > 0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the deferred stub for a note from list metadata. Content is empty and
|
||||
* fetched later via `getDocument` only for new/changed notes.
|
||||
*/
|
||||
function noteSummaryToStub(note: GranolaNoteSummary): ExternalDocument {
|
||||
return {
|
||||
externalId: note.id,
|
||||
title: note.title?.trim() || 'Untitled Note',
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/markdown',
|
||||
contentHash: buildContentHash(note.id, note.updated_at),
|
||||
metadata: {
|
||||
title: note.title?.trim() || undefined,
|
||||
owner: ownerDisplay(note.owner),
|
||||
ownerName: note.owner?.name ?? undefined,
|
||||
ownerEmail: note.owner?.email ?? undefined,
|
||||
noteDate: note.created_at,
|
||||
updatedAt: note.updated_at,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export const granolaConnector: ConnectorConfig = {
|
||||
...granolaConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>,
|
||||
lastSyncAt?: Date
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const maxNotes = parseMaxNotes(sourceConfig)
|
||||
const folderId = parseFolderId(sourceConfig)
|
||||
const createdAfter = parseDateFilter(sourceConfig, 'createdAfter')
|
||||
const createdBefore = parseDateFilter(sourceConfig, 'createdBefore')
|
||||
|
||||
const url = new URL(`${GRANOLA_API_BASE}/notes`)
|
||||
url.searchParams.set('page_size', String(PAGE_SIZE))
|
||||
if (cursor) url.searchParams.set('cursor', cursor)
|
||||
if (lastSyncAt) url.searchParams.set('updated_after', lastSyncAt.toISOString())
|
||||
if (folderId) url.searchParams.set('folder_id', folderId)
|
||||
if (createdAfter) url.searchParams.set('created_after', createdAfter)
|
||||
if (createdBefore) url.searchParams.set('created_before', createdBefore)
|
||||
|
||||
logger.info('Listing Granola notes', {
|
||||
hasCursor: Boolean(cursor),
|
||||
incremental: Boolean(lastSyncAt),
|
||||
scopedToFolder: Boolean(folderId),
|
||||
scopedByCreatedAfter: Boolean(createdAfter),
|
||||
scopedByCreatedBefore: Boolean(createdBefore),
|
||||
})
|
||||
|
||||
const response = await fetchWithRetry(url.toString(), {
|
||||
method: 'GET',
|
||||
headers: granolaHeaders(accessToken),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
logger.error('Failed to list Granola notes', {
|
||||
status: response.status,
|
||||
error: errorText.slice(0, 500),
|
||||
})
|
||||
throw new Error(`Failed to list Granola notes: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as GranolaListNotesResponse
|
||||
const notes = Array.isArray(data.notes) ? data.notes : []
|
||||
const nextCursor = data.cursor?.trim() || undefined
|
||||
|
||||
const allStubs = notes.filter((note) => Boolean(note.id)).map((note) => noteSummaryToStub(note))
|
||||
|
||||
const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
|
||||
let documents = allStubs
|
||||
if (maxNotes > 0) {
|
||||
const remaining = Math.max(0, maxNotes - prevFetched)
|
||||
if (allStubs.length > remaining) {
|
||||
documents = allStubs.slice(0, remaining)
|
||||
}
|
||||
}
|
||||
|
||||
const totalFetched = prevFetched + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
|
||||
const hitLimit = maxNotes > 0 && totalFetched >= maxNotes
|
||||
if (hitLimit && syncContext) syncContext.listingCapped = true
|
||||
|
||||
const hasMore = !hitLimit && Boolean(data.hasMore) && Boolean(nextCursor)
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore ? nextCursor : undefined,
|
||||
hasMore,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
_sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
try {
|
||||
if (!externalId) return null
|
||||
|
||||
const url = `${GRANOLA_API_BASE}/notes/${encodeURIComponent(externalId)}`
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: granolaHeaders(accessToken),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404 || response.status === 410) return null
|
||||
throw new Error(`Failed to fetch Granola note: ${response.status}`)
|
||||
}
|
||||
|
||||
const note = (await response.json()) as GranolaNoteDetail
|
||||
if (!note?.id) return null
|
||||
|
||||
const content = buildContent(note)
|
||||
if (!content) {
|
||||
logger.info('Granola note has no content', { externalId })
|
||||
return null
|
||||
}
|
||||
|
||||
const attendees = collectAttendees(note)
|
||||
const folders = collectFolders(note)
|
||||
const meeting = note.calendar_event?.event_title?.trim() || undefined
|
||||
const meetingDate = note.calendar_event?.scheduled_start_time?.trim() || undefined
|
||||
|
||||
return {
|
||||
externalId: note.id,
|
||||
title: note.title?.trim() || 'Untitled Note',
|
||||
content,
|
||||
contentDeferred: false,
|
||||
mimeType: 'text/markdown',
|
||||
sourceUrl: note.web_url?.trim() || undefined,
|
||||
contentHash: buildContentHash(note.id, note.updated_at),
|
||||
metadata: {
|
||||
title: note.title?.trim() || undefined,
|
||||
owner: ownerDisplay(note.owner),
|
||||
ownerName: note.owner?.name ?? undefined,
|
||||
ownerEmail: note.owner?.email ?? undefined,
|
||||
noteDate: note.created_at,
|
||||
updatedAt: note.updated_at,
|
||||
attendees,
|
||||
folders,
|
||||
meeting,
|
||||
meetingDate,
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Failed to get Granola note', {
|
||||
externalId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const maxNotes = sourceConfig.maxNotes as string | undefined
|
||||
if (maxNotes && (Number.isNaN(Number(maxNotes)) || Number(maxNotes) < 0)) {
|
||||
return { valid: false, error: 'Max notes must be a non-negative number' }
|
||||
}
|
||||
|
||||
const folderId = sourceConfig.folderId
|
||||
if (
|
||||
typeof folderId === 'string' &&
|
||||
folderId.trim() &&
|
||||
!FOLDER_ID_PATTERN.test(folderId.trim())
|
||||
) {
|
||||
return {
|
||||
valid: false,
|
||||
error:
|
||||
'Folder ID must look like fol_ followed by 14 alphanumeric characters (e.g. fol_4y6LduVdwSKC27)',
|
||||
}
|
||||
}
|
||||
|
||||
const createdAfter = sourceConfig.createdAfter
|
||||
if (
|
||||
typeof createdAfter === 'string' &&
|
||||
createdAfter.trim() &&
|
||||
Number.isNaN(new Date(createdAfter.trim()).getTime())
|
||||
) {
|
||||
return {
|
||||
valid: false,
|
||||
error:
|
||||
'Created After must be a valid date (ISO 8601, e.g. 2025-01-01 or 2025-01-01T00:00:00Z)',
|
||||
}
|
||||
}
|
||||
|
||||
const createdBefore = sourceConfig.createdBefore
|
||||
if (
|
||||
typeof createdBefore === 'string' &&
|
||||
createdBefore.trim() &&
|
||||
Number.isNaN(new Date(createdBefore.trim()).getTime())
|
||||
) {
|
||||
return {
|
||||
valid: false,
|
||||
error:
|
||||
'Created Before must be a valid date (ISO 8601, e.g. 2025-12-31 or 2025-12-31T23:59:59Z)',
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const url = new URL(`${GRANOLA_API_BASE}/notes`)
|
||||
url.searchParams.set('page_size', '1')
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
url.toString(),
|
||||
{
|
||||
method: 'GET',
|
||||
headers: granolaHeaders(accessToken),
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
return {
|
||||
valid: false,
|
||||
error: `Granola access failed: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}`,
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.title === 'string' && metadata.title.trim()) {
|
||||
result.title = metadata.title.trim()
|
||||
}
|
||||
|
||||
if (typeof metadata.owner === 'string' && metadata.owner.trim()) {
|
||||
result.owner = metadata.owner.trim()
|
||||
}
|
||||
|
||||
const attendees = joinTagArray(metadata.attendees)
|
||||
if (attendees) result.attendees = attendees
|
||||
|
||||
const folders = joinTagArray(metadata.folders)
|
||||
if (folders) result.folders = folders
|
||||
|
||||
if (typeof metadata.meeting === 'string' && metadata.meeting.trim()) {
|
||||
result.meeting = metadata.meeting.trim()
|
||||
}
|
||||
|
||||
const noteDate = parseTagDate(metadata.noteDate)
|
||||
if (noteDate) result.noteDate = noteDate
|
||||
|
||||
const meetingDate = parseTagDate(metadata.meetingDate)
|
||||
if (meetingDate) result.meetingDate = meetingDate
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { granolaConnector } from '@/connectors/granola/granola'
|
||||
@@ -0,0 +1,69 @@
|
||||
import { GranolaIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const granolaConnectorMeta: ConnectorMeta = {
|
||||
id: 'granola',
|
||||
name: 'Granola',
|
||||
description: 'Sync AI meeting notes and summaries from Granola',
|
||||
version: '1.0.0',
|
||||
icon: GranolaIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'apiKey',
|
||||
label: 'API Key',
|
||||
placeholder: 'Enter your Granola API key',
|
||||
},
|
||||
|
||||
supportsIncrementalSync: true,
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'maxNotes',
|
||||
title: 'Max Notes',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 200 (default: unlimited)',
|
||||
description: 'Cap the number of notes synced. Leave blank to sync all notes.',
|
||||
},
|
||||
{
|
||||
id: 'folderId',
|
||||
title: 'Folder ID',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. fol_4y6LduVdwSKC27',
|
||||
description:
|
||||
'Scope the sync to a single folder and its child folders. Leave blank to sync notes from all folders.',
|
||||
},
|
||||
{
|
||||
id: 'createdAfter',
|
||||
title: 'Created After',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. 2025-01-01 or 2025-01-01T00:00:00Z',
|
||||
description:
|
||||
'Only sync notes created on or after this date (ISO 8601). Leave blank to sync notes regardless of creation date.',
|
||||
},
|
||||
{
|
||||
id: 'createdBefore',
|
||||
title: 'Created Before',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. 2025-12-31 or 2025-12-31T23:59:59Z',
|
||||
description:
|
||||
'Only sync notes created on or before this date (ISO 8601). Leave blank to sync notes regardless of creation date.',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'title', displayName: 'Title', fieldType: 'text' },
|
||||
{ id: 'owner', displayName: 'Owner', fieldType: 'text' },
|
||||
{ id: 'attendees', displayName: 'Attendees', fieldType: 'text' },
|
||||
{ id: 'folders', displayName: 'Folders', fieldType: 'text' },
|
||||
{ id: 'meeting', displayName: 'Meeting', fieldType: 'text' },
|
||||
{ id: 'noteDate', displayName: 'Note Date', fieldType: 'date' },
|
||||
{ id: 'meetingDate', displayName: 'Meeting Date', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,671 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { greenhouseConnectorMeta } from '@/connectors/greenhouse/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { htmlToPlainText, parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('GreenhouseConnector')
|
||||
|
||||
const GREENHOUSE_API_BASE = 'https://harvest.greenhouse.io/v1'
|
||||
|
||||
/**
|
||||
* Upper bound on per-application scorecard requests during a single getDocument call.
|
||||
* A candidate can have many applications (e.g. internal-transfer tracking); without a
|
||||
* cap, one document fetch could fan out into dozens of sequential requests. Mirrors the
|
||||
* Ashby connector's MAX_APPLICATIONS_FOR_FEEDBACK bound.
|
||||
*/
|
||||
const MAX_APPLICATIONS_FOR_SCORECARDS = 10
|
||||
|
||||
/**
|
||||
* Greenhouse Harvest allows up to 500 candidates per page. We page through the
|
||||
* full list using the `page` query parameter and stop when the `Link` response
|
||||
* header no longer advertises a `rel="next"` relationship.
|
||||
*/
|
||||
const CANDIDATES_PER_PAGE = 500
|
||||
|
||||
/**
|
||||
* Minutes of overlap subtracted from `lastSyncAt` when computing the incremental
|
||||
* `updated_after` window. Catches candidates whose `updated_at` lands fractionally
|
||||
* before the recorded sync boundary (clock skew, late-committed writes) at the
|
||||
* cost of re-listing a small number of recently-touched candidates.
|
||||
*/
|
||||
const INCREMENTAL_OVERLAP_MINUTES = 5
|
||||
const MS_PER_MINUTE = 60 * 1000
|
||||
|
||||
interface GreenhouseUser {
|
||||
id?: number
|
||||
first_name?: string | null
|
||||
last_name?: string | null
|
||||
name?: string | null
|
||||
employee_id?: string | null
|
||||
}
|
||||
|
||||
interface GreenhouseEmailAddress {
|
||||
value?: string
|
||||
type?: string
|
||||
}
|
||||
|
||||
interface GreenhouseSource {
|
||||
id?: number
|
||||
public_name?: string | null
|
||||
}
|
||||
|
||||
interface GreenhouseApplication {
|
||||
id?: number
|
||||
status?: string | null
|
||||
applied_at?: string | null
|
||||
last_activity_at?: string | null
|
||||
source?: GreenhouseSource | null
|
||||
recruiter?: GreenhouseUser | null
|
||||
coordinator?: GreenhouseUser | null
|
||||
}
|
||||
|
||||
interface GreenhouseCandidate {
|
||||
id: number
|
||||
first_name?: string | null
|
||||
last_name?: string | null
|
||||
company?: string | null
|
||||
title?: string | null
|
||||
created_at?: string | null
|
||||
updated_at?: string | null
|
||||
last_activity?: string | null
|
||||
email_addresses?: GreenhouseEmailAddress[]
|
||||
tags?: string[]
|
||||
application_ids?: number[]
|
||||
applications?: GreenhouseApplication[]
|
||||
recruiter?: GreenhouseUser | null
|
||||
coordinator?: GreenhouseUser | null
|
||||
}
|
||||
|
||||
interface GreenhouseActivityNote {
|
||||
id?: number
|
||||
created_at?: string | null
|
||||
body?: string | null
|
||||
user?: GreenhouseUser | null
|
||||
visibility?: string | null
|
||||
}
|
||||
|
||||
interface GreenhouseActivityEmail {
|
||||
id?: number
|
||||
created_at?: string | null
|
||||
subject?: string | null
|
||||
body?: string | null
|
||||
to?: string | null
|
||||
from?: string | null
|
||||
user?: GreenhouseUser | null
|
||||
}
|
||||
|
||||
interface GreenhouseActivityEvent {
|
||||
id?: number
|
||||
created_at?: string | null
|
||||
subject?: string | null
|
||||
body?: string | null
|
||||
user?: GreenhouseUser | null
|
||||
}
|
||||
|
||||
interface GreenhouseActivityFeed {
|
||||
notes?: GreenhouseActivityNote[]
|
||||
emails?: GreenhouseActivityEmail[]
|
||||
activities?: GreenhouseActivityEvent[]
|
||||
}
|
||||
|
||||
interface GreenhouseScorecardAttribute {
|
||||
name?: string | null
|
||||
type?: string | null
|
||||
note?: string | null
|
||||
rating?: string | null
|
||||
}
|
||||
|
||||
interface GreenhouseScorecardQuestion {
|
||||
question?: string | null
|
||||
answer?: string | null
|
||||
}
|
||||
|
||||
interface GreenhouseScorecard {
|
||||
id?: number
|
||||
interview?: string | null
|
||||
interviewer?: GreenhouseUser | null
|
||||
submitted_by?: GreenhouseUser | null
|
||||
submitted_at?: string | null
|
||||
interviewed_at?: string | null
|
||||
overall_recommendation?: string | null
|
||||
attributes?: GreenhouseScorecardAttribute[]
|
||||
questions?: GreenhouseScorecardQuestion[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the HTTP Basic authorization header for Greenhouse Harvest. The API key
|
||||
* is used as the username with an empty password, base64-encoded as `apiKey:`.
|
||||
*/
|
||||
function buildAuthHeader(accessToken: string): string {
|
||||
return `Basic ${Buffer.from(`${accessToken}:`).toString('base64')}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the RFC 5988 `Link` response header and returns true when a
|
||||
* `rel="next"` relationship is present, indicating another page exists.
|
||||
*/
|
||||
function hasNextPage(linkHeader: string | null): boolean {
|
||||
if (!linkHeader) return false
|
||||
return /;\s*rel\s*=\s*"?next"?/i.test(linkHeader)
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a display name from a candidate's first and last name, falling back to
|
||||
* the candidate ID when both are missing.
|
||||
*/
|
||||
function candidateDisplayName(candidate: GreenhouseCandidate): string {
|
||||
const parts = [candidate.first_name, candidate.last_name]
|
||||
.map((part) => part?.trim())
|
||||
.filter((part): part is string => Boolean(part))
|
||||
return parts.length > 0 ? parts.join(' ') : `Candidate ${candidate.id}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the metadata-based content hash for a candidate. Both the listing stub
|
||||
* and `getDocument` use the same formula so the sync engine can detect changes
|
||||
* without downloading the deferred content. Greenhouse advances a candidate's
|
||||
* `updated_at` when the candidate record changes; profile-affecting activity
|
||||
* (notes, emails, stage changes, scorecard submissions) typically also touches it,
|
||||
* which is why `updated_after` listing and this hash track the same field.
|
||||
*/
|
||||
function buildContentHash(id: number, updatedAt?: string | null): string {
|
||||
return `greenhouse:${id}:${updatedAt ?? ''}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the source URL for a candidate in the Greenhouse recruiting UI.
|
||||
*/
|
||||
function buildSourceUrl(id: number): string {
|
||||
return `https://app.greenhouse.io/people/${id}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the recruiter, coordinator, and source for a candidate. Greenhouse
|
||||
* exposes these both at the candidate level and per-application; the candidate
|
||||
* level is preferred and the most-recent application is used as a fallback so
|
||||
* the tags are populated even when the candidate-level fields are empty.
|
||||
*/
|
||||
function resolveOwnership(candidate: GreenhouseCandidate): {
|
||||
recruiter?: string
|
||||
coordinator?: string
|
||||
source?: string
|
||||
} {
|
||||
const applications = Array.isArray(candidate.applications) ? candidate.applications : []
|
||||
const latest = applications.reduce<GreenhouseApplication | undefined>((acc, app) => {
|
||||
if (!acc) return app
|
||||
const accTime = acc.applied_at ? Date.parse(acc.applied_at) : 0
|
||||
const appTime = app.applied_at ? Date.parse(app.applied_at) : 0
|
||||
return appTime >= accTime ? app : acc
|
||||
}, undefined)
|
||||
|
||||
const recruiterName = userName(candidate.recruiter ?? latest?.recruiter)
|
||||
const coordinatorName = userName(candidate.coordinator ?? latest?.coordinator)
|
||||
const sourceName = latest?.source?.public_name?.trim()
|
||||
|
||||
return {
|
||||
recruiter: recruiterName !== 'Unknown' ? recruiterName : undefined,
|
||||
coordinator: coordinatorName !== 'Unknown' ? coordinatorName : undefined,
|
||||
source: sourceName || undefined,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the shared metadata block for a candidate, used by both the listing
|
||||
* stub and the fully-hydrated document.
|
||||
*/
|
||||
function buildMetadata(candidate: GreenhouseCandidate): Record<string, unknown> {
|
||||
const emails = (candidate.email_addresses ?? [])
|
||||
.map((e) => e.value?.trim())
|
||||
.filter((value): value is string => Boolean(value))
|
||||
|
||||
const { recruiter, coordinator, source } = resolveOwnership(candidate)
|
||||
const applicationCount = Array.isArray(candidate.application_ids)
|
||||
? candidate.application_ids.length
|
||||
: Array.isArray(candidate.applications)
|
||||
? candidate.applications.length
|
||||
: 0
|
||||
|
||||
return {
|
||||
candidateName: candidateDisplayName(candidate),
|
||||
company: candidate.company ?? undefined,
|
||||
title: candidate.title ?? undefined,
|
||||
emails,
|
||||
tags: Array.isArray(candidate.tags) ? candidate.tags : [],
|
||||
createdAt: candidate.created_at ?? undefined,
|
||||
updatedAt: candidate.updated_at ?? undefined,
|
||||
lastActivity: candidate.last_activity ?? undefined,
|
||||
recruiter,
|
||||
coordinator,
|
||||
source,
|
||||
applicationCount,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a lightweight, content-deferred document stub from a candidate listing
|
||||
* entry. The real content is fetched lazily via `getDocument` for new or changed
|
||||
* candidates only.
|
||||
*/
|
||||
function candidateToStub(candidate: GreenhouseCandidate): ExternalDocument {
|
||||
return {
|
||||
externalId: String(candidate.id),
|
||||
title: candidateDisplayName(candidate),
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: buildSourceUrl(candidate.id),
|
||||
contentHash: buildContentHash(candidate.id, candidate.updated_at),
|
||||
metadata: buildMetadata(candidate),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a user's display name for inclusion in content lines.
|
||||
*/
|
||||
function userName(user?: GreenhouseUser | null): string {
|
||||
if (!user) return 'Unknown'
|
||||
if (user.name?.trim()) return user.name.trim()
|
||||
const parts = [user.first_name, user.last_name]
|
||||
.map((part) => part?.trim())
|
||||
.filter((part): part is string => Boolean(part))
|
||||
return parts.length > 0 ? parts.join(' ') : 'Unknown'
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders a candidate's activity feed (notes, emails, and events) into plain text.
|
||||
*/
|
||||
function formatActivityFeed(feed: GreenhouseActivityFeed): string {
|
||||
const lines: string[] = []
|
||||
|
||||
const notes = feed.notes ?? []
|
||||
if (notes.length > 0) {
|
||||
lines.push('--- Notes ---')
|
||||
for (const note of notes) {
|
||||
const body = htmlToPlainText(note.body ?? '').trim()
|
||||
if (!body) continue
|
||||
const when = note.created_at ? ` (${note.created_at})` : ''
|
||||
lines.push(`[${userName(note.user)}${when}] ${body}`)
|
||||
}
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
const activities = feed.activities ?? []
|
||||
if (activities.length > 0) {
|
||||
lines.push('--- Activities ---')
|
||||
for (const activity of activities) {
|
||||
const body = htmlToPlainText(activity.body ?? '').trim()
|
||||
const subject = activity.subject?.trim()
|
||||
const text = [subject, body].filter(Boolean).join(': ')
|
||||
if (!text) continue
|
||||
const when = activity.created_at ? ` (${activity.created_at})` : ''
|
||||
lines.push(`[${userName(activity.user)}${when}] ${text}`)
|
||||
}
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
const emails = feed.emails ?? []
|
||||
if (emails.length > 0) {
|
||||
lines.push('--- Emails ---')
|
||||
for (const email of emails) {
|
||||
const body = htmlToPlainText(email.body ?? '').trim()
|
||||
const subject = email.subject?.trim()
|
||||
if (!subject && !body) continue
|
||||
const when = email.created_at ? ` (${email.created_at})` : ''
|
||||
if (subject) lines.push(`[${userName(email.user)}${when}] Subject: ${subject}`)
|
||||
if (body) lines.push(body)
|
||||
}
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
return lines.join('\n').trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders interview scorecards (recommendations, attribute ratings, and free-text
|
||||
* question feedback) into plain text.
|
||||
*/
|
||||
function formatScorecards(scorecards: GreenhouseScorecard[]): string {
|
||||
if (scorecards.length === 0) return ''
|
||||
|
||||
const lines: string[] = ['--- Scorecards ---']
|
||||
|
||||
for (const scorecard of scorecards) {
|
||||
const interview = scorecard.interview?.trim() || 'Interview'
|
||||
const reviewer = userName(scorecard.submitted_by ?? scorecard.interviewer)
|
||||
const when = scorecard.submitted_at ? ` (${scorecard.submitted_at})` : ''
|
||||
lines.push(`# ${interview} — ${reviewer}${when}`)
|
||||
|
||||
if (scorecard.overall_recommendation?.trim()) {
|
||||
lines.push(`Overall recommendation: ${scorecard.overall_recommendation.trim()}`)
|
||||
}
|
||||
|
||||
for (const attribute of scorecard.attributes ?? []) {
|
||||
const name = attribute.name?.trim()
|
||||
if (!name) continue
|
||||
const rating = attribute.rating?.trim()
|
||||
const note = htmlToPlainText(attribute.note ?? '').trim()
|
||||
const detail = [rating ? `rating: ${rating}` : '', note].filter(Boolean).join(' — ')
|
||||
lines.push(detail ? `- ${name}: ${detail}` : `- ${name}`)
|
||||
}
|
||||
|
||||
for (const question of scorecard.questions ?? []) {
|
||||
const q = htmlToPlainText(question.question ?? '').trim()
|
||||
const a = htmlToPlainText(question.answer ?? '').trim()
|
||||
if (!q && !a) continue
|
||||
lines.push(q ? `Q: ${q}` : 'Q:')
|
||||
if (a) lines.push(`A: ${a}`)
|
||||
}
|
||||
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
return lines.join('\n').trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Assembles the full document content for a candidate from their profile header,
|
||||
* activity feed, and interview scorecards.
|
||||
*/
|
||||
function formatContent(
|
||||
candidate: GreenhouseCandidate,
|
||||
feed: GreenhouseActivityFeed,
|
||||
scorecards: GreenhouseScorecard[]
|
||||
): string {
|
||||
const sections: string[] = []
|
||||
|
||||
const header: string[] = [`Candidate: ${candidateDisplayName(candidate)}`]
|
||||
if (candidate.title?.trim()) header.push(`Title: ${candidate.title.trim()}`)
|
||||
if (candidate.company?.trim()) header.push(`Company: ${candidate.company.trim()}`)
|
||||
const emails = (candidate.email_addresses ?? [])
|
||||
.map((e) => e.value?.trim())
|
||||
.filter((value): value is string => Boolean(value))
|
||||
if (emails.length > 0) header.push(`Email: ${emails.join(', ')}`)
|
||||
if (Array.isArray(candidate.tags) && candidate.tags.length > 0) {
|
||||
header.push(`Tags: ${candidate.tags.join(', ')}`)
|
||||
}
|
||||
sections.push(header.join('\n'))
|
||||
|
||||
const activity = formatActivityFeed(feed)
|
||||
if (activity) sections.push(activity)
|
||||
|
||||
const scorecardText = formatScorecards(scorecards)
|
||||
if (scorecardText) sections.push(scorecardText)
|
||||
|
||||
return sections.join('\n\n').trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches a single candidate by ID. Returns null on 404.
|
||||
*/
|
||||
async function fetchCandidate(
|
||||
accessToken: string,
|
||||
id: string
|
||||
): Promise<GreenhouseCandidate | null> {
|
||||
const response = await fetchWithRetry(`${GREENHOUSE_API_BASE}/candidates/${id}`, {
|
||||
method: 'GET',
|
||||
headers: { Authorization: buildAuthHeader(accessToken), Accept: 'application/json' },
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) return null
|
||||
throw new Error(`Failed to fetch Greenhouse candidate: ${response.status}`)
|
||||
}
|
||||
|
||||
return (await response.json()) as GreenhouseCandidate
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches a candidate's activity feed. Returns an empty feed when the endpoint
|
||||
* 404s so the candidate's profile header still produces content.
|
||||
*/
|
||||
async function fetchActivityFeed(accessToken: string, id: string): Promise<GreenhouseActivityFeed> {
|
||||
const response = await fetchWithRetry(`${GREENHOUSE_API_BASE}/candidates/${id}/activity_feed`, {
|
||||
method: 'GET',
|
||||
headers: { Authorization: buildAuthHeader(accessToken), Accept: 'application/json' },
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) return {}
|
||||
throw new Error(`Failed to fetch Greenhouse activity feed: ${response.status}`)
|
||||
}
|
||||
|
||||
return (await response.json()) as GreenhouseActivityFeed
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches all scorecards across a candidate's applications. Individual
|
||||
* application failures are tolerated so partial feedback is still indexed.
|
||||
*/
|
||||
async function fetchScorecards(
|
||||
accessToken: string,
|
||||
applicationIds: number[]
|
||||
): Promise<GreenhouseScorecard[]> {
|
||||
const all: GreenhouseScorecard[] = []
|
||||
|
||||
for (const applicationId of applicationIds.slice(0, MAX_APPLICATIONS_FOR_SCORECARDS)) {
|
||||
try {
|
||||
const response = await fetchWithRetry(
|
||||
`${GREENHOUSE_API_BASE}/applications/${applicationId}/scorecards`,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: { Authorization: buildAuthHeader(accessToken), Accept: 'application/json' },
|
||||
}
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) continue
|
||||
throw new Error(`Failed to fetch Greenhouse scorecards: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as GreenhouseScorecard[]
|
||||
if (Array.isArray(data)) all.push(...data)
|
||||
} catch (error) {
|
||||
logger.warn('Failed to fetch scorecards for application', {
|
||||
applicationId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return all
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the `updated_after` value for an incremental sync, subtracting a small
|
||||
* overlap from the last sync timestamp. Returns undefined for full syncs.
|
||||
*/
|
||||
function computeUpdatedAfter(lastSyncAt: Date | undefined): string | undefined {
|
||||
if (!lastSyncAt) return undefined
|
||||
const since = new Date(lastSyncAt.getTime() - INCREMENTAL_OVERLAP_MINUTES * MS_PER_MINUTE)
|
||||
return since.toISOString()
|
||||
}
|
||||
|
||||
export const greenhouseConnector: ConnectorConfig = {
|
||||
...greenhouseConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>,
|
||||
lastSyncAt?: Date
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const maxCandidates = sourceConfig.maxCandidates ? Number(sourceConfig.maxCandidates) : 0
|
||||
const parsedPage = cursor ? Number(cursor) : 1
|
||||
const page = Number.isFinite(parsedPage) && parsedPage > 0 ? Math.floor(parsedPage) : 1
|
||||
const updatedAfter = computeUpdatedAfter(lastSyncAt)
|
||||
const jobId = typeof sourceConfig.jobId === 'string' ? sourceConfig.jobId.trim() : ''
|
||||
const createdAfter =
|
||||
typeof sourceConfig.createdAfter === 'string' ? sourceConfig.createdAfter.trim() : ''
|
||||
const createdBefore =
|
||||
typeof sourceConfig.createdBefore === 'string' ? sourceConfig.createdBefore.trim() : ''
|
||||
|
||||
const queryParams = new URLSearchParams({
|
||||
per_page: String(CANDIDATES_PER_PAGE),
|
||||
page: String(page),
|
||||
})
|
||||
if (updatedAfter) queryParams.set('updated_after', updatedAfter)
|
||||
if (jobId) queryParams.set('job_id', jobId)
|
||||
if (createdAfter) queryParams.set('created_after', createdAfter)
|
||||
if (createdBefore) queryParams.set('created_before', createdBefore)
|
||||
|
||||
const url = `${GREENHOUSE_API_BASE}/candidates?${queryParams.toString()}`
|
||||
|
||||
logger.info('Listing Greenhouse candidates', {
|
||||
page,
|
||||
perPage: CANDIDATES_PER_PAGE,
|
||||
incremental: Boolean(updatedAfter),
|
||||
})
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: { Authorization: buildAuthHeader(accessToken), Accept: 'application/json' },
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
logger.error('Failed to list Greenhouse candidates', {
|
||||
status: response.status,
|
||||
error: errorText.slice(0, 500),
|
||||
})
|
||||
throw new Error(`Failed to list Greenhouse candidates: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as GreenhouseCandidate[]
|
||||
const candidates = Array.isArray(data) ? data : []
|
||||
const linkHasNext = hasNextPage(response.headers.get('link'))
|
||||
|
||||
const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
|
||||
let pageCandidates = candidates
|
||||
if (maxCandidates > 0) {
|
||||
const remaining = Math.max(0, maxCandidates - prevFetched)
|
||||
if (pageCandidates.length > remaining) {
|
||||
pageCandidates = pageCandidates.slice(0, remaining)
|
||||
}
|
||||
}
|
||||
|
||||
const documents = pageCandidates.map(candidateToStub)
|
||||
|
||||
const totalFetched = prevFetched + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
const hitLimit = maxCandidates > 0 && totalFetched >= maxCandidates
|
||||
if (hitLimit && syncContext) syncContext.listingCapped = true
|
||||
|
||||
const hasMore = !hitLimit && linkHasNext
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore ? String(page + 1) : undefined,
|
||||
hasMore,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
_sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
try {
|
||||
if (!externalId) return null
|
||||
|
||||
const candidate = await fetchCandidate(accessToken, externalId)
|
||||
if (!candidate) return null
|
||||
|
||||
const applicationIds = Array.isArray(candidate.application_ids)
|
||||
? candidate.application_ids
|
||||
: []
|
||||
|
||||
const [feed, scorecards] = await Promise.all([
|
||||
fetchActivityFeed(accessToken, externalId),
|
||||
fetchScorecards(accessToken, applicationIds),
|
||||
])
|
||||
|
||||
const content = formatContent(candidate, feed, scorecards)
|
||||
if (!content.trim()) return null
|
||||
|
||||
return {
|
||||
externalId: String(candidate.id),
|
||||
title: candidateDisplayName(candidate),
|
||||
content,
|
||||
contentDeferred: false,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: buildSourceUrl(candidate.id),
|
||||
contentHash: buildContentHash(candidate.id, candidate.updated_at),
|
||||
metadata: buildMetadata(candidate),
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Failed to get Greenhouse candidate', {
|
||||
externalId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const maxCandidates = sourceConfig.maxCandidates as string | undefined
|
||||
if (maxCandidates && (Number.isNaN(Number(maxCandidates)) || Number(maxCandidates) < 0)) {
|
||||
return { valid: false, error: 'Max candidates must be a non-negative number' }
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetchWithRetry(
|
||||
`${GREENHOUSE_API_BASE}/candidates?per_page=1`,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: { Authorization: buildAuthHeader(accessToken), Accept: 'application/json' },
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
return {
|
||||
valid: false,
|
||||
error: `Greenhouse access failed: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}`,
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
const textFields = [
|
||||
'candidateName',
|
||||
'company',
|
||||
'title',
|
||||
'recruiter',
|
||||
'coordinator',
|
||||
'source',
|
||||
] as const
|
||||
for (const field of textFields) {
|
||||
const value = metadata[field]
|
||||
if (typeof value === 'string' && value.trim()) {
|
||||
result[field] = value.trim()
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof metadata.applicationCount === 'number' && metadata.applicationCount >= 0) {
|
||||
result.applicationCount = metadata.applicationCount
|
||||
}
|
||||
|
||||
const dateFields = ['updatedAt', 'lastActivity'] as const
|
||||
for (const field of dateFields) {
|
||||
const parsed = parseTagDate(metadata[field])
|
||||
if (parsed) result[field] = parsed
|
||||
}
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { greenhouseConnector } from '@/connectors/greenhouse/greenhouse'
|
||||
@@ -0,0 +1,72 @@
|
||||
import { GreenhouseIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const greenhouseConnectorMeta: ConnectorMeta = {
|
||||
id: 'greenhouse',
|
||||
name: 'Greenhouse',
|
||||
description: 'Sync candidate activity and interview scorecards from Greenhouse',
|
||||
version: '1.0.0',
|
||||
icon: GreenhouseIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'apiKey',
|
||||
label: 'API Key',
|
||||
placeholder: 'Enter your Greenhouse Harvest API key',
|
||||
},
|
||||
|
||||
supportsIncrementalSync: true,
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'maxCandidates',
|
||||
title: 'Max Candidates',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 500 (default: unlimited)',
|
||||
description:
|
||||
'Cap the number of candidates synced. Leave empty to sync ALL candidates in the organization.',
|
||||
},
|
||||
{
|
||||
id: 'jobId',
|
||||
title: 'Job ID',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. 123456',
|
||||
description:
|
||||
'Sync only candidates who applied to this Greenhouse job. Leave empty to sync candidates across all jobs.',
|
||||
},
|
||||
{
|
||||
id: 'createdAfter',
|
||||
title: 'Created After',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. 2024-01-01T00:00:00Z',
|
||||
description:
|
||||
'Sync only candidates created at or after this ISO 8601 timestamp. Leave empty to sync candidates regardless of creation date.',
|
||||
},
|
||||
{
|
||||
id: 'createdBefore',
|
||||
title: 'Created Before',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. 2024-12-31T23:59:59Z',
|
||||
description:
|
||||
'Sync only candidates created before this ISO 8601 timestamp. Combine with Created After to backfill a bounded date range.',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'candidateName', displayName: 'Candidate Name', fieldType: 'text' },
|
||||
{ id: 'company', displayName: 'Company', fieldType: 'text' },
|
||||
{ id: 'title', displayName: 'Title', fieldType: 'text' },
|
||||
{ id: 'recruiter', displayName: 'Recruiter', fieldType: 'text' },
|
||||
{ id: 'coordinator', displayName: 'Coordinator', fieldType: 'text' },
|
||||
{ id: 'source', displayName: 'Source', fieldType: 'text' },
|
||||
{ id: 'applicationCount', displayName: 'Application Count', fieldType: 'number' },
|
||||
{ id: 'updatedAt', displayName: 'Last Updated', fieldType: 'date' },
|
||||
{ id: 'lastActivity', displayName: 'Last Activity', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { hubspotConnectorMeta } from '@/connectors/hubspot/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('HubSpotConnector')
|
||||
|
||||
const BASE_URL = 'https://api.hubapi.com'
|
||||
const PAGE_SIZE = 100
|
||||
|
||||
/** Properties to fetch per object type. */
|
||||
const OBJECT_PROPERTIES: Record<string, string[]> = {
|
||||
contacts: [
|
||||
'firstname',
|
||||
'lastname',
|
||||
'email',
|
||||
'phone',
|
||||
'company',
|
||||
'jobtitle',
|
||||
'lifecyclestage',
|
||||
'hs_lead_status',
|
||||
'hubspot_owner_id',
|
||||
'createdate',
|
||||
'lastmodifieddate',
|
||||
],
|
||||
companies: [
|
||||
'name',
|
||||
'domain',
|
||||
'industry',
|
||||
'description',
|
||||
'phone',
|
||||
'city',
|
||||
'state',
|
||||
'country',
|
||||
'numberofemployees',
|
||||
'annualrevenue',
|
||||
'hubspot_owner_id',
|
||||
'createdate',
|
||||
'hs_lastmodifieddate',
|
||||
],
|
||||
deals: [
|
||||
'dealname',
|
||||
'amount',
|
||||
'dealstage',
|
||||
'pipeline',
|
||||
'closedate',
|
||||
'hubspot_owner_id',
|
||||
'createdate',
|
||||
'hs_lastmodifieddate',
|
||||
],
|
||||
tickets: [
|
||||
'subject',
|
||||
'content',
|
||||
'hs_pipeline',
|
||||
'hs_pipeline_stage',
|
||||
'hs_ticket_priority',
|
||||
'hubspot_owner_id',
|
||||
'createdate',
|
||||
'hs_lastmodifieddate',
|
||||
],
|
||||
} as const
|
||||
|
||||
/**
|
||||
* Fetches the HubSpot portal ID for the authenticated account.
|
||||
* Caches the result in syncContext to avoid repeated calls.
|
||||
*/
|
||||
async function getPortalId(
|
||||
accessToken: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<string> {
|
||||
if (syncContext?.portalId) {
|
||||
return syncContext.portalId as string
|
||||
}
|
||||
|
||||
const response = await fetchWithRetry(`${BASE_URL}/account-info/v3/details`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
throw new Error(`Failed to fetch HubSpot portal ID: ${response.status} - ${errorText}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const portalId = String(data.portalId)
|
||||
|
||||
if (syncContext) {
|
||||
syncContext.portalId = portalId
|
||||
}
|
||||
|
||||
return portalId
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the document title for a CRM record based on its object type.
|
||||
*/
|
||||
function buildRecordTitle(objectType: string, properties: Record<string, string | null>): string {
|
||||
switch (objectType) {
|
||||
case 'contacts': {
|
||||
const first = properties.firstname || ''
|
||||
const last = properties.lastname || ''
|
||||
const name = `${first} ${last}`.trim()
|
||||
return name || properties.email || 'Unnamed Contact'
|
||||
}
|
||||
case 'companies':
|
||||
return properties.name || properties.domain || 'Unnamed Company'
|
||||
case 'deals':
|
||||
return properties.dealname || 'Unnamed Deal'
|
||||
case 'tickets':
|
||||
return properties.subject || 'Unnamed Ticket'
|
||||
default:
|
||||
return `Record ${properties.hs_object_id || 'Unknown'}`
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a plain-text representation of a CRM record's properties for indexing.
|
||||
*/
|
||||
function buildRecordContent(objectType: string, properties: Record<string, string | null>): string {
|
||||
const parts: string[] = []
|
||||
|
||||
const title = buildRecordTitle(objectType, properties)
|
||||
parts.push(title)
|
||||
|
||||
for (const [key, value] of Object.entries(properties)) {
|
||||
if (value && key !== 'hs_object_id') {
|
||||
const label = key.replace(/_/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase())
|
||||
parts.push(`${label}: ${value}`)
|
||||
}
|
||||
}
|
||||
|
||||
return parts.join('\n').trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a HubSpot CRM record to an ExternalDocument.
|
||||
*/
|
||||
function recordToDocument(
|
||||
record: Record<string, unknown>,
|
||||
objectType: string,
|
||||
portalId: string
|
||||
): ExternalDocument {
|
||||
const id = record.id as string
|
||||
const properties = (record.properties || {}) as Record<string, string | null>
|
||||
|
||||
const content = buildRecordContent(objectType, properties)
|
||||
const title = buildRecordTitle(objectType, properties)
|
||||
|
||||
const lastModified =
|
||||
properties.lastmodifieddate || properties.hs_lastmodifieddate || properties.createdate
|
||||
|
||||
return {
|
||||
externalId: id,
|
||||
title,
|
||||
content,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: `https://app.hubspot.com/contacts/${portalId}/record/${objectType}/${id}`,
|
||||
contentHash: `hubspot:${id}:${lastModified ?? ''}`,
|
||||
metadata: {
|
||||
objectType,
|
||||
owner: properties.hubspot_owner_id || undefined,
|
||||
lastModified: lastModified || undefined,
|
||||
pipeline: properties.pipeline || properties.hs_pipeline || undefined,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export const hubspotConnector: ConnectorConfig = {
|
||||
...hubspotConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const objectType = sourceConfig.objectType as string
|
||||
const maxRecords = sourceConfig.maxRecords ? Number(sourceConfig.maxRecords) : 0
|
||||
const properties = OBJECT_PROPERTIES[objectType] || []
|
||||
|
||||
const portalId = await getPortalId(accessToken, syncContext)
|
||||
|
||||
const sortProperty = objectType === 'contacts' ? 'lastmodifieddate' : 'hs_lastmodifieddate'
|
||||
|
||||
const searchBody: Record<string, unknown> = {
|
||||
properties,
|
||||
sorts: [{ propertyName: sortProperty, direction: 'DESCENDING' }],
|
||||
limit: PAGE_SIZE,
|
||||
}
|
||||
if (cursor) {
|
||||
searchBody.after = cursor
|
||||
}
|
||||
|
||||
logger.info(`Listing HubSpot ${objectType}`, { cursor })
|
||||
|
||||
const response = await fetchWithRetry(`${BASE_URL}/crm/v3/objects/${objectType}/search`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Accept: 'application/json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
body: JSON.stringify(searchBody),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error(`Failed to list HubSpot ${objectType}`, {
|
||||
status: response.status,
|
||||
error: errorText,
|
||||
})
|
||||
throw new Error(`Failed to list HubSpot ${objectType}: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const results = (data.results || []) as Record<string, unknown>[]
|
||||
const paging = data.paging as { next?: { after?: string } } | undefined
|
||||
const nextCursor = paging?.next?.after
|
||||
|
||||
const documents: ExternalDocument[] = results.map((record) =>
|
||||
recordToDocument(record, objectType, portalId)
|
||||
)
|
||||
|
||||
const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
|
||||
if (maxRecords > 0) {
|
||||
const remaining = maxRecords - previouslyFetched
|
||||
if (documents.length > remaining) {
|
||||
documents.splice(remaining)
|
||||
}
|
||||
}
|
||||
|
||||
const totalFetched = previouslyFetched + documents.length
|
||||
if (syncContext) {
|
||||
syncContext.totalDocsFetched = totalFetched
|
||||
}
|
||||
|
||||
const hasMore = Boolean(nextCursor) && (maxRecords <= 0 || totalFetched < maxRecords)
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore ? nextCursor : undefined,
|
||||
hasMore,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
externalId: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocument | null> => {
|
||||
const objectType = sourceConfig.objectType as string
|
||||
const properties = OBJECT_PROPERTIES[objectType] || []
|
||||
|
||||
let portalId = syncContext?.portalId as string | undefined
|
||||
if (!portalId) {
|
||||
portalId = await getPortalId(accessToken)
|
||||
if (syncContext) syncContext.portalId = portalId
|
||||
}
|
||||
|
||||
const params = new URLSearchParams()
|
||||
for (const prop of properties) {
|
||||
params.append('properties', prop)
|
||||
}
|
||||
|
||||
const url = `${BASE_URL}/crm/v3/objects/${objectType}/${externalId}?${params.toString()}`
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) return null
|
||||
throw new Error(`Failed to get HubSpot ${objectType} record: ${response.status}`)
|
||||
}
|
||||
|
||||
const record = await response.json()
|
||||
return recordToDocument(record as Record<string, unknown>, objectType, portalId)
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const objectType = sourceConfig.objectType as string
|
||||
|
||||
if (!objectType) {
|
||||
return { valid: false, error: 'Object type is required' }
|
||||
}
|
||||
|
||||
if (!OBJECT_PROPERTIES[objectType]) {
|
||||
return { valid: false, error: `Unsupported object type: ${objectType}` }
|
||||
}
|
||||
|
||||
const maxRecords = sourceConfig.maxRecords as string | undefined
|
||||
if (maxRecords && (Number.isNaN(Number(maxRecords)) || Number(maxRecords) <= 0)) {
|
||||
return { valid: false, error: 'Max records must be a positive number' }
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetchWithRetry(
|
||||
`${BASE_URL}/crm/v3/objects/${objectType}?limit=1`,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
return {
|
||||
valid: false,
|
||||
error: `Failed to access HubSpot ${objectType}: ${response.status} - ${errorText}`,
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.objectType === 'string') result.objectType = metadata.objectType
|
||||
if (typeof metadata.owner === 'string') result.owner = metadata.owner
|
||||
|
||||
const lastModified = parseTagDate(metadata.lastModified)
|
||||
if (lastModified) result.lastModified = lastModified
|
||||
|
||||
if (typeof metadata.pipeline === 'string') result.pipeline = metadata.pipeline
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { hubspotConnector } from '@/connectors/hubspot/hubspot'
|
||||
@@ -0,0 +1,50 @@
|
||||
import { HubspotIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const hubspotConnectorMeta: ConnectorMeta = {
|
||||
id: 'hubspot',
|
||||
name: 'HubSpot',
|
||||
description: 'Sync CRM records from HubSpot',
|
||||
version: '1.0.0',
|
||||
icon: HubspotIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'oauth',
|
||||
provider: 'hubspot',
|
||||
requiredScopes: [
|
||||
'crm.objects.contacts.read',
|
||||
'crm.objects.companies.read',
|
||||
'crm.objects.deals.read',
|
||||
'tickets',
|
||||
],
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'objectType',
|
||||
title: 'Object Type',
|
||||
type: 'dropdown',
|
||||
required: true,
|
||||
options: [
|
||||
{ label: 'Contacts', id: 'contacts' },
|
||||
{ label: 'Companies', id: 'companies' },
|
||||
{ label: 'Deals', id: 'deals' },
|
||||
{ label: 'Tickets', id: 'tickets' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'maxRecords',
|
||||
title: 'Max Records',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 500 (default: unlimited)',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'objectType', displayName: 'Object Type', fieldType: 'text' },
|
||||
{ id: 'owner', displayName: 'Owner', fieldType: 'text' },
|
||||
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
|
||||
{ id: 'pipeline', displayName: 'Pipeline', fieldType: 'text' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,553 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { incidentioConnectorMeta } from '@/connectors/incidentio/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { htmlToPlainText, parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('IncidentioConnector')
|
||||
|
||||
const INCIDENTIO_API_BASE = 'https://api.incident.io'
|
||||
const PAGE_SIZE = 100
|
||||
/** Cap incident updates fetched per document so a noisy incident can't blow the time budget. */
|
||||
const MAX_UPDATES_PER_INCIDENT = 100
|
||||
|
||||
interface IncidentioNamedRef {
|
||||
id?: string
|
||||
name?: string
|
||||
/** Present on incident_status: the stable status category enum (triage/live/closed/...). */
|
||||
category?: string
|
||||
rank?: number
|
||||
}
|
||||
|
||||
interface IncidentioCustomFieldValue {
|
||||
value_text?: string
|
||||
value_link?: string
|
||||
value_numeric?: string
|
||||
value_option?: { value?: string }
|
||||
value_catalog_entry?: { name?: string }
|
||||
}
|
||||
|
||||
interface IncidentioCustomFieldEntry {
|
||||
custom_field?: { id?: string; name?: string; field_type?: string }
|
||||
values?: IncidentioCustomFieldValue[]
|
||||
}
|
||||
|
||||
interface IncidentioRoleAssignment {
|
||||
role?: { id?: string; name?: string; role_type?: string }
|
||||
assignee?: { id?: string; name?: string; email?: string }
|
||||
}
|
||||
|
||||
interface IncidentioTimestampValue {
|
||||
incident_timestamp?: { id?: string; name?: string; rank?: number }
|
||||
/**
|
||||
* The v2 API nests the timestamp string under `value.value` (an object), not a
|
||||
* flat string. A flat string is tolerated defensively for older shapes.
|
||||
*/
|
||||
value?: { value?: string } | string
|
||||
}
|
||||
|
||||
interface IncidentioIncident {
|
||||
id: string
|
||||
reference?: string
|
||||
name?: string
|
||||
summary?: string
|
||||
mode?: string
|
||||
visibility?: string
|
||||
permalink?: string
|
||||
call_url?: string
|
||||
created_at?: string
|
||||
updated_at?: string
|
||||
slack_channel_id?: string
|
||||
slack_channel_name?: string
|
||||
severity?: IncidentioNamedRef
|
||||
incident_status?: IncidentioNamedRef
|
||||
incident_type?: IncidentioNamedRef
|
||||
custom_field_entries?: IncidentioCustomFieldEntry[]
|
||||
incident_role_assignments?: IncidentioRoleAssignment[]
|
||||
incident_timestamp_values?: IncidentioTimestampValue[]
|
||||
}
|
||||
|
||||
interface IncidentioPaginationMeta {
|
||||
after?: string
|
||||
page_size?: number
|
||||
total_record_count?: number
|
||||
}
|
||||
|
||||
interface IncidentioIncidentsListResponse {
|
||||
incidents?: IncidentioIncident[]
|
||||
pagination_meta?: IncidentioPaginationMeta
|
||||
}
|
||||
|
||||
interface IncidentioIncidentShowResponse {
|
||||
incident?: IncidentioIncident
|
||||
}
|
||||
|
||||
/**
|
||||
* incident.io ActorV2: the entity behind an action, used for incident update
|
||||
* authors. The v2 API documents this as an ActorV2 with exactly one of the
|
||||
* nested `user`/`api_key`/`workflow`/`alert` variants populated. A flat
|
||||
* `{ name, email }` shape is also tolerated defensively; we read whichever is
|
||||
* present so the author resolves regardless of which form the API returns.
|
||||
*/
|
||||
interface IncidentioActor {
|
||||
user?: { id?: string; name?: string; email?: string }
|
||||
api_key?: { id?: string; name?: string }
|
||||
alert?: { id?: string; title?: string }
|
||||
workflow?: { id?: string; name?: string }
|
||||
name?: string
|
||||
email?: string
|
||||
}
|
||||
|
||||
interface IncidentioUpdate {
|
||||
id?: string
|
||||
message?: string
|
||||
new_severity?: IncidentioNamedRef
|
||||
/**
|
||||
* The status change on an update. The v2 API documents `new_incident_status`,
|
||||
* but a flat `new_status` shape also appears in the wild; both are read.
|
||||
*/
|
||||
new_incident_status?: IncidentioNamedRef
|
||||
new_status?: IncidentioNamedRef
|
||||
updater?: IncidentioActor
|
||||
/** Some responses use `author` instead of `updater`; both are read. */
|
||||
author?: IncidentioActor
|
||||
created_at?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a human-readable display name for an update author, covering the
|
||||
* ActorV2 variants and the flat `{ name, email }` shape.
|
||||
*/
|
||||
function actorName(actor: IncidentioActor | undefined): string | undefined {
|
||||
if (!actor) return undefined
|
||||
return (
|
||||
actor.user?.name ||
|
||||
actor.user?.email ||
|
||||
actor.api_key?.name ||
|
||||
actor.workflow?.name ||
|
||||
actor.alert?.title ||
|
||||
actor.name ||
|
||||
actor.email ||
|
||||
undefined
|
||||
)
|
||||
}
|
||||
|
||||
interface IncidentioUpdatesListResponse {
|
||||
incident_updates?: IncidentioUpdate[]
|
||||
pagination_meta?: IncidentioPaginationMeta
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the metadata-based content hash for an incident.
|
||||
*
|
||||
* Uses only the incident `updated_at` timestamp, which incident.io bumps whenever any
|
||||
* field, role assignment, custom field, or status changes. This guarantees the hash is
|
||||
* identical whether produced from the list stub or the fully-hydrated getDocument result,
|
||||
* so the sync engine can detect changes without downloading content.
|
||||
*/
|
||||
function buildContentHash(incident: IncidentioIncident): string {
|
||||
return `incidentio:${incident.id}:${incident.updated_at ?? ''}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the public URL for an incident, preferring the API-provided permalink.
|
||||
*/
|
||||
function buildSourceUrl(incident: IncidentioIncident): string | undefined {
|
||||
return incident.permalink || undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Derives the document title from the incident reference and name.
|
||||
*/
|
||||
function buildTitle(incident: IncidentioIncident): string {
|
||||
const name = incident.name?.trim()
|
||||
const reference = incident.reference?.trim()
|
||||
if (reference && name) return `${reference}: ${name}`
|
||||
return name || reference || 'Untitled Incident'
|
||||
}
|
||||
|
||||
/**
|
||||
* Collects the source-specific metadata fed to mapTags. Shared between the list stub
|
||||
* and getDocument so tag values stay consistent regardless of which path produced the doc.
|
||||
*/
|
||||
function buildMetadata(incident: IncidentioIncident): Record<string, unknown> {
|
||||
const lead = incident.incident_role_assignments?.find(
|
||||
(assignment) => assignment.role?.role_type === 'lead'
|
||||
)
|
||||
const reporter = incident.incident_role_assignments?.find(
|
||||
(assignment) => assignment.role?.role_type === 'reporter'
|
||||
)
|
||||
const reportedBy = (reporter ?? lead)?.assignee?.name
|
||||
|
||||
return {
|
||||
reference: incident.reference,
|
||||
status: incident.incident_status?.name,
|
||||
statusCategory: incident.incident_status?.category,
|
||||
severity: incident.severity?.name,
|
||||
incidentType: incident.incident_type?.name,
|
||||
mode: incident.mode,
|
||||
visibility: incident.visibility,
|
||||
incidentDate: incident.created_at,
|
||||
reportedBy,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders a single custom field value to a human-readable string, covering each of
|
||||
* incident.io's value variants (text, link, numeric, select option, catalog entry).
|
||||
*/
|
||||
function renderCustomFieldValue(value: IncidentioCustomFieldValue): string | undefined {
|
||||
if (value.value_text) return htmlToPlainText(value.value_text)
|
||||
if (value.value_link) return value.value_link
|
||||
if (value.value_numeric) return value.value_numeric
|
||||
if (value.value_option?.value) return value.value_option.value
|
||||
if (value.value_catalog_entry?.name) return value.value_catalog_entry.name
|
||||
return undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats the incident, its custom fields, role assignments, timeline, and status
|
||||
* updates into a single plain-text document. HTML in summary / custom field text is
|
||||
* stripped via htmlToPlainText.
|
||||
*/
|
||||
function formatIncidentContent(incident: IncidentioIncident, updates: IncidentioUpdate[]): string {
|
||||
const parts: string[] = []
|
||||
|
||||
parts.push(`Incident: ${buildTitle(incident)}`)
|
||||
if (incident.incident_status?.name) parts.push(`Status: ${incident.incident_status.name}`)
|
||||
if (incident.severity?.name) parts.push(`Severity: ${incident.severity.name}`)
|
||||
if (incident.incident_type?.name) parts.push(`Type: ${incident.incident_type.name}`)
|
||||
if (incident.created_at) parts.push(`Reported: ${incident.created_at}`)
|
||||
|
||||
if (incident.summary?.trim()) {
|
||||
parts.push('')
|
||||
parts.push('--- Summary ---')
|
||||
parts.push(htmlToPlainText(incident.summary))
|
||||
}
|
||||
|
||||
const roleLines = (incident.incident_role_assignments ?? [])
|
||||
.map((assignment) => {
|
||||
const role = assignment.role?.name
|
||||
const assignee = assignment.assignee?.name || assignment.assignee?.email
|
||||
if (!role || !assignee) return undefined
|
||||
return `${role}: ${assignee}`
|
||||
})
|
||||
.filter((line): line is string => Boolean(line))
|
||||
if (roleLines.length > 0) {
|
||||
parts.push('')
|
||||
parts.push('--- Roles ---')
|
||||
parts.push(...roleLines)
|
||||
}
|
||||
|
||||
const customFieldLines = (incident.custom_field_entries ?? [])
|
||||
.map((entry) => {
|
||||
const name = entry.custom_field?.name
|
||||
if (!name) return undefined
|
||||
const rendered = (entry.values ?? [])
|
||||
.map(renderCustomFieldValue)
|
||||
.filter((v): v is string => Boolean(v))
|
||||
if (rendered.length === 0) return undefined
|
||||
return `${name}: ${rendered.join(', ')}`
|
||||
})
|
||||
.filter((line): line is string => Boolean(line))
|
||||
if (customFieldLines.length > 0) {
|
||||
parts.push('')
|
||||
parts.push('--- Custom Fields ---')
|
||||
parts.push(...customFieldLines)
|
||||
}
|
||||
|
||||
const timestampLines = (incident.incident_timestamp_values ?? [])
|
||||
.map((entry) => {
|
||||
const name = entry.incident_timestamp?.name
|
||||
const value = typeof entry.value === 'string' ? entry.value : entry.value?.value
|
||||
if (!name || !value) return undefined
|
||||
return `${name}: ${value}`
|
||||
})
|
||||
.filter((line): line is string => Boolean(line))
|
||||
if (timestampLines.length > 0) {
|
||||
parts.push('')
|
||||
parts.push('--- Timeline ---')
|
||||
parts.push(...timestampLines)
|
||||
}
|
||||
|
||||
if (updates.length > 0) {
|
||||
parts.push('')
|
||||
parts.push('--- Updates ---')
|
||||
for (const update of updates) {
|
||||
const segments: string[] = []
|
||||
if (update.created_at) segments.push(`[${update.created_at}]`)
|
||||
const author = actorName(update.updater ?? update.author)
|
||||
if (author) segments.push(author)
|
||||
const changes: string[] = []
|
||||
const newStatusName = update.new_incident_status?.name ?? update.new_status?.name
|
||||
if (newStatusName) {
|
||||
changes.push(`status → ${newStatusName}`)
|
||||
}
|
||||
if (update.new_severity?.name) changes.push(`severity → ${update.new_severity.name}`)
|
||||
const message = update.message ? htmlToPlainText(update.message) : ''
|
||||
const tail = [changes.join(', '), message].filter(Boolean).join(' — ')
|
||||
const line = [segments.join(' '), tail].filter(Boolean).join(': ')
|
||||
if (line.trim()) parts.push(line)
|
||||
}
|
||||
}
|
||||
|
||||
return parts.join('\n').trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a lightweight document stub from a list entry. No API calls — content is
|
||||
* deferred to getDocument and only fetched for new or changed incidents.
|
||||
*/
|
||||
function incidentToStub(incident: IncidentioIncident): ExternalDocument {
|
||||
return {
|
||||
externalId: incident.id,
|
||||
title: buildTitle(incident),
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: buildSourceUrl(incident),
|
||||
contentHash: buildContentHash(incident),
|
||||
metadata: buildMetadata(incident),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches all status updates for an incident, following the `after` cursor and capping
|
||||
* the total to keep getDocument bounded for very long-running incidents.
|
||||
*/
|
||||
async function fetchIncidentUpdates(
|
||||
accessToken: string,
|
||||
incidentId: string
|
||||
): Promise<IncidentioUpdate[]> {
|
||||
const updates: IncidentioUpdate[] = []
|
||||
let after: string | undefined
|
||||
|
||||
while (updates.length < MAX_UPDATES_PER_INCIDENT) {
|
||||
const url = new URL(`${INCIDENTIO_API_BASE}/v2/incident_updates`)
|
||||
url.searchParams.set('incident_id', incidentId)
|
||||
url.searchParams.set('page_size', String(PAGE_SIZE))
|
||||
if (after) url.searchParams.set('after', after)
|
||||
|
||||
const response = await fetchWithRetry(url.toString(), {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
logger.warn('Failed to fetch incident updates', { incidentId, status: response.status })
|
||||
break
|
||||
}
|
||||
|
||||
const data = (await response.json()) as IncidentioUpdatesListResponse
|
||||
const page = data.incident_updates ?? []
|
||||
updates.push(...page)
|
||||
|
||||
after = data.pagination_meta?.after?.trim() || undefined
|
||||
if (!after || page.length === 0) break
|
||||
}
|
||||
|
||||
return updates.slice(0, MAX_UPDATES_PER_INCIDENT)
|
||||
}
|
||||
|
||||
export const incidentioConnector: ConnectorConfig = {
|
||||
...incidentioConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>,
|
||||
lastSyncAt?: Date
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const maxIncidents = sourceConfig.maxIncidents ? Number(sourceConfig.maxIncidents) : 0
|
||||
|
||||
const statusCategory =
|
||||
typeof sourceConfig.statusCategory === 'string' ? sourceConfig.statusCategory.trim() : ''
|
||||
const mode = typeof sourceConfig.mode === 'string' ? sourceConfig.mode.trim() : ''
|
||||
|
||||
const url = new URL(`${INCIDENTIO_API_BASE}/v2/incidents`)
|
||||
url.searchParams.set('page_size', String(PAGE_SIZE))
|
||||
url.searchParams.set('sort_by', 'created_at_oldest_first')
|
||||
if (cursor) url.searchParams.set('after', cursor)
|
||||
if (lastSyncAt) url.searchParams.set('updated_at[gte]', lastSyncAt.toISOString())
|
||||
if (statusCategory) url.searchParams.set('status_category[one_of]', statusCategory)
|
||||
if (mode) url.searchParams.set('mode[one_of]', mode)
|
||||
|
||||
logger.info('Listing incident.io incidents', {
|
||||
cursor: cursor ?? 'initial',
|
||||
incremental: Boolean(lastSyncAt),
|
||||
maxIncidents,
|
||||
})
|
||||
|
||||
const response = await fetchWithRetry(url.toString(), {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
logger.error('Failed to list incident.io incidents', {
|
||||
status: response.status,
|
||||
error: errorText.slice(0, 500),
|
||||
})
|
||||
throw new Error(`Failed to list incident.io incidents: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as IncidentioIncidentsListResponse
|
||||
const incidents = (data.incidents ?? []).filter((incident) => Boolean(incident.id))
|
||||
|
||||
const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
|
||||
let documents = incidents.map(incidentToStub)
|
||||
if (maxIncidents > 0) {
|
||||
const remaining = Math.max(0, maxIncidents - prevFetched)
|
||||
if (documents.length > remaining) {
|
||||
documents = documents.slice(0, remaining)
|
||||
}
|
||||
}
|
||||
|
||||
const totalFetched = prevFetched + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
const hitLimit = maxIncidents > 0 && totalFetched >= maxIncidents
|
||||
if (hitLimit && syncContext) syncContext.listingCapped = true
|
||||
|
||||
const after = data.pagination_meta?.after?.trim() || undefined
|
||||
const hasMore = !hitLimit && Boolean(after)
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore ? after : undefined,
|
||||
hasMore,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
_sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
try {
|
||||
if (!externalId) return null
|
||||
|
||||
const url = `${INCIDENTIO_API_BASE}/v2/incidents/${encodeURIComponent(externalId)}`
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404 || response.status === 410) return null
|
||||
throw new Error(`Failed to fetch incident.io incident: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as IncidentioIncidentShowResponse
|
||||
const incident = data.incident
|
||||
if (!incident?.id) return null
|
||||
|
||||
const updates = await fetchIncidentUpdates(accessToken, incident.id)
|
||||
const content = formatIncidentContent(incident, updates)
|
||||
if (!content.trim()) return null
|
||||
|
||||
return {
|
||||
externalId: incident.id,
|
||||
title: buildTitle(incident),
|
||||
content,
|
||||
contentDeferred: false,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: buildSourceUrl(incident),
|
||||
contentHash: buildContentHash(incident),
|
||||
metadata: buildMetadata(incident),
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Failed to get incident.io incident', {
|
||||
externalId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const maxIncidents = sourceConfig.maxIncidents as string | undefined
|
||||
if (maxIncidents && (Number.isNaN(Number(maxIncidents)) || Number(maxIncidents) < 0)) {
|
||||
return { valid: false, error: 'Max incidents must be a non-negative number' }
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetchWithRetry(
|
||||
`${INCIDENTIO_API_BASE}/v2/incidents?page_size=1`,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
return {
|
||||
valid: false,
|
||||
error: `incident.io access failed: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}`,
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.status === 'string' && metadata.status.trim()) {
|
||||
result.status = metadata.status
|
||||
}
|
||||
|
||||
if (typeof metadata.statusCategory === 'string' && metadata.statusCategory.trim()) {
|
||||
result.statusCategory = metadata.statusCategory
|
||||
}
|
||||
|
||||
if (typeof metadata.severity === 'string' && metadata.severity.trim()) {
|
||||
result.severity = metadata.severity
|
||||
}
|
||||
|
||||
if (typeof metadata.incidentType === 'string' && metadata.incidentType.trim()) {
|
||||
result.incidentType = metadata.incidentType
|
||||
}
|
||||
|
||||
if (typeof metadata.mode === 'string' && metadata.mode.trim()) {
|
||||
result.mode = metadata.mode
|
||||
}
|
||||
|
||||
if (typeof metadata.visibility === 'string' && metadata.visibility.trim()) {
|
||||
result.visibility = metadata.visibility
|
||||
}
|
||||
|
||||
const incidentDate = parseTagDate(metadata.incidentDate)
|
||||
if (incidentDate) result.incidentDate = incidentDate
|
||||
|
||||
if (typeof metadata.reportedBy === 'string' && metadata.reportedBy.trim()) {
|
||||
result.reportedBy = metadata.reportedBy
|
||||
}
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { incidentioConnector } from '@/connectors/incidentio/incidentio'
|
||||
@@ -0,0 +1,76 @@
|
||||
import { IncidentioIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const incidentioConnectorMeta: ConnectorMeta = {
|
||||
id: 'incidentio',
|
||||
name: 'incident.io',
|
||||
description: 'Sync incidents and postmortems from incident.io into your knowledge base',
|
||||
version: '1.0.0',
|
||||
icon: IncidentioIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'apiKey',
|
||||
label: 'API Key',
|
||||
placeholder: 'Enter your incident.io API key',
|
||||
},
|
||||
|
||||
supportsIncrementalSync: true,
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'statusCategory',
|
||||
title: 'Status Category',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
options: [
|
||||
{ label: 'All', id: '' },
|
||||
{ label: 'Live (active)', id: 'live' },
|
||||
{ label: 'Paused', id: 'paused' },
|
||||
{ label: 'Closed', id: 'closed' },
|
||||
{ label: 'Triage', id: 'triage' },
|
||||
{ label: 'Learning (post-incident)', id: 'learning' },
|
||||
{ label: 'Declined', id: 'declined' },
|
||||
{ label: 'Merged', id: 'merged' },
|
||||
{ label: 'Canceled', id: 'canceled' },
|
||||
],
|
||||
description:
|
||||
'Only sync incidents in this status category. Leave as All to sync every category.',
|
||||
},
|
||||
{
|
||||
id: 'mode',
|
||||
title: 'Mode',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
options: [
|
||||
{ label: 'All', id: '' },
|
||||
{ label: 'Standard (real incidents)', id: 'standard' },
|
||||
{ label: 'Retrospective', id: 'retrospective' },
|
||||
{ label: 'Test', id: 'test' },
|
||||
{ label: 'Tutorial', id: 'tutorial' },
|
||||
],
|
||||
description:
|
||||
'Only sync incidents of this mode. Use Standard to exclude test/tutorial incidents.',
|
||||
},
|
||||
{
|
||||
id: 'maxIncidents',
|
||||
title: 'Max Incidents',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 200 (default: unlimited)',
|
||||
description: 'Cap the number of incidents synced. Leave empty to sync all incidents.',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'status', displayName: 'Status', fieldType: 'text' },
|
||||
{ id: 'statusCategory', displayName: 'Status Category', fieldType: 'text' },
|
||||
{ id: 'severity', displayName: 'Severity', fieldType: 'text' },
|
||||
{ id: 'incidentType', displayName: 'Incident Type', fieldType: 'text' },
|
||||
{ id: 'mode', displayName: 'Mode', fieldType: 'text' },
|
||||
{ id: 'visibility', displayName: 'Visibility', fieldType: 'text' },
|
||||
{ id: 'incidentDate', displayName: 'Incident Date', fieldType: 'date' },
|
||||
{ id: 'reportedBy', displayName: 'Reported By', fieldType: 'text' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { intercomConnector } from '@/connectors/intercom/intercom'
|
||||
@@ -0,0 +1,491 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { z } from 'zod'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { DEFAULT_MAX_ITEMS, intercomConnectorMeta } from '@/connectors/intercom/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { htmlToPlainText, parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('IntercomConnector')
|
||||
|
||||
const INTERCOM_API_BASE = 'https://api.intercom.io'
|
||||
const ARTICLES_PER_PAGE = 50
|
||||
const CONVERSATIONS_PER_PAGE = 50
|
||||
|
||||
const IntercomAuthorSchema = z
|
||||
.object({
|
||||
type: z.string(),
|
||||
id: z.string(),
|
||||
name: z.string().optional(),
|
||||
})
|
||||
.passthrough()
|
||||
|
||||
const IntercomArticleSchema = z
|
||||
.object({
|
||||
type: z.string().optional(),
|
||||
id: z.string(),
|
||||
title: z.string().optional(),
|
||||
description: z.string().nullable().optional(),
|
||||
body: z.string().nullable().optional(),
|
||||
author_id: z.union([z.number(), z.string()]).optional(),
|
||||
state: z.string(),
|
||||
created_at: z.number(),
|
||||
updated_at: z.number(),
|
||||
url: z.string().optional(),
|
||||
parent_id: z.number().nullable().optional(),
|
||||
parent_type: z.string().nullable().optional(),
|
||||
})
|
||||
.passthrough()
|
||||
|
||||
type IntercomArticle = z.infer<typeof IntercomArticleSchema>
|
||||
|
||||
const IntercomConversationPartSchema = z
|
||||
.object({
|
||||
type: z.string().optional(),
|
||||
id: z.string(),
|
||||
part_type: z.string().optional(),
|
||||
body: z.string().nullable().optional(),
|
||||
created_at: z.number(),
|
||||
author: IntercomAuthorSchema.optional(),
|
||||
})
|
||||
.passthrough()
|
||||
|
||||
const IntercomConversationSchema = z
|
||||
.object({
|
||||
type: z.string().optional(),
|
||||
id: z.string(),
|
||||
created_at: z.number(),
|
||||
updated_at: z.number(),
|
||||
title: z.string().nullable().optional(),
|
||||
state: z.string(),
|
||||
open: z.boolean().optional(),
|
||||
source: z
|
||||
.object({
|
||||
type: z.string(),
|
||||
id: z.string(),
|
||||
subject: z.string().optional(),
|
||||
body: z.string().nullable().optional(),
|
||||
author: IntercomAuthorSchema,
|
||||
delivered_as: z.string().optional(),
|
||||
})
|
||||
.passthrough()
|
||||
.optional(),
|
||||
tags: z
|
||||
.object({
|
||||
type: z.string().optional(),
|
||||
tags: z
|
||||
.array(
|
||||
z
|
||||
.object({
|
||||
id: z.string(),
|
||||
name: z.string(),
|
||||
})
|
||||
.passthrough()
|
||||
)
|
||||
.default([]),
|
||||
})
|
||||
.passthrough()
|
||||
.optional(),
|
||||
conversation_parts: z
|
||||
.object({
|
||||
type: z.string(),
|
||||
conversation_parts: z.array(IntercomConversationPartSchema),
|
||||
total_count: z.number(),
|
||||
})
|
||||
.passthrough()
|
||||
.optional(),
|
||||
})
|
||||
.passthrough()
|
||||
|
||||
type IntercomConversation = z.infer<typeof IntercomConversationSchema>
|
||||
|
||||
/**
|
||||
* Makes a GET request to the Intercom API with Bearer token auth.
|
||||
*/
|
||||
async function intercomApiGet(
|
||||
path: string,
|
||||
accessToken: string,
|
||||
params?: Record<string, string>,
|
||||
retryOptions?: Parameters<typeof fetchWithRetry>[2]
|
||||
): Promise<Record<string, unknown>> {
|
||||
const url = new URL(`${INTERCOM_API_BASE}${path}`)
|
||||
if (params) {
|
||||
for (const [key, value] of Object.entries(params)) {
|
||||
url.searchParams.set(key, value)
|
||||
}
|
||||
}
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
url.toString(),
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
'Intercom-Version': '2.11',
|
||||
},
|
||||
},
|
||||
retryOptions
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorBody = await response.text().catch(() => '')
|
||||
throw new Error(`Intercom API HTTP error ${response.status}: ${errorBody}`)
|
||||
}
|
||||
|
||||
return (await response.json()) as Record<string, unknown>
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches all articles from Intercom, respecting page-based pagination and max items cap.
|
||||
*/
|
||||
async function fetchArticles(
|
||||
accessToken: string,
|
||||
maxItems: number,
|
||||
stateFilter: string
|
||||
): Promise<IntercomArticle[]> {
|
||||
const allArticles: IntercomArticle[] = []
|
||||
let page = 1
|
||||
|
||||
while (allArticles.length < maxItems) {
|
||||
const data = await intercomApiGet('/articles', accessToken, {
|
||||
page: String(page),
|
||||
per_page: String(ARTICLES_PER_PAGE),
|
||||
})
|
||||
|
||||
const articles = z.array(IntercomArticleSchema).parse(data.data ?? [])
|
||||
if (articles.length === 0) break
|
||||
|
||||
for (const article of articles) {
|
||||
if (stateFilter !== 'all' && article.state !== stateFilter) continue
|
||||
allArticles.push(article)
|
||||
if (allArticles.length >= maxItems) break
|
||||
}
|
||||
|
||||
const pages = data.pages as { total_pages?: number } | null
|
||||
if (!pages?.total_pages || page >= pages.total_pages) break
|
||||
page++
|
||||
}
|
||||
|
||||
return allArticles
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches conversations from Intercom using cursor-based pagination.
|
||||
*/
|
||||
async function fetchConversations(
|
||||
accessToken: string,
|
||||
maxItems: number,
|
||||
stateFilter: string
|
||||
): Promise<IntercomConversation[]> {
|
||||
const allConversations: IntercomConversation[] = []
|
||||
let startingAfter: string | undefined
|
||||
|
||||
while (allConversations.length < maxItems) {
|
||||
const params: Record<string, string> = {
|
||||
per_page: String(Math.min(CONVERSATIONS_PER_PAGE, 150)),
|
||||
}
|
||||
if (startingAfter) {
|
||||
params.starting_after = startingAfter
|
||||
}
|
||||
|
||||
const data = await intercomApiGet('/conversations', accessToken, params)
|
||||
const conversations = z.array(IntercomConversationSchema).parse(data.conversations ?? [])
|
||||
if (conversations.length === 0) break
|
||||
|
||||
for (const conversation of conversations) {
|
||||
if (stateFilter !== 'all' && conversation.state !== stateFilter) continue
|
||||
allConversations.push(conversation)
|
||||
if (allConversations.length >= maxItems) break
|
||||
}
|
||||
|
||||
const pages = data.pages as { next?: { starting_after?: string } } | null
|
||||
const nextCursor = pages?.next?.starting_after
|
||||
if (!nextCursor) break
|
||||
startingAfter = nextCursor
|
||||
}
|
||||
|
||||
return allConversations
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the full conversation with conversation_parts included.
|
||||
*/
|
||||
async function fetchConversationDetail(
|
||||
accessToken: string,
|
||||
conversationId: string
|
||||
): Promise<IntercomConversation> {
|
||||
const data = await intercomApiGet(`/conversations/${conversationId}`, accessToken)
|
||||
return IntercomConversationSchema.parse(data)
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a conversation (with parts) into a plain text document.
|
||||
*/
|
||||
function formatConversation(conversation: IntercomConversation): string {
|
||||
const lines: string[] = []
|
||||
|
||||
if (conversation.title) {
|
||||
lines.push(`Subject: ${conversation.title}`)
|
||||
}
|
||||
|
||||
const source = conversation.source
|
||||
const sourceBody = source?.body
|
||||
if (sourceBody) {
|
||||
const authorName = source.author?.name || source.author?.type || 'unknown'
|
||||
const timestamp = new Date(conversation.created_at * 1000).toISOString()
|
||||
lines.push(`[${timestamp}] ${authorName}: ${htmlToPlainText(sourceBody)}`)
|
||||
}
|
||||
|
||||
const parts = conversation.conversation_parts?.conversation_parts || []
|
||||
for (const part of parts) {
|
||||
if (!part.body) continue
|
||||
const authorName = part.author?.name || part.author?.type || 'unknown'
|
||||
const timestamp = new Date(part.created_at * 1000).toISOString()
|
||||
lines.push(`[${timestamp}] ${authorName}: ${htmlToPlainText(part.body)}`)
|
||||
}
|
||||
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts an article to a plain text document.
|
||||
*/
|
||||
function formatArticle(article: IntercomArticle): string {
|
||||
const parts: string[] = []
|
||||
if (article.title) {
|
||||
parts.push(article.title)
|
||||
}
|
||||
if (article.description) {
|
||||
parts.push(article.description)
|
||||
}
|
||||
if (article.body) {
|
||||
parts.push(htmlToPlainText(article.body))
|
||||
}
|
||||
return parts.join('\n\n')
|
||||
}
|
||||
|
||||
export const intercomConnector: ConnectorConfig = {
|
||||
...intercomConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
_cursor?: string,
|
||||
_syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const contentType = (sourceConfig.contentType as string) || 'articles'
|
||||
const articleState = (sourceConfig.articleState as string) || 'published'
|
||||
const conversationState = (sourceConfig.conversationState as string) || 'all'
|
||||
const maxItems = sourceConfig.maxItems ? Number(sourceConfig.maxItems) : DEFAULT_MAX_ITEMS
|
||||
|
||||
const documents: ExternalDocument[] = []
|
||||
|
||||
if (contentType === 'articles' || contentType === 'both') {
|
||||
logger.info('Fetching Intercom articles', { articleState, maxItems })
|
||||
const articles = await fetchArticles(accessToken, maxItems, articleState)
|
||||
|
||||
for (const article of articles) {
|
||||
const content = formatArticle(article)
|
||||
if (!content.trim()) continue
|
||||
|
||||
const updatedAt = new Date(article.updated_at * 1000).toISOString()
|
||||
|
||||
documents.push({
|
||||
externalId: `article-${article.id}`,
|
||||
title: article.title || `Article ${article.id}`,
|
||||
content,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: `https://app.intercom.com/a/apps/_/articles/articles/${article.id}/show`,
|
||||
contentHash: `intercom:article-${article.id}:${article.updated_at}`,
|
||||
metadata: {
|
||||
type: 'article',
|
||||
state: article.state,
|
||||
authorId: String(article.author_id),
|
||||
updatedAt,
|
||||
createdAt: new Date(article.created_at * 1000).toISOString(),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
logger.info('Fetched Intercom articles', { count: articles.length })
|
||||
}
|
||||
|
||||
if (contentType === 'conversations' || contentType === 'both') {
|
||||
logger.info('Fetching Intercom conversations', { conversationState, maxItems })
|
||||
const conversations = await fetchConversations(accessToken, maxItems, conversationState)
|
||||
|
||||
for (const conversation of conversations) {
|
||||
const updatedAt = new Date(conversation.updated_at * 1000).toISOString()
|
||||
const tags = conversation.tags?.tags?.map((t) => t.name) || []
|
||||
|
||||
documents.push({
|
||||
externalId: `conversation-${conversation.id}`,
|
||||
title: conversation.title || `Conversation #${conversation.id}`,
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: `https://app.intercom.com/a/apps/_/inbox/inbox/all/conversations/${conversation.id}`,
|
||||
contentHash: `intercom:conversation-${conversation.id}:${conversation.updated_at}`,
|
||||
metadata: {
|
||||
type: 'conversation',
|
||||
state: conversation.state,
|
||||
tags: tags.join(', '),
|
||||
updatedAt,
|
||||
createdAt: new Date(conversation.created_at * 1000).toISOString(),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
logger.info('Fetched Intercom conversations', { count: conversations.length })
|
||||
}
|
||||
|
||||
return { documents, hasMore: false }
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
try {
|
||||
if (externalId.startsWith('article-')) {
|
||||
const articleId = externalId.replace('article-', '')
|
||||
const data = await intercomApiGet(`/articles/${articleId}`, accessToken)
|
||||
const article = IntercomArticleSchema.parse(data)
|
||||
|
||||
const content = formatArticle(article)
|
||||
if (!content.trim()) return null
|
||||
|
||||
const updatedAt = new Date(article.updated_at * 1000).toISOString()
|
||||
|
||||
return {
|
||||
externalId,
|
||||
title: article.title || `Article ${article.id}`,
|
||||
content,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: `https://app.intercom.com/a/apps/_/articles/articles/${article.id}/show`,
|
||||
contentHash: `intercom:article-${article.id}:${article.updated_at}`,
|
||||
metadata: {
|
||||
type: 'article',
|
||||
state: article.state,
|
||||
authorId: String(article.author_id),
|
||||
updatedAt,
|
||||
createdAt: new Date(article.created_at * 1000).toISOString(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
if (externalId.startsWith('conversation-')) {
|
||||
const conversationId = externalId.replace('conversation-', '')
|
||||
const detail = await fetchConversationDetail(accessToken, conversationId)
|
||||
|
||||
const content = formatConversation(detail)
|
||||
if (!content.trim()) return null
|
||||
|
||||
const updatedAt = new Date(detail.updated_at * 1000).toISOString()
|
||||
const tags = detail.tags?.tags?.map((t) => t.name) || []
|
||||
|
||||
return {
|
||||
externalId,
|
||||
title: detail.title || `Conversation #${detail.id}`,
|
||||
content,
|
||||
contentDeferred: false,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: `https://app.intercom.com/a/apps/_/inbox/inbox/all/conversations/${detail.id}`,
|
||||
contentHash: `intercom:conversation-${detail.id}:${detail.updated_at}`,
|
||||
metadata: {
|
||||
type: 'conversation',
|
||||
state: detail.state,
|
||||
tags: tags.join(', '),
|
||||
updatedAt,
|
||||
createdAt: new Date(detail.created_at * 1000).toISOString(),
|
||||
messageCount: (detail.conversation_parts?.total_count ?? 0) + 1,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
logger.warn('Unknown external ID format', { externalId })
|
||||
return null
|
||||
} catch (error) {
|
||||
logger.warn('Failed to get Intercom document', {
|
||||
externalId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const contentType = sourceConfig.contentType as string | undefined
|
||||
const maxItems = sourceConfig.maxItems as string | undefined
|
||||
|
||||
if (!contentType) {
|
||||
return { valid: false, error: 'Content type is required' }
|
||||
}
|
||||
|
||||
if (maxItems && (Number.isNaN(Number(maxItems)) || Number(maxItems) <= 0)) {
|
||||
return { valid: false, error: 'Max items must be a positive number' }
|
||||
}
|
||||
|
||||
try {
|
||||
// Verify API access by fetching the first page of articles or conversations
|
||||
if (contentType === 'articles' || contentType === 'both') {
|
||||
await intercomApiGet(
|
||||
'/articles',
|
||||
accessToken,
|
||||
{ page: '1', per_page: '1' },
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
}
|
||||
|
||||
if (contentType === 'conversations' || contentType === 'both') {
|
||||
await intercomApiGet(
|
||||
'/conversations',
|
||||
accessToken,
|
||||
{ per_page: '1' },
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.type === 'string') {
|
||||
result.type = metadata.type
|
||||
}
|
||||
|
||||
if (typeof metadata.state === 'string') {
|
||||
result.state = metadata.state
|
||||
}
|
||||
|
||||
if (typeof metadata.tags === 'string' && metadata.tags) {
|
||||
result.tags = metadata.tags
|
||||
}
|
||||
|
||||
if (typeof metadata.authorId === 'string') {
|
||||
result.authorId = metadata.authorId
|
||||
}
|
||||
|
||||
if (typeof metadata.messageCount === 'number') {
|
||||
result.messageCount = metadata.messageCount
|
||||
}
|
||||
|
||||
const updatedAt = parseTagDate(metadata.updatedAt)
|
||||
if (updatedAt) {
|
||||
result.updatedAt = updatedAt
|
||||
}
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { IntercomIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const DEFAULT_MAX_ITEMS = 500
|
||||
|
||||
export const intercomConnectorMeta: ConnectorMeta = {
|
||||
id: 'intercom',
|
||||
name: 'Intercom',
|
||||
description: 'Sync Help Center articles and conversations from Intercom',
|
||||
version: '1.0.0',
|
||||
icon: IntercomIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'apiKey',
|
||||
label: 'Access Token',
|
||||
placeholder: 'Enter your Intercom access token',
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'contentType',
|
||||
title: 'Content Type',
|
||||
type: 'dropdown',
|
||||
required: true,
|
||||
description: 'Choose what to sync from Intercom',
|
||||
options: [
|
||||
{ label: 'Articles Only', id: 'articles' },
|
||||
{ label: 'Conversations Only', id: 'conversations' },
|
||||
{ label: 'Articles & Conversations', id: 'both' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'articleState',
|
||||
title: 'Article State',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
description: 'Filter articles by state (default: published)',
|
||||
options: [
|
||||
{ label: 'Published', id: 'published' },
|
||||
{ label: 'Draft', id: 'draft' },
|
||||
{ label: 'All', id: 'all' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'conversationState',
|
||||
title: 'Conversation State',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
description: 'Filter conversations by state (default: all)',
|
||||
options: [
|
||||
{ label: 'Open', id: 'open' },
|
||||
{ label: 'Closed', id: 'closed' },
|
||||
{ label: 'All', id: 'all' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'maxItems',
|
||||
title: 'Max Items',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: `e.g. 200 (default: ${DEFAULT_MAX_ITEMS})`,
|
||||
description: 'Maximum number of articles or conversations to sync',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'type', displayName: 'Content Type', fieldType: 'text' },
|
||||
{ id: 'state', displayName: 'State', fieldType: 'text' },
|
||||
{ id: 'tags', displayName: 'Tags', fieldType: 'text' },
|
||||
{ id: 'authorId', displayName: 'Author ID', fieldType: 'text' },
|
||||
{ id: 'messageCount', displayName: 'Message Count', fieldType: 'number' },
|
||||
{ id: 'updatedAt', displayName: 'Last Updated', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { jiraConnector } from '@/connectors/jira/jira'
|
||||
@@ -0,0 +1,357 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { jiraConnectorMeta } from '@/connectors/jira/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils'
|
||||
import { extractAdfText, getJiraCloudId } from '@/tools/jira/utils'
|
||||
|
||||
const logger = createLogger('JiraConnector')
|
||||
|
||||
const PAGE_SIZE = 50
|
||||
|
||||
/**
|
||||
* Builds a JQL clause restricting issues to the given project keys.
|
||||
* Single key uses `project = "X"`; multiple keys use `project in ("X","Y")`.
|
||||
* Each key is escaped for inclusion in a JQL double-quoted string.
|
||||
*/
|
||||
function buildProjectClause(projectKeys: string[]): string {
|
||||
const escapeKey = (key: string) => key.replace(/\\/g, '\\\\').replace(/"/g, '\\"')
|
||||
if (projectKeys.length === 1) {
|
||||
return `project = "${escapeKey(projectKeys[0])}"`
|
||||
}
|
||||
const list = projectKeys.map((k) => `"${escapeKey(k)}"`).join(',')
|
||||
return `project in (${list})`
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a plain-text representation of a Jira issue for knowledge base indexing.
|
||||
*/
|
||||
function buildIssueContent(fields: Record<string, unknown>): string {
|
||||
const parts: string[] = []
|
||||
|
||||
const summary = fields.summary as string | undefined
|
||||
if (summary) parts.push(summary)
|
||||
|
||||
const description = extractAdfText(fields.description)
|
||||
if (description) parts.push(description)
|
||||
|
||||
const comments = fields.comment as { comments?: Array<{ body?: unknown }> } | undefined
|
||||
if (comments?.comments) {
|
||||
for (const comment of comments.comments) {
|
||||
const text = extractAdfText(comment.body)
|
||||
if (text) parts.push(text)
|
||||
}
|
||||
}
|
||||
|
||||
return parts.join('\n\n').trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts common metadata fields from a Jira issue into an ExternalDocument
|
||||
* stub with deferred content. The contentHash is metadata-based so it is
|
||||
* identical whether produced during listing or full fetch.
|
||||
*/
|
||||
function issueToStub(issue: Record<string, unknown>, domain: string): ExternalDocument {
|
||||
const fields = (issue.fields || {}) as Record<string, unknown>
|
||||
const key = issue.key as string
|
||||
const issueType = fields.issuetype as Record<string, unknown> | undefined
|
||||
const status = fields.status as Record<string, unknown> | undefined
|
||||
const priority = fields.priority as Record<string, unknown> | undefined
|
||||
const assignee = fields.assignee as Record<string, unknown> | undefined
|
||||
const reporter = fields.reporter as Record<string, unknown> | undefined
|
||||
const project = fields.project as Record<string, unknown> | undefined
|
||||
const labels = Array.isArray(fields.labels) ? (fields.labels as string[]) : []
|
||||
const updated = (fields.updated as string) ?? ''
|
||||
|
||||
return {
|
||||
externalId: String(issue.id),
|
||||
title: `${key}: ${(fields.summary as string) || 'Untitled'}`,
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: `https://${domain}/browse/${key}`,
|
||||
contentHash: `jira:${issue.id}:${updated}`,
|
||||
metadata: {
|
||||
key,
|
||||
issueType: issueType?.name,
|
||||
status: status?.name,
|
||||
priority: priority?.name,
|
||||
assignee: assignee?.displayName,
|
||||
reporter: reporter?.displayName,
|
||||
project: project?.key,
|
||||
labels,
|
||||
created: fields.created,
|
||||
updated: fields.updated,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a fully-fetched Jira issue (with description and comments) into an
|
||||
* ExternalDocument with resolved content.
|
||||
*/
|
||||
function issueToFullDocument(issue: Record<string, unknown>, domain: string): ExternalDocument {
|
||||
const stub = issueToStub(issue, domain)
|
||||
const fields = (issue.fields || {}) as Record<string, unknown>
|
||||
const content = buildIssueContent(fields)
|
||||
|
||||
return {
|
||||
...stub,
|
||||
content,
|
||||
contentDeferred: false,
|
||||
}
|
||||
}
|
||||
|
||||
export const jiraConnector: ConnectorConfig = {
|
||||
...jiraConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const domain = sourceConfig.domain as string
|
||||
const projectKeys = parseMultiValue(sourceConfig.projectKey)
|
||||
const jqlFilter = (sourceConfig.jql as string) || ''
|
||||
const maxIssues = sourceConfig.maxIssues ? Number(sourceConfig.maxIssues) : 0
|
||||
|
||||
if (projectKeys.length === 0) {
|
||||
throw new Error('At least one project key is required')
|
||||
}
|
||||
|
||||
let cloudId = syncContext?.cloudId as string | undefined
|
||||
if (!cloudId) {
|
||||
cloudId = await getJiraCloudId(domain, accessToken)
|
||||
if (syncContext) syncContext.cloudId = cloudId
|
||||
}
|
||||
|
||||
const projectClause = buildProjectClause(projectKeys)
|
||||
let jql = `${projectClause} ORDER BY updated DESC`
|
||||
if (jqlFilter.trim()) {
|
||||
jql = `${projectClause} AND (${jqlFilter.trim()}) ORDER BY updated DESC`
|
||||
}
|
||||
|
||||
/**
|
||||
* Collected-count is encoded in the cursor as `${pageToken}|${count}` so
|
||||
* the maxIssues cap works correctly even when the caller doesn't pass
|
||||
* syncContext. Falls back to syncContext.collectedCount for backwards
|
||||
* compatibility with cursors emitted before this format existed.
|
||||
*/
|
||||
let pageToken: string | undefined
|
||||
let collectedSoFar = (syncContext?.collectedCount as number | undefined) ?? 0
|
||||
if (cursor) {
|
||||
const sep = cursor.lastIndexOf('|')
|
||||
if (sep > 0) {
|
||||
pageToken = cursor.slice(0, sep)
|
||||
const parsed = Number(cursor.slice(sep + 1))
|
||||
if (Number.isFinite(parsed) && parsed >= 0) collectedSoFar = parsed
|
||||
} else {
|
||||
pageToken = cursor
|
||||
}
|
||||
}
|
||||
|
||||
const remaining = maxIssues > 0 ? Math.max(0, maxIssues - collectedSoFar) : PAGE_SIZE
|
||||
if (maxIssues > 0 && remaining === 0) {
|
||||
return { documents: [], hasMore: false }
|
||||
}
|
||||
|
||||
const params = new URLSearchParams()
|
||||
params.append('jql', jql)
|
||||
params.append('maxResults', String(Math.min(PAGE_SIZE, remaining)))
|
||||
params.append(
|
||||
'fields',
|
||||
'summary,issuetype,status,priority,assignee,reporter,project,labels,created,updated'
|
||||
)
|
||||
if (pageToken) params.append('nextPageToken', pageToken)
|
||||
|
||||
const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/search/jql?${params.toString()}`
|
||||
|
||||
logger.info(`Listing Jira issues for ${projectKeys.length} project(s)`, {
|
||||
projectKeys,
|
||||
hasCursor: Boolean(cursor),
|
||||
})
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error('Failed to search Jira issues', {
|
||||
status: response.status,
|
||||
error: errorText,
|
||||
})
|
||||
throw new Error(`Failed to search Jira issues: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
let issues = (data.issues || []) as Record<string, unknown>[]
|
||||
/**
|
||||
* `/rest/api/3/search/jql` signals end-of-results purely by the absence
|
||||
* of `nextPageToken`. `data.isLast` is unreliable on this endpoint and
|
||||
* has been observed returning `true` alongside a valid token
|
||||
* (JRACLOUD-95477), so we ignore it.
|
||||
*/
|
||||
const nextPageToken = data.nextPageToken as string | undefined
|
||||
const isLast = !nextPageToken
|
||||
|
||||
if (maxIssues > 0 && issues.length > remaining) {
|
||||
issues = issues.slice(0, remaining)
|
||||
}
|
||||
|
||||
const documents: ExternalDocument[] = issues.map((issue) => issueToStub(issue, domain))
|
||||
|
||||
const newCollected = collectedSoFar + issues.length
|
||||
if (syncContext) syncContext.collectedCount = newCollected
|
||||
|
||||
const reachedCap = maxIssues > 0 && newCollected >= maxIssues
|
||||
const hasMore = !isLast && !reachedCap
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore && nextPageToken ? `${nextPageToken}|${newCollected}` : undefined,
|
||||
hasMore,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
externalId: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocument | null> => {
|
||||
const domain = sourceConfig.domain as string
|
||||
let cloudId = syncContext?.cloudId as string | undefined
|
||||
if (!cloudId) {
|
||||
cloudId = await getJiraCloudId(domain, accessToken)
|
||||
if (syncContext) syncContext.cloudId = cloudId
|
||||
}
|
||||
|
||||
const params = new URLSearchParams()
|
||||
params.append(
|
||||
'fields',
|
||||
'summary,description,comment,issuetype,status,priority,assignee,reporter,project,labels,created,updated'
|
||||
)
|
||||
|
||||
const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${externalId}?${params.toString()}`
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) return null
|
||||
throw new Error(`Failed to get Jira issue: ${response.status}`)
|
||||
}
|
||||
|
||||
const issue = await response.json()
|
||||
return issueToFullDocument(issue, domain)
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const domain = sourceConfig.domain as string
|
||||
const projectKeys = parseMultiValue(sourceConfig.projectKey)
|
||||
|
||||
if (!domain || projectKeys.length === 0) {
|
||||
return { valid: false, error: 'Domain and at least one project key are required' }
|
||||
}
|
||||
|
||||
const maxIssues = sourceConfig.maxIssues as string | undefined
|
||||
if (maxIssues && (Number.isNaN(Number(maxIssues)) || Number(maxIssues) <= 0)) {
|
||||
return { valid: false, error: 'Max issues must be a positive number' }
|
||||
}
|
||||
|
||||
const jqlFilter = (sourceConfig.jql as string | undefined)?.trim() || ''
|
||||
|
||||
try {
|
||||
const cloudId = await getJiraCloudId(domain, accessToken, VALIDATE_RETRY_OPTIONS)
|
||||
|
||||
const projectClause = buildProjectClause(projectKeys)
|
||||
const params = new URLSearchParams()
|
||||
params.append('jql', projectClause)
|
||||
params.append('maxResults', '1')
|
||||
|
||||
const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/search/jql?${params.toString()}`
|
||||
const response = await fetchWithRetry(
|
||||
url,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
if (response.status === 400) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `One or more projects not found (${projectKeys.join(', ')}) or JQL is invalid`,
|
||||
}
|
||||
}
|
||||
return { valid: false, error: `Failed to validate: ${response.status} - ${errorText}` }
|
||||
}
|
||||
|
||||
if (jqlFilter) {
|
||||
const filterParams = new URLSearchParams()
|
||||
filterParams.append('jql', `${projectClause} AND (${jqlFilter})`)
|
||||
filterParams.append('maxResults', '1')
|
||||
|
||||
const filterUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/search/jql?${filterParams.toString()}`
|
||||
const filterResponse = await fetchWithRetry(
|
||||
filterUrl,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!filterResponse.ok) {
|
||||
return { valid: false, error: 'Invalid JQL filter. Check syntax and field names.' }
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
return { valid: false, error: toError(error).message || 'Failed to validate configuration' }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.issueType === 'string') result.issueType = metadata.issueType
|
||||
if (typeof metadata.status === 'string') result.status = metadata.status
|
||||
if (typeof metadata.priority === 'string') result.priority = metadata.priority
|
||||
|
||||
const labels = joinTagArray(metadata.labels)
|
||||
if (labels) result.labels = labels
|
||||
|
||||
if (typeof metadata.assignee === 'string') result.assignee = metadata.assignee
|
||||
|
||||
const updated = parseTagDate(metadata.updated)
|
||||
if (updated) result.updated = updated
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import { JiraIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const jiraConnectorMeta: ConnectorMeta = {
|
||||
id: 'jira',
|
||||
name: 'Jira',
|
||||
description: 'Sync issues from a Jira project',
|
||||
version: '1.0.0',
|
||||
icon: JiraIcon,
|
||||
|
||||
auth: { mode: 'oauth', provider: 'jira', requiredScopes: ['read:jira-work', 'offline_access'] },
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'domain',
|
||||
title: 'Jira Domain',
|
||||
type: 'short-input',
|
||||
placeholder: 'yoursite.atlassian.net',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'projectSelector',
|
||||
title: 'Projects',
|
||||
type: 'selector',
|
||||
selectorKey: 'jira.projects',
|
||||
canonicalParamId: 'projectKey',
|
||||
mode: 'basic',
|
||||
multi: true,
|
||||
dependsOn: ['domain'],
|
||||
placeholder: 'Select one or more projects',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'projectKey',
|
||||
title: 'Project Keys',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'projectKey',
|
||||
mode: 'advanced',
|
||||
multi: true,
|
||||
placeholder: 'e.g. ENG, PROJ (comma-separated for multiple)',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'jql',
|
||||
title: 'JQL Filter',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. status = "Done" AND type = Bug',
|
||||
},
|
||||
{
|
||||
id: 'maxIssues',
|
||||
title: 'Max Issues',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 500 (default: unlimited)',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'issueType', displayName: 'Issue Type', fieldType: 'text' },
|
||||
{ id: 'status', displayName: 'Status', fieldType: 'text' },
|
||||
{ id: 'priority', displayName: 'Priority', fieldType: 'text' },
|
||||
{ id: 'labels', displayName: 'Labels', fieldType: 'text' },
|
||||
{ id: 'assignee', displayName: 'Assignee', fieldType: 'text' },
|
||||
{ id: 'updated', displayName: 'Last Updated', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { jsmConnector } from '@/connectors/jsm/jsm'
|
||||
@@ -0,0 +1,556 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { jsmConnectorMeta } from '@/connectors/jsm/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { parseTagDate } from '@/connectors/utils'
|
||||
import { extractAdfText, getJiraCloudId } from '@/tools/jira/utils'
|
||||
import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils'
|
||||
|
||||
const logger = createLogger('JsmConnector')
|
||||
|
||||
const PAGE_SIZE = 50
|
||||
|
||||
/**
|
||||
* Allowed `requestStatus` filter values for `GET /rest/servicedeskapi/request`.
|
||||
* When omitted, the JSM API defaults to `ALL_REQUESTS`.
|
||||
*/
|
||||
const VALID_REQUEST_STATUS = ['OPEN_REQUESTS', 'CLOSED_REQUESTS', 'ALL_REQUESTS'] as const
|
||||
type JsmRequestStatus = (typeof VALID_REQUEST_STATUS)[number]
|
||||
|
||||
/**
|
||||
* Allowed `requestOwnership` filter values for `GET /rest/servicedeskapi/request`.
|
||||
*
|
||||
* This param scopes results to the OAuth user's relationship to each request. When
|
||||
* omitted, the JSM API defaults to `OWNED_REQUESTS` — i.e. only requests the
|
||||
* authenticated user reported. For a knowledge-base sync the user almost always
|
||||
* wants every request in the service desk, so the connector defaults this to
|
||||
* `ALL_REQUESTS` (which the JSM API treats as "owned + participated") rather than
|
||||
* relying on the API's narrower default.
|
||||
*/
|
||||
const VALID_REQUEST_OWNERSHIP = ['OWNED_REQUESTS', 'PARTICIPATED_REQUESTS', 'ALL_REQUESTS'] as const
|
||||
type JsmRequestOwnership = (typeof VALID_REQUEST_OWNERSHIP)[number]
|
||||
|
||||
/**
|
||||
* Which comments to include in synced documents.
|
||||
*/
|
||||
const VALID_COMMENT_SCOPE = ['none', 'public', 'all'] as const
|
||||
type JsmCommentScope = (typeof VALID_COMMENT_SCOPE)[number]
|
||||
|
||||
/**
|
||||
* A JSM date object as returned by the Service Desk REST API. The same shape is
|
||||
* used for `createdDate`, `currentStatus.statusDate`, and comment `created`.
|
||||
*/
|
||||
interface JsmDate {
|
||||
iso8601?: string
|
||||
friendly?: string
|
||||
epochMillis?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Subset of a JSM customer request returned by `GET /request` and
|
||||
* `GET /request/{issueIdOrKey}`. Only the fields the connector reads are typed.
|
||||
*/
|
||||
interface JsmRequest {
|
||||
issueId?: string
|
||||
issueKey?: string
|
||||
requestTypeId?: string
|
||||
serviceDeskId?: string
|
||||
createdDate?: JsmDate
|
||||
currentStatus?: {
|
||||
status?: string
|
||||
statusCategory?: string
|
||||
statusDate?: JsmDate
|
||||
}
|
||||
reporter?: {
|
||||
displayName?: string
|
||||
emailAddress?: string
|
||||
}
|
||||
requestFieldValues?: Array<{
|
||||
fieldId?: string
|
||||
label?: string
|
||||
value?: unknown
|
||||
renderedValue?: unknown
|
||||
}>
|
||||
_links?: {
|
||||
web?: string
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A single comment on a JSM request. The JSM API returns the comment `body` as a
|
||||
* plain string containing Jira wiki markup (not an ADF document), so no rich-text
|
||||
* extraction is required.
|
||||
*/
|
||||
interface JsmComment {
|
||||
id?: string
|
||||
body?: string
|
||||
public?: boolean
|
||||
author?: {
|
||||
displayName?: string
|
||||
}
|
||||
created?: JsmDate
|
||||
}
|
||||
|
||||
/**
|
||||
* Paginated envelope shared by every JSM Service Desk list endpoint.
|
||||
*/
|
||||
interface JsmPage<T> {
|
||||
values?: T[]
|
||||
size?: number
|
||||
isLastPage?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the resolved sync options off the raw `sourceConfig`, normalizing
|
||||
* enum-like fields to their valid set and clamping the numeric cap. Centralized
|
||||
* so `listDocuments`, `getDocument`, and `validateConfig` agree on defaults.
|
||||
*/
|
||||
function resolveOptions(sourceConfig: Record<string, unknown>): {
|
||||
requestStatus: JsmRequestStatus
|
||||
requestOwnership: JsmRequestOwnership
|
||||
requestTypeId: string
|
||||
searchTerm: string
|
||||
commentScope: JsmCommentScope
|
||||
maxRequests: number
|
||||
} {
|
||||
const requestStatus = VALID_REQUEST_STATUS.includes(
|
||||
sourceConfig.requestStatus as JsmRequestStatus
|
||||
)
|
||||
? (sourceConfig.requestStatus as JsmRequestStatus)
|
||||
: 'ALL_REQUESTS'
|
||||
|
||||
const requestOwnership = VALID_REQUEST_OWNERSHIP.includes(
|
||||
sourceConfig.requestOwnership as JsmRequestOwnership
|
||||
)
|
||||
? (sourceConfig.requestOwnership as JsmRequestOwnership)
|
||||
: 'ALL_REQUESTS'
|
||||
|
||||
const commentScope = VALID_COMMENT_SCOPE.includes(sourceConfig.comments as JsmCommentScope)
|
||||
? (sourceConfig.comments as JsmCommentScope)
|
||||
: 'public'
|
||||
|
||||
const requestTypeId =
|
||||
typeof sourceConfig.requestTypeId === 'string' ? sourceConfig.requestTypeId.trim() : ''
|
||||
const searchTerm =
|
||||
typeof sourceConfig.searchTerm === 'string' ? sourceConfig.searchTerm.trim() : ''
|
||||
|
||||
const parsedMax = sourceConfig.maxRequests ? Number(sourceConfig.maxRequests) : 0
|
||||
const maxRequests = Number.isFinite(parsedMax) && parsedMax > 0 ? Math.floor(parsedMax) : 0
|
||||
|
||||
return { requestStatus, requestOwnership, requestTypeId, searchTerm, commentScope, maxRequests }
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts a plain-text value for a given request field id (e.g. `summary`,
|
||||
* `description`) from a request's `requestFieldValues`. The JSM API returns
|
||||
* `value` either as a plain string (wiki markup) or, for some rich-text fields,
|
||||
* as an ADF document — both are handled.
|
||||
*/
|
||||
function getFieldText(request: JsmRequest, fieldId: string): string {
|
||||
const field = request.requestFieldValues?.find((f) => f.fieldId === fieldId)
|
||||
if (!field) return ''
|
||||
const { value } = field
|
||||
if (typeof value === 'string') return value
|
||||
if (value && typeof value === 'object') {
|
||||
const adf = extractAdfText(value)
|
||||
if (adf) return adf
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the best available "change indicator" timestamp for a request.
|
||||
*
|
||||
* The JSM list endpoint does NOT return an updated/last-modified field — only
|
||||
* `createdDate` and `currentStatus.statusDate` are present. We use
|
||||
* `statusDate` (the time the request last changed status) when available, and
|
||||
* fall back to `createdDate`. This is the change signal encoded into the
|
||||
* contentHash. Note: edits that do not change status (e.g. a new comment) are
|
||||
* not reflected here, so such changes may not trigger a re-sync.
|
||||
*/
|
||||
function getChangeIndicator(request: JsmRequest): string {
|
||||
const statusDate = request.currentStatus?.statusDate
|
||||
if (statusDate?.epochMillis != null) return String(statusDate.epochMillis)
|
||||
if (statusDate?.iso8601) return statusDate.iso8601
|
||||
const created = request.createdDate
|
||||
if (created?.epochMillis != null) return String(created.epochMillis)
|
||||
if (created?.iso8601) return created.iso8601
|
||||
return ''
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a stub ExternalDocument from a request returned by the list endpoint.
|
||||
* Content is deferred — description and comments require a per-request API call
|
||||
* fetched lazily in `getDocument`. The contentHash is metadata-only so it is
|
||||
* identical whether produced here or in `getDocument`.
|
||||
*/
|
||||
function requestToStub(request: JsmRequest, domain: string): ExternalDocument {
|
||||
const issueId = String(request.issueId ?? '')
|
||||
const issueKey = request.issueKey ?? issueId
|
||||
const summary = getFieldText(request, 'summary') || 'Untitled'
|
||||
const status = request.currentStatus?.status
|
||||
|
||||
const bareDomain = domain
|
||||
.trim()
|
||||
.replace(/^https?:\/\//i, '')
|
||||
.replace(/\/+$/, '')
|
||||
|
||||
return {
|
||||
externalId: issueId,
|
||||
title: `${issueKey}: ${summary}`,
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: request._links?.web || `https://${bareDomain}/browse/${issueKey}`,
|
||||
contentHash: `jsm:${issueId}:${getChangeIndicator(request)}`,
|
||||
metadata: {
|
||||
issueKey,
|
||||
requestTypeId: request.requestTypeId,
|
||||
serviceDeskId: request.serviceDeskId,
|
||||
status,
|
||||
reporter: request.reporter?.displayName,
|
||||
created: request.createdDate?.iso8601,
|
||||
/**
|
||||
* The list endpoint has no true "last updated" field; `statusDate` is the
|
||||
* closest available signal (time of last status change). Mapped to the
|
||||
* `updated` tag and documented as such.
|
||||
*/
|
||||
statusDate: request.currentStatus?.statusDate?.iso8601,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders a readable plain-text document from a fully-fetched request and its
|
||||
* comments. Includes summary, description, reporter, status, and comment thread.
|
||||
*/
|
||||
function buildContent(request: JsmRequest, comments: JsmComment[]): string {
|
||||
const parts: string[] = []
|
||||
|
||||
const summary = getFieldText(request, 'summary')
|
||||
if (summary) parts.push(summary)
|
||||
|
||||
const description = getFieldText(request, 'description')
|
||||
if (description) parts.push(description)
|
||||
|
||||
const status = request.currentStatus?.status
|
||||
if (status) parts.push(`Status: ${status}`)
|
||||
|
||||
const reporter = request.reporter?.displayName
|
||||
if (reporter) parts.push(`Reporter: ${reporter}`)
|
||||
|
||||
if (comments.length > 0) {
|
||||
parts.push('Comments:')
|
||||
for (const comment of comments) {
|
||||
const body = (comment.body ?? '').trim()
|
||||
if (!body) continue
|
||||
const author = comment.author?.displayName
|
||||
parts.push(author ? `${author}: ${body}` : body)
|
||||
}
|
||||
}
|
||||
|
||||
return parts.join('\n\n').trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves and caches the Jira cloud ID for a domain across a sync run.
|
||||
*/
|
||||
async function resolveCloudId(
|
||||
domain: string,
|
||||
accessToken: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<string> {
|
||||
const cached = syncContext?.cloudId as string | undefined
|
||||
if (cached) return cached
|
||||
const cloudId = await getJiraCloudId(domain, accessToken)
|
||||
if (syncContext) syncContext.cloudId = cloudId
|
||||
return cloudId
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches comments for a request, following offset pagination until the API
|
||||
* signals `isLastPage`. When `publicOnly` is true the `public=true` filter is
|
||||
* applied so internal/agent-only comments are excluded.
|
||||
*/
|
||||
async function fetchComments(
|
||||
baseUrl: string,
|
||||
accessToken: string,
|
||||
issueIdOrKey: string,
|
||||
publicOnly: boolean
|
||||
): Promise<JsmComment[]> {
|
||||
const comments: JsmComment[] = []
|
||||
let start = 0
|
||||
|
||||
while (true) {
|
||||
const params = new URLSearchParams({
|
||||
start: String(start),
|
||||
limit: String(PAGE_SIZE),
|
||||
})
|
||||
/**
|
||||
* The JSM comment endpoint exposes `public` and `internal` as independent
|
||||
* inclusion filters that both default to `true`. Requesting public-only
|
||||
* therefore requires explicitly disabling `internal` — passing `public=true`
|
||||
* alone would still return agent-only/internal comments.
|
||||
*/
|
||||
if (publicOnly) {
|
||||
params.append('public', 'true')
|
||||
params.append('internal', 'false')
|
||||
}
|
||||
const url = `${baseUrl}/request/${encodeURIComponent(issueIdOrKey)}/comment?${params.toString()}`
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: getJsmHeaders(accessToken),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
logger.warn('Failed to fetch JSM comments', {
|
||||
issueIdOrKey,
|
||||
status: response.status,
|
||||
})
|
||||
break
|
||||
}
|
||||
|
||||
const data = (await response.json()) as JsmPage<JsmComment>
|
||||
const values = data.values ?? []
|
||||
comments.push(...values)
|
||||
|
||||
if (data.isLastPage || values.length === 0) break
|
||||
start += values.length
|
||||
}
|
||||
|
||||
return comments
|
||||
}
|
||||
|
||||
export const jsmConnector: ConnectorConfig = {
|
||||
...jsmConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const domain = sourceConfig.domain as string
|
||||
const serviceDeskId = sourceConfig.serviceDeskId as string
|
||||
|
||||
if (!domain || !serviceDeskId) {
|
||||
throw new Error('Domain and service desk ID are required')
|
||||
}
|
||||
|
||||
const { requestStatus, requestOwnership, requestTypeId, searchTerm, maxRequests } =
|
||||
resolveOptions(sourceConfig)
|
||||
|
||||
const cloudId = await resolveCloudId(domain, accessToken, syncContext)
|
||||
const baseUrl = getJsmApiBaseUrl(cloudId)
|
||||
|
||||
/**
|
||||
* `start|collected` is encoded in the cursor so the maxRequests cap holds
|
||||
* across pages even if syncContext is not threaded through by the caller.
|
||||
*/
|
||||
let start = 0
|
||||
let collectedSoFar = (syncContext?.collectedCount as number | undefined) ?? 0
|
||||
if (cursor) {
|
||||
const sep = cursor.indexOf('|')
|
||||
if (sep > 0) {
|
||||
const parsedStart = Number(cursor.slice(0, sep))
|
||||
const parsedCount = Number(cursor.slice(sep + 1))
|
||||
if (Number.isFinite(parsedStart) && parsedStart >= 0) start = parsedStart
|
||||
if (Number.isFinite(parsedCount) && parsedCount >= 0) collectedSoFar = parsedCount
|
||||
} else {
|
||||
const parsedStart = Number(cursor)
|
||||
if (Number.isFinite(parsedStart) && parsedStart >= 0) start = parsedStart
|
||||
}
|
||||
}
|
||||
|
||||
const remaining = maxRequests > 0 ? Math.max(0, maxRequests - collectedSoFar) : PAGE_SIZE
|
||||
if (maxRequests > 0 && remaining === 0) {
|
||||
return { documents: [], hasMore: false }
|
||||
}
|
||||
|
||||
const params = new URLSearchParams({
|
||||
serviceDeskId,
|
||||
requestStatus,
|
||||
start: String(start),
|
||||
limit: String(Math.min(PAGE_SIZE, remaining)),
|
||||
})
|
||||
params.append('requestOwnership', requestOwnership)
|
||||
if (requestTypeId) params.append('requestTypeId', requestTypeId)
|
||||
if (searchTerm) params.append('searchTerm', searchTerm)
|
||||
|
||||
const url = `${baseUrl}/request?${params.toString()}`
|
||||
|
||||
logger.info('Listing JSM requests', {
|
||||
serviceDeskId,
|
||||
requestStatus,
|
||||
requestOwnership,
|
||||
hasCursor: Boolean(cursor),
|
||||
})
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: getJsmHeaders(accessToken),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error('Failed to list JSM requests', { status: response.status, error: errorText })
|
||||
throw new Error(`Failed to list JSM requests: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as JsmPage<JsmRequest>
|
||||
let requests = data.values ?? []
|
||||
|
||||
let slicedSome = false
|
||||
if (maxRequests > 0 && requests.length > remaining) {
|
||||
slicedSome = true
|
||||
requests = requests.slice(0, remaining)
|
||||
}
|
||||
|
||||
const documents = requests.map((request) => requestToStub(request, domain))
|
||||
|
||||
const newCollected = collectedSoFar + requests.length
|
||||
if (syncContext) syncContext.collectedCount = newCollected
|
||||
|
||||
const reachedCap = maxRequests > 0 && newCollected >= maxRequests
|
||||
|
||||
/**
|
||||
* When `maxRequests` truncates the listing before the source is exhausted,
|
||||
* flag the run as capped so the sync engine skips deletion reconciliation —
|
||||
* otherwise unseen requests beyond the cap would be deleted on every sync.
|
||||
* `slicedSome` covers truncation on the final page: requests dropped from
|
||||
* this page still exist even when `isLastPage` is true. (The requested
|
||||
* `limit` never exceeds the remaining budget, so a slice should be
|
||||
* impossible — this is defense in depth against the API over-returning.)
|
||||
*/
|
||||
if (((reachedCap && !data.isLastPage) || slicedSome) && syncContext) {
|
||||
syncContext.listingCapped = true
|
||||
}
|
||||
|
||||
const hasMore = !data.isLastPage && requests.length > 0 && !reachedCap
|
||||
const nextStart = start + requests.length
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore ? `${nextStart}|${newCollected}` : undefined,
|
||||
hasMore,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
externalId: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocument | null> => {
|
||||
const domain = sourceConfig.domain as string
|
||||
const { commentScope } = resolveOptions(sourceConfig)
|
||||
const cloudId = await resolveCloudId(domain, accessToken, syncContext)
|
||||
const baseUrl = getJsmApiBaseUrl(cloudId)
|
||||
|
||||
const requestUrl = `${baseUrl}/request/${encodeURIComponent(externalId)}?expand=status`
|
||||
const response = await fetchWithRetry(requestUrl, {
|
||||
method: 'GET',
|
||||
headers: getJsmHeaders(accessToken),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) return null
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
logger.warn('Access denied fetching JSM request', { externalId, status: response.status })
|
||||
return null
|
||||
}
|
||||
throw new Error(`Failed to get JSM request: ${response.status}`)
|
||||
}
|
||||
|
||||
const request = (await response.json()) as JsmRequest
|
||||
|
||||
const comments =
|
||||
commentScope === 'none'
|
||||
? []
|
||||
: await fetchComments(baseUrl, accessToken, externalId, commentScope === 'public')
|
||||
|
||||
const stub = requestToStub(request, domain)
|
||||
const content = buildContent(request, comments)
|
||||
|
||||
return {
|
||||
...stub,
|
||||
content,
|
||||
contentDeferred: false,
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const domain = sourceConfig.domain as string
|
||||
const serviceDeskId = sourceConfig.serviceDeskId as string
|
||||
|
||||
if (!domain || !serviceDeskId) {
|
||||
return { valid: false, error: 'Domain and service desk ID are required' }
|
||||
}
|
||||
|
||||
if (sourceConfig.maxRequests) {
|
||||
const max = Number(sourceConfig.maxRequests)
|
||||
if (Number.isNaN(max) || max <= 0) {
|
||||
return { valid: false, error: 'Max requests must be a positive number' }
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const cloudId = await getJiraCloudId(domain, accessToken, VALIDATE_RETRY_OPTIONS)
|
||||
const baseUrl = getJsmApiBaseUrl(cloudId)
|
||||
const url = `${baseUrl}/servicedesk/${encodeURIComponent(serviceDeskId)}`
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
url,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: getJsmHeaders(accessToken),
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) {
|
||||
return { valid: false, error: `Service desk "${serviceDeskId}" not found` }
|
||||
}
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
return {
|
||||
valid: false,
|
||||
error: 'Access denied. Check the connected account has access to this service desk.',
|
||||
}
|
||||
}
|
||||
const errorText = await response.text()
|
||||
return { valid: false, error: `Failed to validate: ${response.status} - ${errorText}` }
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
return { valid: false, error: toError(error).message || 'Failed to validate configuration' }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.status === 'string') result.status = metadata.status
|
||||
if (typeof metadata.requestTypeId === 'string') result.requestTypeId = metadata.requestTypeId
|
||||
if (typeof metadata.reporter === 'string') result.reporter = metadata.reporter
|
||||
|
||||
const created = parseTagDate(metadata.created)
|
||||
if (created) result.created = created
|
||||
|
||||
/**
|
||||
* The list endpoint exposes no true last-updated field; `statusDate` (time
|
||||
* of last status change) is the closest available signal and surfaces under
|
||||
* the "Last Status Change" tag.
|
||||
*/
|
||||
const statusDate = parseTagDate(metadata.statusDate)
|
||||
if (statusDate) result.updated = statusDate
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
import { JiraServiceManagementIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const jsmConnectorMeta: ConnectorMeta = {
|
||||
id: 'jsm',
|
||||
name: 'Jira Service Management',
|
||||
description: 'Sync service desk requests from Jira Service Management into your knowledge base',
|
||||
version: '1.0.0',
|
||||
icon: JiraServiceManagementIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'oauth',
|
||||
provider: 'jira',
|
||||
requiredScopes: [
|
||||
/**
|
||||
* Atlassian enforces granular scope sets all-or-nothing; the classic scope
|
||||
* alone authorizes the request read endpoints, so require it to flag stale
|
||||
* credentials that predate it in the provider scope list.
|
||||
*/
|
||||
'read:servicedesk-request',
|
||||
'read:servicedesk:jira-service-management',
|
||||
'read:request:jira-service-management',
|
||||
'read:request.comment:jira-service-management',
|
||||
'read:request.status:jira-service-management',
|
||||
/**
|
||||
* Requests embed a `reporter` user object whose `displayName` is surfaced
|
||||
* in document content and the Reporter tag. Atlassian only populates
|
||||
* embedded user data when the user-read scope is granted, so request it
|
||||
* here. Present in the `jira` OAuth provider config as `read:jira-user`.
|
||||
*/
|
||||
'read:jira-user',
|
||||
'offline_access',
|
||||
],
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'domain',
|
||||
title: 'Jira Domain',
|
||||
type: 'short-input',
|
||||
placeholder: 'yoursite.atlassian.net',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'serviceDeskSelector',
|
||||
title: 'Service Desk',
|
||||
type: 'selector',
|
||||
selectorKey: 'jsm.serviceDesks',
|
||||
canonicalParamId: 'serviceDeskId',
|
||||
mode: 'basic',
|
||||
dependsOn: ['domain'],
|
||||
placeholder: 'Select a service desk',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'serviceDeskId',
|
||||
title: 'Service Desk ID',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'serviceDeskId',
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. 1, 2',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'requestTypeSelector',
|
||||
title: 'Request Type',
|
||||
type: 'selector',
|
||||
selectorKey: 'jsm.requestTypes',
|
||||
canonicalParamId: 'requestTypeId',
|
||||
mode: 'basic',
|
||||
dependsOn: ['domain', 'serviceDeskSelector'],
|
||||
placeholder: 'All request types',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'requestTypeId',
|
||||
title: 'Request Type ID',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'requestTypeId',
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. 10 (leave blank for all)',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'requestStatus',
|
||||
title: 'Request Status',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
options: [
|
||||
{ label: 'All requests', id: 'ALL_REQUESTS' },
|
||||
{ label: 'Open requests', id: 'OPEN_REQUESTS' },
|
||||
{ label: 'Closed requests', id: 'CLOSED_REQUESTS' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'requestOwnership',
|
||||
title: 'Request Ownership',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
description:
|
||||
'Which requests the connected account can see. "Owned + participated" is the broadest scope a customer token can sync.',
|
||||
options: [
|
||||
{ label: 'Owned + participated', id: 'ALL_REQUESTS' },
|
||||
{ label: 'Owned only', id: 'OWNED_REQUESTS' },
|
||||
{ label: 'Participated only', id: 'PARTICIPATED_REQUESTS' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'comments',
|
||||
title: 'Include Comments',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
description: 'Comments require an extra API call per request during sync.',
|
||||
options: [
|
||||
{ label: 'Public comments only', id: 'public' },
|
||||
{ label: 'All comments (incl. internal)', id: 'all' },
|
||||
{ label: 'No comments', id: 'none' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'searchTerm',
|
||||
title: 'Search Filter',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. password reset (optional)',
|
||||
},
|
||||
{
|
||||
id: 'maxRequests',
|
||||
title: 'Max Requests',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 500 (default: unlimited)',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'status', displayName: 'Status', fieldType: 'text' },
|
||||
{ id: 'requestTypeId', displayName: 'Request Type', fieldType: 'text' },
|
||||
{ id: 'reporter', displayName: 'Reporter', fieldType: 'text' },
|
||||
{ id: 'created', displayName: 'Created', fieldType: 'date' },
|
||||
{ id: 'updated', displayName: 'Last Status Change', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { linearConnector } from '@/connectors/linear/linear'
|
||||
@@ -0,0 +1,377 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import type { RetryOptions } from '@/lib/knowledge/documents/utils'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { linearConnectorMeta } from '@/connectors/linear/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('LinearConnector')
|
||||
|
||||
const LINEAR_API = 'https://api.linear.app/graphql'
|
||||
|
||||
/**
|
||||
* Strips Markdown formatting to produce plain text.
|
||||
*/
|
||||
function markdownToPlainText(md: string): string {
|
||||
let text = md
|
||||
.replace(/!\[.*?\]\(.*?\)/g, '') // images
|
||||
.replace(/\[([^\]]*)\]\(.*?\)/g, '$1') // links
|
||||
.replace(/#{1,6}\s+/g, '') // headings
|
||||
.replace(/(\*\*|__)(.*?)\1/g, '$2') // bold
|
||||
.replace(/(\*|_)(.*?)\1/g, '$2') // italic
|
||||
.replace(/~~(.*?)~~/g, '$1') // strikethrough
|
||||
.replace(/`{3}[\s\S]*?`{3}/g, '') // code blocks
|
||||
.replace(/`([^`]*)`/g, '$1') // inline code
|
||||
.replace(/^\s*[-*+]\s+/gm, '') // list items
|
||||
.replace(/^\s*\d+\.\s+/gm, '') // ordered list items
|
||||
.replace(/^\s*>\s+/gm, '') // blockquotes
|
||||
.replace(/---+/g, '') // horizontal rules
|
||||
text = text.replace(/\s+/g, ' ').trim()
|
||||
return text
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes a GraphQL query against the Linear API.
|
||||
*/
|
||||
async function linearGraphQL(
|
||||
accessToken: string,
|
||||
query: string,
|
||||
variables?: Record<string, unknown>,
|
||||
retryOptions?: RetryOptions
|
||||
): Promise<Record<string, unknown>> {
|
||||
const response = await fetchWithRetry(
|
||||
LINEAR_API,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
body: JSON.stringify({ query, variables }),
|
||||
},
|
||||
retryOptions
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error('Linear GraphQL request failed', { status: response.status, error: errorText })
|
||||
throw new Error(`Linear API error: ${response.status}`)
|
||||
}
|
||||
|
||||
const json = (await response.json()) as { data?: Record<string, unknown>; errors?: unknown[] }
|
||||
if (json.errors) {
|
||||
logger.error('Linear GraphQL errors', { errors: json.errors })
|
||||
throw new Error(`Linear GraphQL error: ${JSON.stringify(json.errors)}`)
|
||||
}
|
||||
|
||||
return json.data as Record<string, unknown>
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a formatted text document from a Linear issue.
|
||||
*/
|
||||
function buildIssueContent(issue: Record<string, unknown>): string {
|
||||
const parts: string[] = []
|
||||
|
||||
const identifier = issue.identifier as string | undefined
|
||||
const title = (issue.title as string) || 'Untitled'
|
||||
parts.push(`${identifier ? `${identifier}: ` : ''}${title}`)
|
||||
|
||||
const state = issue.state as Record<string, unknown> | undefined
|
||||
if (state?.name) parts.push(`Status: ${state.name}`)
|
||||
|
||||
const priority = issue.priorityLabel as string | undefined
|
||||
if (priority) parts.push(`Priority: ${priority}`)
|
||||
|
||||
const assignee = issue.assignee as Record<string, unknown> | undefined
|
||||
if (assignee?.name) parts.push(`Assignee: ${assignee.name}`)
|
||||
|
||||
const labelsConn = issue.labels as Record<string, unknown> | undefined
|
||||
const labelNodes = (labelsConn?.nodes || []) as Record<string, unknown>[]
|
||||
if (labelNodes.length > 0) {
|
||||
parts.push(`Labels: ${labelNodes.map((l) => l.name as string).join(', ')}`)
|
||||
}
|
||||
|
||||
const description = issue.description as string | undefined
|
||||
if (description) {
|
||||
parts.push('')
|
||||
parts.push(markdownToPlainText(description))
|
||||
}
|
||||
|
||||
return parts.join('\n')
|
||||
}
|
||||
|
||||
const ISSUE_FIELDS = `
|
||||
id
|
||||
identifier
|
||||
title
|
||||
description
|
||||
priority
|
||||
priorityLabel
|
||||
url
|
||||
createdAt
|
||||
updatedAt
|
||||
state { name }
|
||||
assignee { name }
|
||||
labels { nodes { name } }
|
||||
team { name key }
|
||||
project { name }
|
||||
`
|
||||
|
||||
const ISSUE_BY_ID_QUERY = `
|
||||
query GetIssue($id: ID!) {
|
||||
issue(id: $id) {
|
||||
${ISSUE_FIELDS}
|
||||
}
|
||||
}
|
||||
`
|
||||
|
||||
const TEAMS_QUERY = `
|
||||
query { teams { nodes { id name key } } }
|
||||
`
|
||||
|
||||
/**
|
||||
* Dynamically builds a GraphQL issues query with only the filter clauses
|
||||
* that have values, preventing null comparators from being sent to Linear.
|
||||
*/
|
||||
function buildIssuesQuery(
|
||||
sourceConfig: Record<string, unknown>,
|
||||
teamIds: string[],
|
||||
projectIds: string[]
|
||||
): {
|
||||
query: string
|
||||
variables: Record<string, unknown>
|
||||
} {
|
||||
const stateFilter = (sourceConfig.stateFilter as string) || ''
|
||||
|
||||
const varDefs: string[] = ['$first: Int!', '$after: String']
|
||||
const filterClauses: string[] = []
|
||||
const variables: Record<string, unknown> = {}
|
||||
|
||||
if (teamIds.length === 1) {
|
||||
varDefs.push('$teamId: ID!')
|
||||
filterClauses.push('team: { id: { eq: $teamId } }')
|
||||
variables.teamId = teamIds[0]
|
||||
} else if (teamIds.length > 1) {
|
||||
varDefs.push('$teamIds: [ID!]!')
|
||||
filterClauses.push('team: { id: { in: $teamIds } }')
|
||||
variables.teamIds = teamIds
|
||||
}
|
||||
|
||||
if (projectIds.length === 1) {
|
||||
varDefs.push('$projectId: ID!')
|
||||
filterClauses.push('project: { id: { eq: $projectId } }')
|
||||
variables.projectId = projectIds[0]
|
||||
} else if (projectIds.length > 1) {
|
||||
varDefs.push('$projectIds: [ID!]!')
|
||||
filterClauses.push('project: { id: { in: $projectIds } }')
|
||||
variables.projectIds = projectIds
|
||||
}
|
||||
|
||||
if (stateFilter) {
|
||||
const states = stateFilter
|
||||
.split(',')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean)
|
||||
if (states.length > 0) {
|
||||
varDefs.push('$stateFilter: [String!]!')
|
||||
filterClauses.push('state: { name: { in: $stateFilter } }')
|
||||
variables.stateFilter = states
|
||||
}
|
||||
}
|
||||
|
||||
const filterArg = filterClauses.length > 0 ? `, filter: { ${filterClauses.join(', ')} }` : ''
|
||||
|
||||
const query = `
|
||||
query ListIssues(${varDefs.join(', ')}) {
|
||||
issues(first: $first, after: $after${filterArg}) {
|
||||
nodes {
|
||||
${ISSUE_FIELDS}
|
||||
}
|
||||
pageInfo {
|
||||
hasNextPage
|
||||
endCursor
|
||||
}
|
||||
}
|
||||
}
|
||||
`
|
||||
|
||||
return { query, variables }
|
||||
}
|
||||
|
||||
export const linearConnector: ConnectorConfig = {
|
||||
...linearConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const maxIssues = sourceConfig.maxIssues ? Number(sourceConfig.maxIssues) : 0
|
||||
const pageSize = maxIssues > 0 ? Math.min(maxIssues, 50) : 50
|
||||
|
||||
const teamIds = parseMultiValue(sourceConfig.teamId)
|
||||
const projectIds = parseMultiValue(sourceConfig.projectId)
|
||||
|
||||
const { query, variables } = buildIssuesQuery(sourceConfig, teamIds, projectIds)
|
||||
const allVars = { ...variables, first: pageSize, after: cursor || undefined }
|
||||
|
||||
logger.info('Listing Linear issues', {
|
||||
cursor,
|
||||
pageSize,
|
||||
teamFilterCount: teamIds.length,
|
||||
projectFilterCount: projectIds.length,
|
||||
})
|
||||
|
||||
const data = await linearGraphQL(accessToken, query, allVars)
|
||||
const issuesConn = data.issues as Record<string, unknown>
|
||||
const nodes = (issuesConn.nodes || []) as Record<string, unknown>[]
|
||||
const pageInfo = issuesConn.pageInfo as Record<string, unknown>
|
||||
|
||||
const documents: ExternalDocument[] = nodes.map((issue) => {
|
||||
const content = buildIssueContent(issue)
|
||||
const contentHash = `linear:${issue.id}:${issue.updatedAt}`
|
||||
|
||||
const labelNodes = ((issue.labels as Record<string, unknown>)?.nodes || []) as Record<
|
||||
string,
|
||||
unknown
|
||||
>[]
|
||||
|
||||
return {
|
||||
externalId: issue.id as string,
|
||||
title: `${(issue.identifier as string) || ''}: ${(issue.title as string) || 'Untitled'}`,
|
||||
content,
|
||||
mimeType: 'text/plain' as const,
|
||||
sourceUrl: (issue.url as string) || undefined,
|
||||
contentHash,
|
||||
metadata: {
|
||||
identifier: issue.identifier,
|
||||
state: (issue.state as Record<string, unknown>)?.name,
|
||||
priority: issue.priorityLabel,
|
||||
assignee: (issue.assignee as Record<string, unknown>)?.name,
|
||||
labels: labelNodes.map((l) => l.name as string),
|
||||
team: (issue.team as Record<string, unknown>)?.name,
|
||||
project: (issue.project as Record<string, unknown>)?.name,
|
||||
lastModified: issue.updatedAt,
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
const hasNextPage = Boolean(pageInfo.hasNextPage)
|
||||
const endCursor = (pageInfo.endCursor as string) || undefined
|
||||
|
||||
const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
const hitLimit = maxIssues > 0 && totalFetched >= maxIssues
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasNextPage && !hitLimit ? endCursor : undefined,
|
||||
hasMore: hasNextPage && !hitLimit,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
try {
|
||||
const data = await linearGraphQL(accessToken, ISSUE_BY_ID_QUERY, { id: externalId })
|
||||
const issue = data.issue as Record<string, unknown> | null
|
||||
|
||||
if (!issue) return null
|
||||
|
||||
const content = buildIssueContent(issue)
|
||||
const contentHash = `linear:${issue.id}:${issue.updatedAt}`
|
||||
|
||||
const labelNodes = ((issue.labels as Record<string, unknown>)?.nodes || []) as Record<
|
||||
string,
|
||||
unknown
|
||||
>[]
|
||||
|
||||
return {
|
||||
externalId: issue.id as string,
|
||||
title: `${(issue.identifier as string) || ''}: ${(issue.title as string) || 'Untitled'}`,
|
||||
content,
|
||||
mimeType: 'text/plain' as const,
|
||||
sourceUrl: (issue.url as string) || undefined,
|
||||
contentHash,
|
||||
metadata: {
|
||||
identifier: issue.identifier,
|
||||
state: (issue.state as Record<string, unknown>)?.name,
|
||||
priority: issue.priorityLabel,
|
||||
assignee: (issue.assignee as Record<string, unknown>)?.name,
|
||||
labels: labelNodes.map((l) => l.name as string),
|
||||
team: (issue.team as Record<string, unknown>)?.name,
|
||||
project: (issue.project as Record<string, unknown>)?.name,
|
||||
lastModified: issue.updatedAt,
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to get Linear issue', {
|
||||
externalId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const maxIssues = sourceConfig.maxIssues as string | undefined
|
||||
if (maxIssues && (Number.isNaN(Number(maxIssues)) || Number(maxIssues) <= 0)) {
|
||||
return { valid: false, error: 'Max issues must be a positive number' }
|
||||
}
|
||||
|
||||
try {
|
||||
const data = await linearGraphQL(accessToken, TEAMS_QUERY, undefined, VALIDATE_RETRY_OPTIONS)
|
||||
const teamsConn = data.teams as Record<string, unknown>
|
||||
const teams = (teamsConn.nodes || []) as Record<string, unknown>[]
|
||||
|
||||
if (teams.length === 0) {
|
||||
return {
|
||||
valid: false,
|
||||
error: 'No teams found — check that the OAuth token has read access',
|
||||
}
|
||||
}
|
||||
|
||||
const requestedTeamIds = parseMultiValue(sourceConfig.teamId)
|
||||
if (requestedTeamIds.length > 0) {
|
||||
const availableIds = new Set(teams.map((t) => t.id as string))
|
||||
const missing = requestedTeamIds.filter((id) => !availableIds.has(id))
|
||||
if (missing.length > 0) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `Team ID(s) not found: ${missing.join(', ')}. Available teams: ${teams.map((t) => `${t.name} (${t.id})`).join(', ')}`,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
const labels = joinTagArray(metadata.labels)
|
||||
if (labels) result.labels = labels
|
||||
|
||||
if (typeof metadata.state === 'string') result.state = metadata.state
|
||||
if (typeof metadata.priority === 'string') result.priority = metadata.priority
|
||||
if (typeof metadata.assignee === 'string') result.assignee = metadata.assignee
|
||||
|
||||
const lastModified = parseTagDate(metadata.lastModified)
|
||||
if (lastModified) result.lastModified = lastModified
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
import { LinearIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const linearConnectorMeta: ConnectorMeta = {
|
||||
id: 'linear',
|
||||
name: 'Linear',
|
||||
description: 'Sync issues from Linear',
|
||||
version: '1.0.0',
|
||||
icon: LinearIcon,
|
||||
|
||||
auth: { mode: 'oauth', provider: 'linear', requiredScopes: ['read'] },
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'teamSelector',
|
||||
title: 'Teams',
|
||||
type: 'selector',
|
||||
selectorKey: 'linear.teams',
|
||||
canonicalParamId: 'teamId',
|
||||
mode: 'basic',
|
||||
multi: true,
|
||||
placeholder: 'Select one or more teams (optional)',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'teamId',
|
||||
title: 'Team IDs',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'teamId',
|
||||
mode: 'advanced',
|
||||
multi: true,
|
||||
placeholder: 'e.g. abc123, def456 (comma-separated for multiple)',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'projectSelector',
|
||||
title: 'Projects',
|
||||
type: 'selector',
|
||||
selectorKey: 'linear.projects',
|
||||
canonicalParamId: 'projectId',
|
||||
mode: 'basic',
|
||||
multi: true,
|
||||
dependsOn: ['teamSelector'],
|
||||
placeholder: 'Select one or more projects (optional)',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'projectId',
|
||||
title: 'Project IDs',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'projectId',
|
||||
mode: 'advanced',
|
||||
multi: true,
|
||||
placeholder: 'e.g. def456, ghi789 (comma-separated for multiple)',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'stateFilter',
|
||||
title: 'State Filter',
|
||||
type: 'short-input',
|
||||
placeholder: 'e.g. In Progress, Todo',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'maxIssues',
|
||||
title: 'Max Issues',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 500 (default: unlimited)',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'labels', displayName: 'Labels', fieldType: 'text' },
|
||||
{ id: 'state', displayName: 'State', fieldType: 'text' },
|
||||
{ id: 'priority', displayName: 'Priority', fieldType: 'text' },
|
||||
{ id: 'assignee', displayName: 'Assignee', fieldType: 'text' },
|
||||
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { microsoftTeamsConnector } from '@/connectors/microsoft-teams/microsoft-teams'
|
||||
@@ -0,0 +1,77 @@
|
||||
import { MicrosoftTeamsIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const DEFAULT_MAX_MESSAGES = 1000
|
||||
|
||||
export const microsoftTeamsConnectorMeta: ConnectorMeta = {
|
||||
id: 'microsoft_teams',
|
||||
name: 'Microsoft Teams',
|
||||
description: 'Sync channel messages from Microsoft Teams',
|
||||
version: '1.0.0',
|
||||
icon: MicrosoftTeamsIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'oauth',
|
||||
provider: 'microsoft-teams',
|
||||
requiredScopes: ['ChannelMessage.Read.All', 'Channel.ReadBasic.All'],
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'teamSelector',
|
||||
title: 'Team',
|
||||
type: 'selector',
|
||||
selectorKey: 'microsoft.teams',
|
||||
canonicalParamId: 'teamId',
|
||||
mode: 'basic',
|
||||
placeholder: 'Select a team',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'teamId',
|
||||
title: 'Team ID',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'teamId',
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. fbe2bf47-16c8-47cf-b4a5-4b9b187c508b',
|
||||
required: true,
|
||||
description: 'The ID of the Microsoft Teams team',
|
||||
},
|
||||
{
|
||||
id: 'channelSelector',
|
||||
title: 'Channels',
|
||||
type: 'selector',
|
||||
selectorKey: 'microsoft.channels',
|
||||
canonicalParamId: 'channel',
|
||||
mode: 'basic',
|
||||
multi: true,
|
||||
dependsOn: ['teamSelector'],
|
||||
placeholder: 'Select one or more channels',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'channel',
|
||||
title: 'Channels',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'channel',
|
||||
mode: 'advanced',
|
||||
multi: true,
|
||||
placeholder: 'e.g. General, Announcements (comma-separated for multiple)',
|
||||
required: true,
|
||||
description: 'Channel names or IDs to sync messages from',
|
||||
},
|
||||
{
|
||||
id: 'maxMessages',
|
||||
title: 'Max Messages',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: `e.g. 500 (default: ${DEFAULT_MAX_MESSAGES})`,
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'channelName', displayName: 'Channel Name', fieldType: 'text' },
|
||||
{ id: 'messageCount', displayName: 'Message Count', fieldType: 'number' },
|
||||
{ id: 'lastActivity', displayName: 'Last Activity', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import {
|
||||
DEFAULT_MAX_MESSAGES,
|
||||
microsoftTeamsConnectorMeta,
|
||||
} from '@/connectors/microsoft-teams/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import {
|
||||
computeContentHash,
|
||||
htmlToPlainText,
|
||||
parseMultiValue,
|
||||
parseTagDate,
|
||||
} from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('MicrosoftTeamsConnector')
|
||||
|
||||
const GRAPH_API_BASE = 'https://graph.microsoft.com/v1.0'
|
||||
const MESSAGES_PER_PAGE = 50
|
||||
|
||||
interface TeamsMessage {
|
||||
id: string
|
||||
messageType: string
|
||||
createdDateTime: string
|
||||
lastModifiedDateTime?: string
|
||||
deletedDateTime?: string | null
|
||||
from?: {
|
||||
user?: {
|
||||
id: string
|
||||
displayName: string
|
||||
}
|
||||
application?: {
|
||||
id: string
|
||||
displayName: string
|
||||
}
|
||||
}
|
||||
body: {
|
||||
contentType: string
|
||||
content: string
|
||||
}
|
||||
subject?: string | null
|
||||
}
|
||||
|
||||
interface TeamsChannel {
|
||||
id: string
|
||||
displayName: string
|
||||
description?: string | null
|
||||
}
|
||||
|
||||
interface TeamsMessagesResponse {
|
||||
'@odata.nextLink'?: string
|
||||
value: TeamsMessage[]
|
||||
}
|
||||
|
||||
interface TeamsChannelsResponse {
|
||||
'@odata.nextLink'?: string
|
||||
value: TeamsChannel[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Calls the Microsoft Graph API with the given path and access token.
|
||||
*/
|
||||
async function graphApiGet<T>(
|
||||
path: string,
|
||||
accessToken: string,
|
||||
retryOptions?: Parameters<typeof fetchWithRetry>[2]
|
||||
): Promise<T> {
|
||||
const url = path.startsWith('https://') ? path : `${GRAPH_API_BASE}${path}`
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
url,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
Accept: 'application/json',
|
||||
},
|
||||
},
|
||||
retryOptions
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorBody = await response.text().catch(() => '')
|
||||
throw new Error(`Microsoft Graph API error: ${response.status} ${errorBody}`)
|
||||
}
|
||||
|
||||
return (await response.json()) as T
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches all messages from a channel, up to a maximum count, handling pagination.
|
||||
*/
|
||||
async function fetchChannelMessages(
|
||||
accessToken: string,
|
||||
teamId: string,
|
||||
channelId: string,
|
||||
maxMessages: number
|
||||
): Promise<{ messages: TeamsMessage[]; lastActivityTs?: string }> {
|
||||
const allMessages: TeamsMessage[] = []
|
||||
let nextLink: string | undefined
|
||||
let lastActivityTs: string | undefined
|
||||
|
||||
const initialPath = `/teams/${encodeURIComponent(teamId)}/channels/${encodeURIComponent(channelId)}/messages?$top=${Math.min(MESSAGES_PER_PAGE, maxMessages)}`
|
||||
|
||||
let currentUrl: string = initialPath
|
||||
|
||||
while (allMessages.length < maxMessages) {
|
||||
const data = await graphApiGet<TeamsMessagesResponse>(currentUrl, accessToken)
|
||||
const messages = data.value || []
|
||||
|
||||
if (messages.length === 0) break
|
||||
|
||||
// Filter to actual user messages (skip system/event messages)
|
||||
const userMessages = messages.filter(
|
||||
(msg) => msg.messageType === 'message' && !msg.deletedDateTime
|
||||
)
|
||||
|
||||
// Messages are sorted by lastModifiedDateTime (per Graph docs), so the first
|
||||
// user message on the first page reflects the most recent activity.
|
||||
if (!lastActivityTs && userMessages.length > 0) {
|
||||
const first = userMessages[0]
|
||||
lastActivityTs = first.lastModifiedDateTime || first.createdDateTime
|
||||
}
|
||||
|
||||
allMessages.push(...userMessages)
|
||||
|
||||
nextLink = data['@odata.nextLink']
|
||||
if (!nextLink) break
|
||||
currentUrl = nextLink
|
||||
}
|
||||
|
||||
return { messages: allMessages.slice(0, maxMessages), lastActivityTs }
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts fetched messages into a single document content string.
|
||||
* Each line: "[ISO timestamp] username: message text"
|
||||
*/
|
||||
function formatMessages(messages: TeamsMessage[]): string {
|
||||
const lines: string[] = []
|
||||
|
||||
// Process in reverse so oldest messages come first
|
||||
const chronological = [...messages].reverse()
|
||||
|
||||
for (const msg of chronological) {
|
||||
const bodyText =
|
||||
msg.body.contentType === 'html' ? htmlToPlainText(msg.body.content) : msg.body.content
|
||||
|
||||
if (!bodyText.trim()) continue
|
||||
|
||||
const timestamp = msg.createdDateTime
|
||||
const userName = msg.from?.user?.displayName || msg.from?.application?.displayName || 'unknown'
|
||||
|
||||
lines.push(`[${timestamp}] ${userName}: ${bodyText}`)
|
||||
}
|
||||
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a channel name or ID to a channel object within the given team.
|
||||
*/
|
||||
async function resolveChannel(
|
||||
accessToken: string,
|
||||
teamId: string,
|
||||
channelInput: string
|
||||
): Promise<TeamsChannel | null> {
|
||||
const trimmed = channelInput.trim()
|
||||
|
||||
// Fetch all channels for the team
|
||||
let nextLink: string | undefined
|
||||
// $select avoids the expensive `email` property per Graph perf guidance.
|
||||
const initialPath = `/teams/${encodeURIComponent(teamId)}/channels?$select=id,displayName,description`
|
||||
let currentUrl: string = initialPath
|
||||
|
||||
do {
|
||||
const data = await graphApiGet<TeamsChannelsResponse>(currentUrl, accessToken)
|
||||
const channels = data.value || []
|
||||
|
||||
// Try matching by ID first, then by display name (case-insensitive)
|
||||
const match = channels.find(
|
||||
(ch) => ch.id === trimmed || ch.displayName.toLowerCase() === trimmed.toLowerCase()
|
||||
)
|
||||
if (match) return match
|
||||
|
||||
nextLink = data['@odata.nextLink']
|
||||
if (nextLink) {
|
||||
currentUrl = nextLink
|
||||
}
|
||||
} while (nextLink)
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
export const microsoftTeamsConnector: ConnectorConfig = {
|
||||
...microsoftTeamsConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
_cursor?: string,
|
||||
_syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const teamId = sourceConfig.teamId as string
|
||||
const channelInputs = parseMultiValue(sourceConfig.channel)
|
||||
if (!teamId?.trim()) {
|
||||
throw new Error('Team ID is required')
|
||||
}
|
||||
if (channelInputs.length === 0) {
|
||||
throw new Error('At least one channel is required')
|
||||
}
|
||||
|
||||
const maxMessages = sourceConfig.maxMessages
|
||||
? Number(sourceConfig.maxMessages)
|
||||
: DEFAULT_MAX_MESSAGES
|
||||
|
||||
logger.info('Syncing Microsoft Teams channels', {
|
||||
teamId,
|
||||
channels: channelInputs,
|
||||
maxMessages,
|
||||
})
|
||||
|
||||
const documents: ExternalDocument[] = []
|
||||
|
||||
for (const channelInput of channelInputs) {
|
||||
const channel = await resolveChannel(accessToken, teamId, channelInput)
|
||||
if (!channel) {
|
||||
throw new Error(`Channel not found: ${channelInput}`)
|
||||
}
|
||||
|
||||
const { messages, lastActivityTs } = await fetchChannelMessages(
|
||||
accessToken,
|
||||
teamId,
|
||||
channel.id,
|
||||
maxMessages
|
||||
)
|
||||
|
||||
const content = formatMessages(messages)
|
||||
if (!content.trim()) {
|
||||
logger.info(`No messages found in channel: ${channel.displayName}`)
|
||||
continue
|
||||
}
|
||||
|
||||
const contentHash = await computeContentHash(content)
|
||||
|
||||
const sourceUrl = `https://teams.microsoft.com/l/channel/${encodeURIComponent(channel.id)}/${encodeURIComponent(channel.displayName)}?groupId=${encodeURIComponent(teamId)}`
|
||||
|
||||
documents.push({
|
||||
externalId: channel.id,
|
||||
title: channel.displayName,
|
||||
content,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl,
|
||||
contentHash,
|
||||
metadata: {
|
||||
channelName: channel.displayName,
|
||||
messageCount: messages.length,
|
||||
lastActivity: lastActivityTs || undefined,
|
||||
description: channel.description || undefined,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// All selected channels are emitted in a single page; no pagination needed
|
||||
return {
|
||||
documents,
|
||||
hasMore: false,
|
||||
}
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
const teamId = sourceConfig.teamId as string
|
||||
if (!teamId?.trim()) {
|
||||
return null
|
||||
}
|
||||
|
||||
const maxMessages = sourceConfig.maxMessages
|
||||
? Number(sourceConfig.maxMessages)
|
||||
: DEFAULT_MAX_MESSAGES
|
||||
|
||||
try {
|
||||
// Fetch channel info
|
||||
const channelPath = `/teams/${encodeURIComponent(teamId)}/channels/${encodeURIComponent(externalId)}`
|
||||
const channel = await graphApiGet<TeamsChannel>(channelPath, accessToken)
|
||||
|
||||
const { messages, lastActivityTs } = await fetchChannelMessages(
|
||||
accessToken,
|
||||
teamId,
|
||||
externalId,
|
||||
maxMessages
|
||||
)
|
||||
|
||||
const content = formatMessages(messages)
|
||||
if (!content.trim()) return null
|
||||
|
||||
const contentHash = await computeContentHash(content)
|
||||
|
||||
const sourceUrl = `https://teams.microsoft.com/l/channel/${encodeURIComponent(channel.id)}/${encodeURIComponent(channel.displayName)}?groupId=${encodeURIComponent(teamId)}`
|
||||
|
||||
return {
|
||||
externalId: channel.id,
|
||||
title: channel.displayName,
|
||||
content,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl,
|
||||
contentHash,
|
||||
metadata: {
|
||||
channelName: channel.displayName,
|
||||
messageCount: messages.length,
|
||||
lastActivity: lastActivityTs || undefined,
|
||||
description: channel.description || undefined,
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Failed to get Microsoft Teams channel document', {
|
||||
externalId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const teamId = sourceConfig.teamId as string | undefined
|
||||
const channelInputs = parseMultiValue(sourceConfig.channel)
|
||||
const maxMessages = sourceConfig.maxMessages as string | undefined
|
||||
|
||||
if (!teamId?.trim()) {
|
||||
return { valid: false, error: 'Team ID is required' }
|
||||
}
|
||||
|
||||
if (channelInputs.length === 0) {
|
||||
return { valid: false, error: 'At least one channel is required' }
|
||||
}
|
||||
|
||||
if (maxMessages && (Number.isNaN(Number(maxMessages)) || Number(maxMessages) <= 0)) {
|
||||
return { valid: false, error: 'Max messages must be a positive number' }
|
||||
}
|
||||
|
||||
try {
|
||||
for (const channelInput of channelInputs) {
|
||||
const channel = await resolveChannel(accessToken, teamId, channelInput)
|
||||
if (!channel) {
|
||||
return { valid: false, error: `Channel not found: ${channelInput}` }
|
||||
}
|
||||
|
||||
// Verify we can read messages by fetching a single message
|
||||
const messagesPath = `/teams/${encodeURIComponent(teamId)}/channels/${encodeURIComponent(channel.id)}/messages?$top=1`
|
||||
await graphApiGet<TeamsMessagesResponse>(messagesPath, accessToken, VALIDATE_RETRY_OPTIONS)
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.channelName === 'string') {
|
||||
result.channelName = metadata.channelName
|
||||
}
|
||||
|
||||
if (typeof metadata.messageCount === 'number') {
|
||||
result.messageCount = metadata.messageCount
|
||||
}
|
||||
|
||||
const lastActivity = parseTagDate(metadata.lastActivity)
|
||||
if (lastActivity) {
|
||||
result.lastActivity = lastActivity
|
||||
}
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { mondayConnector } from '@/connectors/monday/monday'
|
||||
@@ -0,0 +1,61 @@
|
||||
import { MondayIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const mondayConnectorMeta: ConnectorMeta = {
|
||||
id: 'monday',
|
||||
name: 'Monday.com',
|
||||
description: 'Sync board items and updates from Monday.com into your knowledge base',
|
||||
version: '1.0.0',
|
||||
icon: MondayIcon,
|
||||
|
||||
auth: {
|
||||
mode: 'oauth',
|
||||
provider: 'monday',
|
||||
requiredScopes: ['boards:read', 'updates:read', 'me:read'],
|
||||
},
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'boardSelector',
|
||||
title: 'Boards',
|
||||
type: 'selector',
|
||||
selectorKey: 'monday.boards',
|
||||
canonicalParamId: 'boardIds',
|
||||
mode: 'basic',
|
||||
multi: true,
|
||||
required: false,
|
||||
placeholder: 'Select boards (empty = all active boards)',
|
||||
description:
|
||||
'Boards to sync. Leave empty to sync items from every active board you can access.',
|
||||
},
|
||||
{
|
||||
id: 'boardIds',
|
||||
title: 'Board IDs',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'boardIds',
|
||||
mode: 'advanced',
|
||||
multi: true,
|
||||
required: false,
|
||||
placeholder: 'e.g. 1234567890, 9876543210 (empty = all active boards)',
|
||||
description:
|
||||
'Comma-separated board IDs to sync — find a board ID in its URL (.../boards/<id>). Leave empty to sync items from every active board you can access.',
|
||||
},
|
||||
{
|
||||
id: 'maxItems',
|
||||
title: 'Max Items',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 500 (default: unlimited)',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'boardName', displayName: 'Board', fieldType: 'text' },
|
||||
{ id: 'groupTitle', displayName: 'Group', fieldType: 'text' },
|
||||
{ id: 'itemName', displayName: 'Item', fieldType: 'text' },
|
||||
{ id: 'state', displayName: 'State', fieldType: 'text' },
|
||||
{ id: 'creatorName', displayName: 'Creator', fieldType: 'text' },
|
||||
{ id: 'createdAt', displayName: 'Created', fieldType: 'date' },
|
||||
{ id: 'updatedAt', displayName: 'Last Updated', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,514 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { mondayConnectorMeta } from '@/connectors/monday/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { parseMultiValue, parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('MondayConnector')
|
||||
|
||||
/**
|
||||
* monday.com GraphQL endpoint. All requests are POSTed here.
|
||||
* @see https://developer.monday.com/api-reference/docs/basics
|
||||
*/
|
||||
const MONDAY_API_URL = 'https://api.monday.com/v2'
|
||||
|
||||
/**
|
||||
* Stable monday.com API version pinned via the `API-Version` header. monday.com
|
||||
* keeps at least three quarterly versions live; `2024-10` was deprecated on
|
||||
* 2026-02-15, so this is pinned to the current stable release.
|
||||
* @see https://developer.monday.com/api-reference/docs/api-versioning
|
||||
*/
|
||||
const MONDAY_API_VERSION = '2026-04'
|
||||
|
||||
/** Max items requested per `items_page` / `next_items_page` call (monday.com max is 500). */
|
||||
const ITEMS_PAGE_SIZE = 100
|
||||
|
||||
/** Max boards requested per `boards` listing page (monday.com max is 500). */
|
||||
const BOARDS_PAGE_SIZE = 100
|
||||
|
||||
/** Max updates fetched per item for content extraction. */
|
||||
const UPDATES_LIMIT = 50
|
||||
|
||||
interface MondayColumnValue {
|
||||
id: string
|
||||
text: string | null
|
||||
column: { id: string; title: string } | null
|
||||
}
|
||||
|
||||
interface MondayUpdate {
|
||||
id: string
|
||||
text_body: string | null
|
||||
created_at: string | null
|
||||
creator: { name: string | null } | null
|
||||
}
|
||||
|
||||
interface MondayItem {
|
||||
id: string
|
||||
name: string | null
|
||||
state: string | null
|
||||
created_at: string | null
|
||||
updated_at: string | null
|
||||
url: string | null
|
||||
board: { id: string; name: string | null } | null
|
||||
group: { id: string; title: string | null } | null
|
||||
creator: { name: string | null } | null
|
||||
column_values: MondayColumnValue[]
|
||||
updates: MondayUpdate[]
|
||||
}
|
||||
|
||||
interface MondayItemsPage {
|
||||
cursor: string | null
|
||||
items: MondayItem[]
|
||||
}
|
||||
|
||||
interface MondayBoard {
|
||||
id: string
|
||||
name: string | null
|
||||
items_page: MondayItemsPage | null
|
||||
}
|
||||
|
||||
/**
|
||||
* Pagination state encoded into `nextCursor`. Tracks which board in the
|
||||
* configured/accessible list is being read (`boardIndex`) and the opaque
|
||||
* `items_page` cursor within that board (`itemsCursor`).
|
||||
*/
|
||||
interface CursorState {
|
||||
boardIndex: number
|
||||
itemsCursor?: string
|
||||
}
|
||||
|
||||
function encodeCursor(state: CursorState): string {
|
||||
return Buffer.from(JSON.stringify(state), 'utf8').toString('base64url')
|
||||
}
|
||||
|
||||
function decodeCursor(cursor?: string): CursorState {
|
||||
if (!cursor) return { boardIndex: 0 }
|
||||
try {
|
||||
const json = Buffer.from(cursor, 'base64url').toString('utf8')
|
||||
const parsed = JSON.parse(json) as Partial<CursorState>
|
||||
return {
|
||||
boardIndex: Number(parsed.boardIndex) || 0,
|
||||
itemsCursor: typeof parsed.itemsCursor === 'string' ? parsed.itemsCursor : undefined,
|
||||
}
|
||||
} catch {
|
||||
return { boardIndex: 0 }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* monday.com uses the raw access token in the `Authorization` header — it is NOT
|
||||
* prefixed with "Bearer". The `API-Version` header pins the schema version.
|
||||
* @see https://developer.monday.com/api-reference/docs/authentication
|
||||
*/
|
||||
function mondayHeaders(accessToken: string): Record<string, string> {
|
||||
return {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: accessToken,
|
||||
'API-Version': MONDAY_API_VERSION,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes a GraphQL query against the monday.com API, surfacing GraphQL-level
|
||||
* errors (which return HTTP 200 with an `errors` array) as thrown errors.
|
||||
*/
|
||||
async function mondayGraphQL<T>(
|
||||
accessToken: string,
|
||||
query: string,
|
||||
variables: Record<string, unknown> = {},
|
||||
retryOptions?: Parameters<typeof fetchWithRetry>[2]
|
||||
): Promise<T> {
|
||||
const response = await fetchWithRetry(
|
||||
MONDAY_API_URL,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: mondayHeaders(accessToken),
|
||||
body: JSON.stringify({ query, variables }),
|
||||
},
|
||||
retryOptions
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
throw new Error(
|
||||
`monday.com API HTTP error: ${response.status}${errorText ? ` — ${errorText.slice(0, 200)}` : ''}`
|
||||
)
|
||||
}
|
||||
|
||||
const data = (await response.json()) as {
|
||||
data?: T
|
||||
errors?: { message?: string }[]
|
||||
error_message?: string
|
||||
}
|
||||
|
||||
if (data.errors && data.errors.length > 0) {
|
||||
const message = data.errors
|
||||
.map((e) => e.message)
|
||||
.filter(Boolean)
|
||||
.join('; ')
|
||||
throw new Error(`monday.com API error: ${message || 'Unknown GraphQL error'}`)
|
||||
}
|
||||
if (data.error_message) {
|
||||
throw new Error(`monday.com API error: ${data.error_message}`)
|
||||
}
|
||||
|
||||
return data.data as T
|
||||
}
|
||||
|
||||
/**
|
||||
* GraphQL selection set for an item, shared between listing and single-item
|
||||
* fetches so the resolved fields stay in sync.
|
||||
*/
|
||||
const ITEM_FIELDS = `
|
||||
id
|
||||
name
|
||||
state
|
||||
created_at
|
||||
updated_at
|
||||
url
|
||||
board { id name }
|
||||
group { id title }
|
||||
creator { name }
|
||||
column_values {
|
||||
id
|
||||
text
|
||||
column { id title }
|
||||
}
|
||||
updates(limit: ${UPDATES_LIMIT}) {
|
||||
id
|
||||
text_body
|
||||
created_at
|
||||
creator { name }
|
||||
}
|
||||
`
|
||||
|
||||
/**
|
||||
* Builds the change-detection hash from item identity + last-modified time. Must
|
||||
* be identical whether produced inline during listing or via a `getDocument`
|
||||
* fetch, since monday's `updated_at` advances on any item or column change.
|
||||
*/
|
||||
function buildContentHash(itemId: string, updatedAt: string | null | undefined): string {
|
||||
return `monday:${itemId}:${updatedAt ?? ''}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a stable board name + id for an item, preferring the nested `board`
|
||||
* field and falling back to the board the item was listed under.
|
||||
*/
|
||||
function resolveBoard(
|
||||
item: MondayItem,
|
||||
fallback?: { id: string; name: string | null }
|
||||
): { id: string; name: string } {
|
||||
const id = item.board?.id ?? fallback?.id ?? ''
|
||||
const name = item.board?.name ?? fallback?.name ?? ''
|
||||
return { id, name }
|
||||
}
|
||||
|
||||
function buildSourceUrl(item: MondayItem): string | undefined {
|
||||
return item.url ?? undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the metadata object carried on every document, used both for tag mapping
|
||||
* (`mapTags`) and for downstream display. Kept in one place so listing and
|
||||
* single-item fetches emit identical metadata.
|
||||
*/
|
||||
function itemMetadata(
|
||||
item: MondayItem,
|
||||
board: { id: string; name: string }
|
||||
): Record<string, unknown> {
|
||||
return {
|
||||
boardId: board.id,
|
||||
boardName: board.name,
|
||||
itemName: item.name ?? '',
|
||||
groupTitle: item.group?.title ?? '',
|
||||
state: item.state ?? '',
|
||||
creatorName: item.creator?.name ?? '',
|
||||
createdAt: item.created_at ?? undefined,
|
||||
updatedAt: item.updated_at ?? undefined,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Produces a fully-hydrated document from a single `items_page` element. The list
|
||||
* query already selects the complete item payload (`column_values` + `updates`),
|
||||
* so content is built inline here — avoiding a redundant per-item `getDocument`
|
||||
* round-trip. `contentHash` derives solely from id + `updated_at`, so it is
|
||||
* identical whether produced here or via `getDocument`.
|
||||
*/
|
||||
function itemToDocument(
|
||||
item: MondayItem,
|
||||
fallbackBoard?: { id: string; name: string | null }
|
||||
): ExternalDocument {
|
||||
const board = resolveBoard(item, fallbackBoard)
|
||||
return {
|
||||
externalId: item.id,
|
||||
title: item.name?.trim() || 'Untitled Item',
|
||||
content: formatItemContent(item, board),
|
||||
contentDeferred: false,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: buildSourceUrl(item),
|
||||
contentHash: buildContentHash(item.id, item.updated_at),
|
||||
metadata: itemMetadata(item, board),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats an item's column values and updates into a plain-text document. The
|
||||
* resolved board is passed in so listing (which has a board fallback) and
|
||||
* single-item fetches produce identical content.
|
||||
*/
|
||||
function formatItemContent(item: MondayItem, board: { id: string; name: string }): string {
|
||||
const parts: string[] = []
|
||||
|
||||
if (board.name) parts.push(`Board: ${board.name}`)
|
||||
parts.push(`Item: ${item.name?.trim() || 'Untitled Item'}`)
|
||||
if (item.group?.title) parts.push(`Group: ${item.group.title}`)
|
||||
if (item.creator?.name) parts.push(`Created by: ${item.creator.name}`)
|
||||
if (item.created_at) parts.push(`Created: ${item.created_at}`)
|
||||
if (item.updated_at) parts.push(`Updated: ${item.updated_at}`)
|
||||
|
||||
const columns = item.column_values.filter((cv) => cv.text?.trim())
|
||||
if (columns.length > 0) {
|
||||
parts.push('')
|
||||
parts.push('--- Fields ---')
|
||||
for (const cv of columns) {
|
||||
const title = cv.column?.title?.trim() || cv.id
|
||||
parts.push(`${title}: ${cv.text}`)
|
||||
}
|
||||
}
|
||||
|
||||
const updates = item.updates.filter((u) => u.text_body?.trim())
|
||||
if (updates.length > 0) {
|
||||
parts.push('')
|
||||
parts.push('--- Updates ---')
|
||||
for (const update of updates) {
|
||||
const author = update.creator?.name?.trim() || 'Unknown'
|
||||
parts.push(`Update by ${author}: ${update.text_body}`)
|
||||
}
|
||||
}
|
||||
|
||||
return parts.join('\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the list of board ids the connector should sync. When `boardIds` is
|
||||
* configured, those are used verbatim; otherwise all accessible active boards
|
||||
* are enumerated.
|
||||
*/
|
||||
async function resolveBoardIds(
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ id: string; name: string | null }[]> {
|
||||
const configured = parseMultiValue(sourceConfig.boardIds)
|
||||
if (configured.length > 0) {
|
||||
return configured.map((id) => ({ id, name: null }))
|
||||
}
|
||||
|
||||
const boards: { id: string; name: string | null }[] = []
|
||||
let page = 1
|
||||
for (;;) {
|
||||
const data = await mondayGraphQL<{ boards: { id: string; name: string | null }[] | null }>(
|
||||
accessToken,
|
||||
`query ($limit: Int!, $page: Int!) {
|
||||
boards(limit: $limit, page: $page, state: active) {
|
||||
id
|
||||
name
|
||||
}
|
||||
}`,
|
||||
{ limit: BOARDS_PAGE_SIZE, page }
|
||||
)
|
||||
const batch = data.boards ?? []
|
||||
boards.push(...batch)
|
||||
if (batch.length < BOARDS_PAGE_SIZE) break
|
||||
page += 1
|
||||
}
|
||||
return boards
|
||||
}
|
||||
|
||||
export const mondayConnector: ConnectorConfig = {
|
||||
...mondayConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const maxItems = sourceConfig.maxItems ? Number(sourceConfig.maxItems) : 0
|
||||
const state = decodeCursor(cursor)
|
||||
|
||||
const boards =
|
||||
(syncContext?.boards as { id: string; name: string | null }[] | undefined) ??
|
||||
(await resolveBoardIds(accessToken, sourceConfig))
|
||||
if (syncContext) syncContext.boards = boards
|
||||
|
||||
if (state.boardIndex >= boards.length) {
|
||||
return { documents: [], hasMore: false }
|
||||
}
|
||||
|
||||
const board = boards[state.boardIndex]
|
||||
const fallbackBoard = { id: board.id, name: board.name }
|
||||
|
||||
const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
|
||||
const pageLimit =
|
||||
maxItems > 0
|
||||
? Math.min(ITEMS_PAGE_SIZE, Math.max(1, maxItems - prevFetched))
|
||||
: ITEMS_PAGE_SIZE
|
||||
|
||||
let itemsPage: MondayItemsPage | null
|
||||
if (state.itemsCursor) {
|
||||
const data = await mondayGraphQL<{ next_items_page: MondayItemsPage | null }>(
|
||||
accessToken,
|
||||
`query ($cursor: String!, $limit: Int!) {
|
||||
next_items_page(cursor: $cursor, limit: $limit) {
|
||||
cursor
|
||||
items { ${ITEM_FIELDS} }
|
||||
}
|
||||
}`,
|
||||
{ cursor: state.itemsCursor, limit: pageLimit }
|
||||
)
|
||||
itemsPage = data.next_items_page
|
||||
} else {
|
||||
const data = await mondayGraphQL<{ boards: MondayBoard[] | null }>(
|
||||
accessToken,
|
||||
`query ($ids: [ID!], $limit: Int!) {
|
||||
boards(ids: $ids) {
|
||||
id
|
||||
name
|
||||
items_page(limit: $limit) {
|
||||
cursor
|
||||
items { ${ITEM_FIELDS} }
|
||||
}
|
||||
}
|
||||
}`,
|
||||
{ ids: [board.id], limit: pageLimit }
|
||||
)
|
||||
itemsPage = data.boards?.[0]?.items_page ?? null
|
||||
}
|
||||
|
||||
const items = itemsPage?.items ?? []
|
||||
const nextItemsCursor = itemsPage?.cursor?.trim() || undefined
|
||||
|
||||
logger.info('Listing Monday.com items', {
|
||||
boardIndex: state.boardIndex,
|
||||
boardTotal: boards.length,
|
||||
boardId: board.id,
|
||||
itemCount: items.length,
|
||||
hasItemsCursor: Boolean(state.itemsCursor),
|
||||
})
|
||||
|
||||
const allDocuments = items.map((item) => itemToDocument(item, fallbackBoard))
|
||||
|
||||
let documents = allDocuments
|
||||
if (maxItems > 0) {
|
||||
const remaining = Math.max(0, maxItems - prevFetched)
|
||||
if (allDocuments.length > remaining) {
|
||||
documents = allDocuments.slice(0, remaining)
|
||||
}
|
||||
}
|
||||
|
||||
const totalFetched = prevFetched + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
const hitLimit = maxItems > 0 && totalFetched >= maxItems
|
||||
if (hitLimit && syncContext) syncContext.listingCapped = true
|
||||
|
||||
let nextCursor: string | undefined
|
||||
let hasMore = false
|
||||
|
||||
if (hitLimit) {
|
||||
nextCursor = undefined
|
||||
} else if (nextItemsCursor) {
|
||||
nextCursor = encodeCursor({ boardIndex: state.boardIndex, itemsCursor: nextItemsCursor })
|
||||
hasMore = true
|
||||
} else if (state.boardIndex + 1 < boards.length) {
|
||||
nextCursor = encodeCursor({ boardIndex: state.boardIndex + 1 })
|
||||
hasMore = true
|
||||
}
|
||||
|
||||
return { documents, nextCursor, hasMore }
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
_sourceConfig: Record<string, unknown>,
|
||||
externalId: string
|
||||
): Promise<ExternalDocument | null> => {
|
||||
try {
|
||||
if (!externalId) return null
|
||||
|
||||
const data = await mondayGraphQL<{ items: MondayItem[] | null }>(
|
||||
accessToken,
|
||||
`query ($ids: [ID!]) {
|
||||
items(ids: $ids) { ${ITEM_FIELDS} }
|
||||
}`,
|
||||
{ ids: [externalId] }
|
||||
)
|
||||
|
||||
const item = data.items?.[0]
|
||||
if (!item) return null
|
||||
|
||||
const doc = itemToDocument(item)
|
||||
if (!doc.content.trim()) return null
|
||||
|
||||
return doc
|
||||
} catch (error) {
|
||||
logger.warn('Failed to get Monday.com item', {
|
||||
externalId,
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const maxItems = sourceConfig.maxItems as string | undefined
|
||||
if (maxItems && (Number.isNaN(Number(maxItems)) || Number(maxItems) < 0)) {
|
||||
return { valid: false, error: 'Max items must be a non-negative number' }
|
||||
}
|
||||
|
||||
try {
|
||||
await mondayGraphQL(accessToken, `query { me { id } }`, {}, VALIDATE_RETRY_OPTIONS)
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
if (typeof metadata.boardName === 'string' && metadata.boardName.trim()) {
|
||||
result.boardName = metadata.boardName
|
||||
}
|
||||
|
||||
if (typeof metadata.groupTitle === 'string' && metadata.groupTitle.trim()) {
|
||||
result.groupTitle = metadata.groupTitle
|
||||
}
|
||||
|
||||
if (typeof metadata.itemName === 'string' && metadata.itemName.trim()) {
|
||||
result.itemName = metadata.itemName
|
||||
}
|
||||
|
||||
if (typeof metadata.state === 'string' && metadata.state.trim()) {
|
||||
result.state = metadata.state
|
||||
}
|
||||
|
||||
if (typeof metadata.creatorName === 'string' && metadata.creatorName.trim()) {
|
||||
result.creatorName = metadata.creatorName
|
||||
}
|
||||
|
||||
const createdAt = parseTagDate(metadata.createdAt)
|
||||
if (createdAt) result.createdAt = createdAt
|
||||
|
||||
const updatedAt = parseTagDate(metadata.updatedAt)
|
||||
if (updatedAt) result.updatedAt = updatedAt
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { notionConnector } from '@/connectors/notion/notion'
|
||||
@@ -0,0 +1,86 @@
|
||||
import { NotionIcon } from '@/components/icons'
|
||||
import type { ConnectorMeta } from '@/connectors/types'
|
||||
|
||||
export const notionConnectorMeta: ConnectorMeta = {
|
||||
id: 'notion',
|
||||
name: 'Notion',
|
||||
description: 'Sync pages from a Notion workspace',
|
||||
version: '1.0.0',
|
||||
icon: NotionIcon,
|
||||
|
||||
auth: { mode: 'oauth', provider: 'notion', requiredScopes: [] },
|
||||
|
||||
configFields: [
|
||||
{
|
||||
id: 'scope',
|
||||
title: 'Sync Scope',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
options: [
|
||||
{ label: 'Entire workspace', id: 'workspace' },
|
||||
{ label: 'Specific database', id: 'database' },
|
||||
{ label: 'Specific page (and children)', id: 'page' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'databaseSelector',
|
||||
title: 'Databases',
|
||||
type: 'selector',
|
||||
selectorKey: 'notion.databases',
|
||||
canonicalParamId: 'databaseId',
|
||||
mode: 'basic',
|
||||
multi: true,
|
||||
placeholder: 'Select one or more databases',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'databaseId',
|
||||
title: 'Database IDs',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'databaseId',
|
||||
mode: 'advanced',
|
||||
multi: true,
|
||||
required: false,
|
||||
placeholder: 'e.g. 8a3b5f6e-..., 9c4d6e7f-... (comma-separated for multiple)',
|
||||
},
|
||||
{
|
||||
id: 'rootPageSelector',
|
||||
title: 'Page',
|
||||
type: 'selector',
|
||||
selectorKey: 'notion.pages',
|
||||
canonicalParamId: 'rootPageId',
|
||||
mode: 'basic',
|
||||
placeholder: 'Select a page',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'rootPageId',
|
||||
title: 'Page ID',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'rootPageId',
|
||||
mode: 'advanced',
|
||||
required: false,
|
||||
placeholder: 'e.g. 8a3b5f6e-1234-5678-abcd-ef0123456789',
|
||||
},
|
||||
{
|
||||
id: 'searchQuery',
|
||||
title: 'Search Filter',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. meeting notes, project plan',
|
||||
},
|
||||
{
|
||||
id: 'maxPages',
|
||||
title: 'Max Pages',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
placeholder: 'e.g. 500 (default: unlimited)',
|
||||
},
|
||||
],
|
||||
|
||||
tagDefinitions: [
|
||||
{ id: 'tags', displayName: 'Tags', fieldType: 'text' },
|
||||
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
|
||||
{ id: 'created', displayName: 'Created', fieldType: 'date' },
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,634 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage, toError } from '@sim/utils/errors'
|
||||
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
|
||||
import { notionConnectorMeta } from '@/connectors/notion/meta'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
import { joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils'
|
||||
|
||||
const logger = createLogger('NotionConnector')
|
||||
|
||||
const NOTION_API_VERSION = '2022-06-28'
|
||||
const NOTION_BASE_URL = 'https://api.notion.com/v1'
|
||||
|
||||
/**
|
||||
* Extracts the title from a Notion page's properties.
|
||||
*/
|
||||
function extractTitle(properties: Record<string, unknown>): string {
|
||||
for (const value of Object.values(properties)) {
|
||||
const prop = value as Record<string, unknown>
|
||||
if (prop.type === 'title' && Array.isArray(prop.title) && prop.title.length > 0) {
|
||||
return prop.title.map((t: Record<string, unknown>) => (t.plain_text as string) || '').join('')
|
||||
}
|
||||
}
|
||||
return 'Untitled'
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts plain text from a rich_text array.
|
||||
*/
|
||||
function richTextToPlain(richText: Record<string, unknown>[]): string {
|
||||
return richText.map((t) => (t.plain_text as string) || '').join('')
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts plain text content from Notion blocks.
|
||||
*/
|
||||
function blocksToPlainText(blocks: Record<string, unknown>[]): string {
|
||||
return blocks
|
||||
.map((block) => {
|
||||
const type = block.type as string
|
||||
const blockData = block[type] as Record<string, unknown> | undefined
|
||||
if (!blockData) return ''
|
||||
|
||||
if (type === 'code') {
|
||||
const richText = blockData.rich_text as Record<string, unknown>[] | undefined
|
||||
const language = (blockData.language as string) || ''
|
||||
const code = richText ? richTextToPlain(richText) : ''
|
||||
return language ? `\`\`\`${language}\n${code}\n\`\`\`` : `\`\`\`\n${code}\n\`\`\``
|
||||
}
|
||||
|
||||
if (type === 'equation') {
|
||||
const expression = (blockData.expression as string) || ''
|
||||
return expression ? `$$${expression}$$` : ''
|
||||
}
|
||||
|
||||
const richText = blockData.rich_text as Record<string, unknown>[] | undefined
|
||||
if (!richText) return ''
|
||||
|
||||
const text = richTextToPlain(richText)
|
||||
|
||||
switch (type) {
|
||||
case 'heading_1':
|
||||
return `# ${text}`
|
||||
case 'heading_2':
|
||||
return `## ${text}`
|
||||
case 'heading_3':
|
||||
return `### ${text}`
|
||||
case 'bulleted_list_item':
|
||||
return `- ${text}`
|
||||
case 'numbered_list_item':
|
||||
return `1. ${text}`
|
||||
case 'to_do': {
|
||||
const checked = (blockData.checked as boolean) ? '[x]' : '[ ]'
|
||||
return `${checked} ${text}`
|
||||
}
|
||||
case 'quote':
|
||||
return `> ${text}`
|
||||
case 'callout':
|
||||
return text
|
||||
case 'toggle':
|
||||
return text
|
||||
default:
|
||||
return text
|
||||
}
|
||||
})
|
||||
.filter(Boolean)
|
||||
.join('\n\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches all block children for a page, handling pagination.
|
||||
*/
|
||||
async function fetchAllBlocks(
|
||||
accessToken: string,
|
||||
pageId: string
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
const allBlocks: Record<string, unknown>[] = []
|
||||
let cursor: string | undefined
|
||||
let hasMore = true
|
||||
|
||||
while (hasMore) {
|
||||
const params = new URLSearchParams({ page_size: '100' })
|
||||
if (cursor) params.append('start_cursor', cursor)
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
`${NOTION_BASE_URL}/blocks/${pageId}/children?${params.toString()}`,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Notion-Version': NOTION_API_VERSION,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
logger.warn(`Failed to fetch blocks for page ${pageId}`, { status: response.status })
|
||||
break
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
allBlocks.push(...(data.results || []))
|
||||
cursor = data.next_cursor ?? undefined
|
||||
hasMore = data.has_more === true
|
||||
}
|
||||
|
||||
return allBlocks
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts multi_select tags from page properties.
|
||||
*/
|
||||
function extractTags(properties: Record<string, unknown>): string[] {
|
||||
const tags: string[] = []
|
||||
for (const value of Object.values(properties)) {
|
||||
const prop = value as Record<string, unknown>
|
||||
if (prop.type === 'multi_select' && Array.isArray(prop.multi_select)) {
|
||||
for (const item of prop.multi_select) {
|
||||
const name = (item as Record<string, unknown>).name as string
|
||||
if (name) tags.push(name)
|
||||
}
|
||||
}
|
||||
if (prop.type === 'select' && prop.select) {
|
||||
const name = (prop.select as Record<string, unknown>).name as string
|
||||
if (name) tags.push(name)
|
||||
}
|
||||
}
|
||||
return tags
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a Notion page to a lightweight metadata stub (no content fetching).
|
||||
*/
|
||||
function pageToStub(page: Record<string, unknown>): ExternalDocument {
|
||||
const pageId = page.id as string
|
||||
const properties = (page.properties || {}) as Record<string, unknown>
|
||||
const title = extractTitle(properties)
|
||||
const url = page.url as string
|
||||
const lastEditedTime = (page.last_edited_time as string) ?? ''
|
||||
|
||||
const tags = extractTags(properties)
|
||||
|
||||
return {
|
||||
externalId: pageId,
|
||||
title: title || 'Untitled',
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: url,
|
||||
contentHash: `notion:${pageId}:${lastEditedTime}`,
|
||||
metadata: {
|
||||
tags,
|
||||
lastModified: page.last_edited_time as string,
|
||||
createdTime: page.created_time as string,
|
||||
parentType: (page.parent as Record<string, unknown>)?.type,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export const notionConnector: ConnectorConfig = {
|
||||
...notionConnectorMeta,
|
||||
|
||||
listDocuments: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> => {
|
||||
const scope = (sourceConfig.scope as string) || 'workspace'
|
||||
const databaseIds = parseMultiValue(sourceConfig.databaseId)
|
||||
const rootPageId = (sourceConfig.rootPageId as string)?.trim()
|
||||
const maxPages = sourceConfig.maxPages ? Number(sourceConfig.maxPages) : 0
|
||||
|
||||
if (scope === 'database' && databaseIds.length > 0) {
|
||||
return listFromDatabases(accessToken, databaseIds, maxPages, cursor, syncContext)
|
||||
}
|
||||
|
||||
if (scope === 'page' && rootPageId) {
|
||||
return listFromParentPage(accessToken, rootPageId, maxPages, cursor, syncContext)
|
||||
}
|
||||
|
||||
// Default: workspace-wide search
|
||||
const searchQuery = (sourceConfig.searchQuery as string) || ''
|
||||
return listFromWorkspace(accessToken, searchQuery, maxPages, cursor, syncContext)
|
||||
},
|
||||
|
||||
getDocument: async (
|
||||
accessToken: string,
|
||||
_sourceConfig: Record<string, unknown>,
|
||||
externalId: string,
|
||||
_syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocument | null> => {
|
||||
const response = await fetchWithRetry(`${NOTION_BASE_URL}/pages/${externalId}`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Notion-Version': NOTION_API_VERSION,
|
||||
},
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) return null
|
||||
throw new Error(`Failed to get Notion page: ${response.status}`)
|
||||
}
|
||||
|
||||
const page = await response.json()
|
||||
if (page.archived) return null
|
||||
|
||||
try {
|
||||
const blocks = await fetchAllBlocks(accessToken, externalId)
|
||||
const blockContent = blocksToPlainText(blocks)
|
||||
const stub = pageToStub(page)
|
||||
const content = blockContent.trim() || stub.title
|
||||
return { ...stub, content, contentDeferred: false }
|
||||
} catch (error) {
|
||||
logger.warn(`Failed to fetch content for Notion page: ${externalId}`, {
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
},
|
||||
|
||||
validateConfig: async (
|
||||
accessToken: string,
|
||||
sourceConfig: Record<string, unknown>
|
||||
): Promise<{ valid: boolean; error?: string }> => {
|
||||
const scope = (sourceConfig.scope as string) || 'workspace'
|
||||
const databaseIds = parseMultiValue(sourceConfig.databaseId)
|
||||
const rootPageId = (sourceConfig.rootPageId as string)?.trim()
|
||||
const maxPages = sourceConfig.maxPages as string | undefined
|
||||
|
||||
if (maxPages && (Number.isNaN(Number(maxPages)) || Number(maxPages) <= 0)) {
|
||||
return { valid: false, error: 'Max pages must be a positive number' }
|
||||
}
|
||||
|
||||
if (scope === 'database' && databaseIds.length === 0) {
|
||||
return {
|
||||
valid: false,
|
||||
error: 'At least one database is required when scope is "Specific database"',
|
||||
}
|
||||
}
|
||||
|
||||
if (scope === 'page' && !rootPageId) {
|
||||
return { valid: false, error: 'Page ID is required when scope is "Specific page"' }
|
||||
}
|
||||
|
||||
try {
|
||||
// Verify the token works
|
||||
if (scope === 'database' && databaseIds.length > 0) {
|
||||
// Verify every database is accessible
|
||||
for (const databaseId of databaseIds) {
|
||||
const response = await fetchWithRetry(
|
||||
`${NOTION_BASE_URL}/databases/${databaseId}`,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Notion-Version': NOTION_API_VERSION,
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
if (!response.ok) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `Cannot access database ${databaseId}: ${response.status}`,
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (scope === 'page' && rootPageId) {
|
||||
// Verify page is accessible
|
||||
const response = await fetchWithRetry(
|
||||
`${NOTION_BASE_URL}/pages/${rootPageId}`,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Notion-Version': NOTION_API_VERSION,
|
||||
},
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
if (!response.ok) {
|
||||
return { valid: false, error: `Cannot access page: ${response.status}` }
|
||||
}
|
||||
} else {
|
||||
// Workspace scope — just verify token works
|
||||
const response = await fetchWithRetry(
|
||||
`${NOTION_BASE_URL}/search`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Notion-Version': NOTION_API_VERSION,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({ page_size: 1 }),
|
||||
},
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
return { valid: false, error: `Cannot access Notion workspace: ${errorText}` }
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: true }
|
||||
} catch (error) {
|
||||
const message = getErrorMessage(error, 'Failed to validate configuration')
|
||||
return { valid: false, error: message }
|
||||
}
|
||||
},
|
||||
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
const tags = joinTagArray(metadata.tags)
|
||||
if (tags) result.tags = tags
|
||||
|
||||
const lastModified = parseTagDate(metadata.lastModified)
|
||||
if (lastModified) result.lastModified = lastModified
|
||||
|
||||
const created = parseTagDate(metadata.createdTime)
|
||||
if (created) result.created = created
|
||||
|
||||
return result
|
||||
},
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists pages from the entire workspace using the search API.
|
||||
*/
|
||||
async function listFromWorkspace(
|
||||
accessToken: string,
|
||||
searchQuery: string,
|
||||
maxPages: number,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> {
|
||||
const body: Record<string, unknown> = {
|
||||
page_size: 100,
|
||||
filter: { value: 'page', property: 'object' },
|
||||
sort: { direction: 'descending', timestamp: 'last_edited_time' },
|
||||
}
|
||||
|
||||
if (searchQuery.trim()) {
|
||||
body.query = searchQuery.trim()
|
||||
}
|
||||
|
||||
if (cursor) {
|
||||
body.start_cursor = cursor
|
||||
}
|
||||
|
||||
logger.info('Listing Notion pages from workspace', { searchQuery, cursor })
|
||||
|
||||
const response = await fetchWithRetry(`${NOTION_BASE_URL}/search`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Notion-Version': NOTION_API_VERSION,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error('Failed to search Notion', { status: response.status, error: errorText })
|
||||
throw new Error(`Failed to search Notion: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const results = (data.results || []) as Record<string, unknown>[]
|
||||
const pages = results.filter((r) => r.object === 'page' && !(r.archived as boolean))
|
||||
|
||||
const documents = pages.map(pageToStub)
|
||||
|
||||
const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
const hitLimit = maxPages > 0 && totalFetched >= maxPages
|
||||
if (hitLimit && syncContext) syncContext.listingCapped = true
|
||||
|
||||
const nextCursor = hitLimit ? undefined : ((data.next_cursor as string) ?? undefined)
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor,
|
||||
hasMore: hitLimit ? false : data.has_more === true,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists pages from one or more Notion databases.
|
||||
*
|
||||
* Notion's `/v1/databases/{database_id}/query` endpoint is per-database — there
|
||||
* is no batch query endpoint — so multiple databases are walked sequentially.
|
||||
*
|
||||
* Cursor format:
|
||||
* - Single database: the Notion `start_cursor` string directly, or undefined.
|
||||
* - Multiple databases: JSON-encoded `{ databaseIndex, cursor }` where
|
||||
* `databaseIndex` is the position into `databaseIds` currently being drained
|
||||
* and `cursor` is the Notion `start_cursor` for that database (or undefined
|
||||
* when starting a fresh database).
|
||||
*
|
||||
* Page IDs returned by Notion are globally-unique UUIDs, so each page's
|
||||
* `externalId` does not need to be namespaced by database.
|
||||
*/
|
||||
async function listFromDatabases(
|
||||
accessToken: string,
|
||||
databaseIds: string[],
|
||||
maxPages: number,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> {
|
||||
let databaseIndex = 0
|
||||
let startCursor: string | undefined
|
||||
|
||||
if (cursor) {
|
||||
if (databaseIds.length === 1) {
|
||||
// Single-database path: cursor is always a bare Notion `next_cursor` string,
|
||||
// matching the legacy pre-multi-select format. Never JSON-decode here.
|
||||
startCursor = cursor
|
||||
} else {
|
||||
try {
|
||||
const parsed = JSON.parse(cursor) as unknown
|
||||
if (
|
||||
parsed &&
|
||||
typeof parsed === 'object' &&
|
||||
typeof (parsed as { databaseIndex?: unknown }).databaseIndex === 'number'
|
||||
) {
|
||||
const compound = parsed as { databaseIndex: number; cursor?: string }
|
||||
databaseIndex = compound.databaseIndex
|
||||
startCursor = typeof compound.cursor === 'string' ? compound.cursor : undefined
|
||||
} else {
|
||||
// Legacy single-DB cursor carried forward into a now-multi-DB config:
|
||||
// treat it as the start cursor for the first database.
|
||||
startCursor = cursor
|
||||
}
|
||||
} catch {
|
||||
startCursor = cursor
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const documents: ExternalDocument[] = []
|
||||
let nextCursor: string | undefined
|
||||
let hasMore = false
|
||||
|
||||
while (databaseIndex < databaseIds.length) {
|
||||
const databaseId = databaseIds[databaseIndex]
|
||||
const body: Record<string, unknown> = { page_size: 100 }
|
||||
if (startCursor) body.start_cursor = startCursor
|
||||
|
||||
logger.info('Querying Notion database', {
|
||||
databaseId,
|
||||
databaseIndex,
|
||||
databaseCount: databaseIds.length,
|
||||
startCursor,
|
||||
})
|
||||
|
||||
const response = await fetchWithRetry(`${NOTION_BASE_URL}/databases/${databaseId}/query`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Notion-Version': NOTION_API_VERSION,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error('Failed to query Notion database', {
|
||||
databaseId,
|
||||
status: response.status,
|
||||
error: errorText,
|
||||
})
|
||||
throw new Error(`Failed to query Notion database ${databaseId}: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const results = (data.results || []) as Record<string, unknown>[]
|
||||
const pages = results.filter((r) => r.object === 'page' && !(r.archived as boolean))
|
||||
documents.push(...pages.map(pageToStub))
|
||||
|
||||
if (data.has_more === true && typeof data.next_cursor === 'string') {
|
||||
const nextStart = data.next_cursor as string
|
||||
nextCursor =
|
||||
databaseIds.length === 1 ? nextStart : JSON.stringify({ databaseIndex, cursor: nextStart })
|
||||
hasMore = true
|
||||
break
|
||||
}
|
||||
|
||||
databaseIndex++
|
||||
startCursor = undefined
|
||||
|
||||
if (databaseIndex < databaseIds.length) {
|
||||
nextCursor =
|
||||
databaseIds.length === 1 ? undefined : JSON.stringify({ databaseIndex, cursor: undefined })
|
||||
hasMore = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
const hitLimit = maxPages > 0 && totalFetched >= maxPages
|
||||
if (hitLimit) {
|
||||
if (syncContext) syncContext.listingCapped = true
|
||||
hasMore = false
|
||||
nextCursor = undefined
|
||||
}
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor: hasMore ? nextCursor : undefined,
|
||||
hasMore,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists child pages under a specific parent page.
|
||||
*
|
||||
* Uses the blocks children endpoint to find child_page blocks,
|
||||
* then fetches each page's metadata to build lightweight stubs.
|
||||
*/
|
||||
async function listFromParentPage(
|
||||
accessToken: string,
|
||||
rootPageId: string,
|
||||
maxPages: number,
|
||||
cursor?: string,
|
||||
syncContext?: Record<string, unknown>
|
||||
): Promise<ExternalDocumentList> {
|
||||
const params = new URLSearchParams({ page_size: '100' })
|
||||
if (cursor) params.append('start_cursor', cursor)
|
||||
|
||||
logger.info('Listing child pages under root page', { rootPageId, cursor })
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
`${NOTION_BASE_URL}/blocks/${rootPageId}/children?${params.toString()}`,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Notion-Version': NOTION_API_VERSION,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
logger.error('Failed to list child blocks', { status: response.status, error: errorText })
|
||||
throw new Error(`Failed to list child blocks: ${response.status}`)
|
||||
}
|
||||
|
||||
const data = await response.json()
|
||||
const blockResults = (data.results || []) as Record<string, unknown>[]
|
||||
|
||||
// Filter to child_page blocks only (child_database blocks cannot be fetched via the Pages API)
|
||||
const childPageIds = blockResults
|
||||
.filter((b) => b.type === 'child_page')
|
||||
.map((b) => b.id as string)
|
||||
|
||||
// Also include the root page itself on the first call (no cursor)
|
||||
const pageIdsToFetch = !cursor ? [rootPageId, ...childPageIds] : childPageIds
|
||||
|
||||
// Fetch page metadata (not content) in concurrent batches to build stubs
|
||||
const CHILD_PAGE_CONCURRENCY = 5
|
||||
|
||||
const documents: ExternalDocument[] = []
|
||||
for (let i = 0; i < pageIdsToFetch.length; i += CHILD_PAGE_CONCURRENCY) {
|
||||
const cumulativeSoFar = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length
|
||||
if (maxPages > 0 && cumulativeSoFar >= maxPages) break
|
||||
const batch = pageIdsToFetch.slice(i, i + CHILD_PAGE_CONCURRENCY)
|
||||
const results = await Promise.all(
|
||||
batch.map(async (pageId) => {
|
||||
try {
|
||||
const pageResponse = await fetchWithRetry(`${NOTION_BASE_URL}/pages/${pageId}`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
'Notion-Version': NOTION_API_VERSION,
|
||||
},
|
||||
})
|
||||
if (!pageResponse.ok) {
|
||||
logger.warn(`Failed to fetch child page ${pageId}`, { status: pageResponse.status })
|
||||
return null
|
||||
}
|
||||
const page = await pageResponse.json()
|
||||
if (page.archived) return null
|
||||
return pageToStub(page)
|
||||
} catch (error) {
|
||||
logger.warn(`Failed to process child page ${pageId}`, {
|
||||
error: toError(error).message,
|
||||
})
|
||||
return null
|
||||
}
|
||||
})
|
||||
)
|
||||
documents.push(...(results.filter(Boolean) as ExternalDocument[]))
|
||||
}
|
||||
|
||||
const totalFetched = ((syncContext?.totalDocsFetched as number) ?? 0) + documents.length
|
||||
if (syncContext) syncContext.totalDocsFetched = totalFetched
|
||||
const hitLimit = maxPages > 0 && totalFetched >= maxPages
|
||||
if (hitLimit && syncContext) syncContext.listingCapped = true
|
||||
|
||||
const nextCursor = hitLimit ? undefined : ((data.next_cursor as string) ?? undefined)
|
||||
|
||||
return {
|
||||
documents,
|
||||
nextCursor,
|
||||
hasMore: hitLimit ? false : data.has_more === true,
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user