import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { validateServiceNowInstanceUrl } from '@/lib/core/security/input-validation' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_ITEMS, servicenowConnectorMeta } from '@/connectors/servicenow/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { htmlToPlainText, parseTagDate } from '@/connectors/utils' const logger = createLogger('ServiceNowConnector') const PAGE_SIZE = 100 /** * ServiceNow sys_id whitelist: 32-character lowercase hex strings. * * The encoded query language uses `^` as the AND separator and `^OR` as the * OR separator with no escape syntax, so any user-supplied value interpolated * into a `sysparm_query` clause must be validated up front. Path-based * fetches (`/api/now/table/{table}/{sys_id}`) likewise treat the sys_id as a * URL path segment and must be constrained to safe characters. */ const SYS_ID_PATTERN = /^[a-f0-9]{32}$/i const NUMERIC_ID_PATTERN = /^\d+$/ /** * Reject characters that have meaning in a ServiceNow encoded query * (`^` is the operator separator; control chars and quotes can break the * URL). All other Unicode characters — including accented letters used in * categories like "Général" or "Ação" — are allowed. */ const KB_CATEGORY_DISALLOWED = /[\^"'`\u0000-\u001f\u007f]/ const VALID_WORKFLOW_STATES = new Set(['published', 'draft', 'review', 'retired', 'outdated']) interface ServiceNowRecord { sys_id: string sys_updated_on?: string sys_created_on?: string sys_created_by?: string sys_updated_by?: string } interface KBArticle extends ServiceNowRecord { short_description?: string text?: string wiki?: string workflow_state?: string kb_category?: string | { display_value?: string } kb_knowledge_base?: string | { display_value?: string } number?: string author?: string | { display_value?: string } } interface Incident extends ServiceNowRecord { number?: string short_description?: string description?: string state?: string priority?: string category?: string assigned_to?: string | { display_value?: string } opened_by?: string | { display_value?: string } close_notes?: string comments_and_work_notes?: string work_notes?: string resolution_notes?: string } /** * Normalizes and validates the ServiceNow instance URL. * * Prepends https:// if the scheme is missing, strips trailing slashes, then * enforces a ServiceNow-owned domain allowlist to prevent SSRF — the instance * URL is user-controlled and was previously fetched server-side with no * validation. */ function resolveServiceNowInstanceUrl(rawUrl: string): string { let url = (rawUrl ?? '').trim().replace(/\/+$/, '') if (url && !url.startsWith('https://') && !url.startsWith('http://')) { url = `https://${url}` } const validation = validateServiceNowInstanceUrl(url) if (!validation.isValid) { throw new Error(validation.error || 'Invalid instance URL') } return validation.sanitized ?? url } /** * Builds Basic Auth header from username and API key/password. */ function buildAuthHeader(accessToken: string, sourceConfig: Record): string { const username = sourceConfig.username as string const encoded = Buffer.from(`${username}:${accessToken}`).toString('base64') return `Basic ${encoded}` } /** * Calls the ServiceNow Table API. */ async function serviceNowApiGet( instanceUrl: string, tableName: string, authHeader: string, params: Record, retryOptions?: Parameters[2] ): Promise<{ result: Record[]; nextOffset?: number; totalCount?: number }> { const queryParams = new URLSearchParams(params) const url = `${instanceUrl}/api/now/table/${tableName}?${queryParams.toString()}` const response = await fetchWithRetry( url, { method: 'GET', headers: { Authorization: authHeader, Accept: 'application/json', 'Content-Type': 'application/json', }, }, retryOptions ) if (!response.ok) { const errorText = await response.text().catch(() => 'Unknown error') throw new Error(`ServiceNow API error (${response.status}): ${errorText}`) } const data = (await response.json()) as { result: Record[] } const totalCountHeader = response.headers.get('X-Total-Count') const totalCount = totalCountHeader ? Number(totalCountHeader) : undefined const offset = Number(params.sysparm_offset || '0') const limit = Number(params.sysparm_limit || String(PAGE_SIZE)) const resultCount = data.result?.length ?? 0 const nextOffset = resultCount >= limit ? offset + limit : undefined return { result: data.result || [], nextOffset, totalCount, } } /** * Fetches a single ServiceNow record by sys_id via the path-based Table API * endpoint (`GET /api/now/table/{tableName}/{sys_id}`), which returns a * `{ result: }` object rather than the array shape returned by the * list endpoint. Returns `null` when the record is not found (404). */ async function serviceNowApiGetById( instanceUrl: string, tableName: string, sysId: string, authHeader: string, params: Record, retryOptions?: Parameters[2] ): Promise | null> { const queryParams = new URLSearchParams(params) const queryString = queryParams.toString() const url = `${instanceUrl}/api/now/table/${tableName}/${sysId}${queryString ? `?${queryString}` : ''}` const response = await fetchWithRetry( url, { method: 'GET', headers: { Authorization: authHeader, Accept: 'application/json', 'Content-Type': 'application/json', }, }, retryOptions ) if (response.status === 404) { return null } if (!response.ok) { const errorText = await response.text().catch(() => 'Unknown error') throw new Error(`ServiceNow API error (${response.status}): ${errorText}`) } const data = (await response.json()) as { result?: Record } return data.result ?? null } function isServiceNowRecord(record: unknown): record is ServiceNowRecord & Record { return ( typeof record === 'object' && record !== null && !Array.isArray(record) && typeof (record as Record).sys_id === 'string' && ((record as Record).sys_id as string).length > 0 ) } /** * Extracts a display value from a field that may be a string or a reference object. * When sysparm_display_value=true, fields are plain strings. * When sysparm_display_value=all, fields are objects with display_value/value. * This helper normalises both shapes. */ function displayValue(field: unknown): string | undefined { if (typeof field === 'string') return field || undefined if (field && typeof field === 'object') { const obj = field as Record if ('display_value' in obj && typeof obj.display_value === 'string') { return obj.display_value || undefined } if ('value' in obj && typeof obj.value === 'string') { return obj.value || undefined } } return undefined } /** * Extracts the raw value from a field that may be a string or an object * returned by sysparm_display_value=all. Prefers `value` over `display_value` * so that coded values (e.g. state "1") are preserved for mapping functions. */ function rawValue(field: unknown): string | undefined { if (typeof field === 'string') return field || undefined if (field && typeof field === 'object') { const obj = field as Record if ('value' in obj && typeof obj.value === 'string') { return obj.value || undefined } if ('display_value' in obj && typeof obj.display_value === 'string') { return obj.display_value || undefined } } return undefined } /** * Maps ServiceNow state codes to human-readable labels for incidents. */ function incidentStateLabel(state: string | undefined): string { const stateMap: Record = { '1': 'New', '2': 'In Progress', '3': 'On Hold', '6': 'Resolved', '7': 'Closed', '8': 'Canceled', } return state ? stateMap[state] || state : 'Unknown' } /** * Maps ServiceNow priority codes to human-readable labels. */ function priorityLabel(priority: string | undefined): string { const priorityMap: Record = { '1': 'Critical', '2': 'High', '3': 'Moderate', '4': 'Low', '5': 'Planning', } return priority ? priorityMap[priority] || priority : 'Unknown' } /** * Converts a KB article record to an ExternalDocument. */ function kbArticleToDocument(article: KBArticle, instanceUrl: string): ExternalDocument { const title = rawValue(article.short_description) || rawValue(article.number) || article.sys_id /** * Wiki-template KB articles populate `wiki` with the body and leave * `text` empty; HTML-template articles do the opposite. Falling back * to `wiki` keeps both layouts indexable. */ const articleText = rawValue(article.text) || rawValue(article.wiki) || '' const content = htmlToPlainText(articleText) const sysId = rawValue(article.sys_id) || article.sys_id const updatedOn = rawValue(article.sys_updated_on) || '' const contentHash = `servicenow:${sysId}:${updatedOn}` const sourceUrl = `${instanceUrl}/kb_view.do?sys_kb_id=${sysId}` return { externalId: sysId, title, content, mimeType: 'text/plain', sourceUrl, contentHash, metadata: { type: 'kb_article', number: rawValue(article.number), workflowState: rawValue(article.workflow_state), category: displayValue(article.kb_category), knowledgeBase: displayValue(article.kb_knowledge_base), author: displayValue(article.author) || rawValue(article.sys_created_by), lastUpdated: rawValue(article.sys_updated_on), createdOn: rawValue(article.sys_created_on), }, } } /** * Converts an incident record to an ExternalDocument. */ function incidentToDocument(incident: Incident, instanceUrl: string): ExternalDocument { const number = rawValue(incident.number) const shortDesc = rawValue(incident.short_description) const title = number ? `${number}: ${shortDesc || 'Untitled'}` : shortDesc || incident.sys_id const parts: string[] = [] if (shortDesc) { parts.push(`Summary: ${shortDesc}`) } const description = rawValue(incident.description) if (description) { parts.push(`Description: ${htmlToPlainText(description)}`) } const state = rawValue(incident.state) const priority = rawValue(incident.priority) parts.push(`State: ${incidentStateLabel(state)}`) parts.push(`Priority: ${priorityLabel(priority)}`) const category = rawValue(incident.category) if (category) { parts.push(`Category: ${category}`) } if (displayValue(incident.assigned_to)) { parts.push(`Assigned To: ${displayValue(incident.assigned_to)}`) } if (displayValue(incident.opened_by)) { parts.push(`Opened By: ${displayValue(incident.opened_by)}`) } const resolutionNotes = rawValue(incident.resolution_notes) if (resolutionNotes) { parts.push(`Resolution Notes: ${htmlToPlainText(resolutionNotes)}`) } const closeNotes = rawValue(incident.close_notes) if (closeNotes) { parts.push(`Close Notes: ${htmlToPlainText(closeNotes)}`) } const content = parts.join('\n') const sysId = rawValue(incident.sys_id) || incident.sys_id const updatedOn = rawValue(incident.sys_updated_on) || '' const contentHash = `servicenow:${sysId}:${updatedOn}` const sourceUrl = `${instanceUrl}/incident.do?sys_id=${sysId}` return { externalId: sysId, title, content, mimeType: 'text/plain', sourceUrl, contentHash, metadata: { type: 'incident', number, state: incidentStateLabel(state), priority: priorityLabel(priority), category, assignedTo: displayValue(incident.assigned_to), openedBy: displayValue(incident.opened_by), author: displayValue(incident.opened_by) || rawValue(incident.sys_created_by), lastUpdated: rawValue(incident.sys_updated_on), createdOn: rawValue(incident.sys_created_on), }, } } /** * Builds the sysparm_query filter string for KB articles. */ function buildKBQuery(sourceConfig: Record): string { const parts: string[] = [] const workflowState = sourceConfig.workflowState as string | undefined if (workflowState && workflowState !== 'all') { if (VALID_WORKFLOW_STATES.has(workflowState)) { parts.push(`workflow_state=${workflowState}`) } else { logger.warn('Skipping workflowState filter: value is not in the allowed set', { workflowState, }) } } const kbCategory = sourceConfig.kbCategory as string | undefined const trimmedCategory = kbCategory?.trim() if (trimmedCategory) { if (!KB_CATEGORY_DISALLOWED.test(trimmedCategory)) { parts.push(`kb_category.label=${trimmedCategory}`) } else { logger.warn('Skipping kbCategory filter: value contains disallowed characters', { kbCategory: trimmedCategory, }) } } parts.push('ORDERBYDESCsys_updated_on') return parts.join('^') } /** * Builds the sysparm_query filter string for incidents. */ function buildIncidentQuery(sourceConfig: Record): string { const parts: string[] = [] const incidentState = sourceConfig.incidentState as string | undefined if (incidentState && incidentState !== 'all') { if (NUMERIC_ID_PATTERN.test(incidentState)) { parts.push(`state=${incidentState}`) } else { logger.warn('Skipping incidentState filter: value is not a numeric ID', { incidentState }) } } const incidentPriority = sourceConfig.incidentPriority as string | undefined if (incidentPriority && incidentPriority !== 'all') { if (NUMERIC_ID_PATTERN.test(incidentPriority)) { parts.push(`priority=${incidentPriority}`) } else { logger.warn('Skipping incidentPriority filter: value is not a numeric ID', { incidentPriority, }) } } parts.push('ORDERBYDESCsys_updated_on') return parts.join('^') } export const servicenowConnector: ConnectorConfig = { ...servicenowConnectorMeta, listDocuments: async ( accessToken: string, sourceConfig: Record, cursor?: string, _syncContext?: Record ): Promise => { const instanceUrl = resolveServiceNowInstanceUrl(sourceConfig.instanceUrl as string) const contentType = (sourceConfig.contentType as string) || 'kb_knowledge' const maxItems = sourceConfig.maxItems ? Number(sourceConfig.maxItems) : DEFAULT_MAX_ITEMS const authHeader = buildAuthHeader(accessToken, sourceConfig) const offset = cursor ? Number(cursor) : 0 const remaining = maxItems - offset if (remaining <= 0) { return { documents: [], hasMore: false } } const limit = Math.min(PAGE_SIZE, remaining) const isKB = contentType === 'kb_knowledge' const tableName = isKB ? 'kb_knowledge' : 'incident' const query = isKB ? buildKBQuery(sourceConfig) : buildIncidentQuery(sourceConfig) const fields = isKB ? 'sys_id,short_description,text,wiki,workflow_state,kb_category,kb_knowledge_base,number,author,sys_created_by,sys_updated_by,sys_updated_on,sys_created_on' : 'sys_id,number,short_description,description,state,priority,category,assigned_to,opened_by,close_notes,resolution_notes,sys_created_by,sys_updated_by,sys_updated_on,sys_created_on' const params: Record = { sysparm_limit: String(limit), sysparm_offset: String(offset), sysparm_query: query, sysparm_fields: fields, sysparm_display_value: 'all', } logger.info('Fetching ServiceNow records', { table: tableName, offset, limit, query, }) const { result, nextOffset } = await serviceNowApiGet( instanceUrl, tableName, authHeader, params ) const documents: ExternalDocument[] = [] for (const record of result) { if (!isServiceNowRecord(record)) { logger.warn('Skipping ServiceNow record without sys_id', { table: tableName }) continue } const doc = isKB ? kbArticleToDocument(record, instanceUrl) : incidentToDocument(record, instanceUrl) if (doc.content.trim()) { documents.push(doc) } } const hasMore = nextOffset !== undefined && nextOffset < maxItems const nextCursor = hasMore ? String(nextOffset) : undefined logger.info('Fetched ServiceNow documents', { count: documents.length, hasMore, nextCursor, }) return { documents, nextCursor, hasMore, } }, getDocument: async ( accessToken: string, sourceConfig: Record, externalId: string ): Promise => { const contentType = (sourceConfig.contentType as string) || 'kb_knowledge' const authHeader = buildAuthHeader(accessToken, sourceConfig) const isKB = contentType === 'kb_knowledge' const tableName = isKB ? 'kb_knowledge' : 'incident' if (!SYS_ID_PATTERN.test(externalId)) { logger.warn('Rejecting ServiceNow getDocument with invalid sys_id', { externalId, table: tableName, }) return null } const fields = isKB ? 'sys_id,short_description,text,wiki,workflow_state,kb_category,kb_knowledge_base,number,author,sys_created_by,sys_updated_by,sys_updated_on,sys_created_on' : 'sys_id,number,short_description,description,state,priority,category,assigned_to,opened_by,close_notes,resolution_notes,sys_created_by,sys_updated_by,sys_updated_on,sys_created_on' const instanceUrl = resolveServiceNowInstanceUrl(sourceConfig.instanceUrl as string) try { const record = await serviceNowApiGetById(instanceUrl, tableName, externalId, authHeader, { sysparm_fields: fields, sysparm_display_value: 'all', }) if (!record || !isServiceNowRecord(record)) { return null } const doc = isKB ? kbArticleToDocument(record, instanceUrl) : incidentToDocument(record, instanceUrl) return doc.content.trim() ? doc : null } catch (error) { logger.warn('Failed to get ServiceNow document', { externalId, table: tableName, error: toError(error).message, }) return null } }, validateConfig: async ( accessToken: string, sourceConfig: Record ): Promise<{ valid: boolean; error?: string }> => { const instanceUrl = sourceConfig.instanceUrl as string | undefined const username = sourceConfig.username as string | undefined const contentType = sourceConfig.contentType as string | undefined const maxItems = sourceConfig.maxItems as string | undefined if (!instanceUrl?.trim()) { return { valid: false, error: 'Instance URL is required' } } if (!username?.trim()) { return { valid: false, error: 'Username is required' } } if (!contentType) { return { valid: false, error: 'Content type is required' } } if (maxItems && (Number.isNaN(Number(maxItems)) || Number(maxItems) <= 0)) { return { valid: false, error: 'Max items must be a positive number' } } let normalizedUrl: string try { normalizedUrl = resolveServiceNowInstanceUrl(instanceUrl) } catch (error) { return { valid: false, error: toError(error).message } } const authHeader = buildAuthHeader(accessToken, sourceConfig) const tableName = contentType === 'kb_knowledge' ? 'kb_knowledge' : 'incident' try { await serviceNowApiGet( normalizedUrl, tableName, authHeader, { sysparm_limit: '1', sysparm_offset: '0', }, VALIDATE_RETRY_OPTIONS ) return { valid: true } } catch (error) { return { valid: false, error: toError(error).message || 'Failed to connect to ServiceNow' } } }, mapTags: (metadata: Record): Record => { const result: Record = {} if (typeof metadata.type === 'string') { result.type = metadata.type === 'kb_article' ? 'KB Article' : 'Incident' } const state = metadata.state ?? metadata.workflowState if (typeof state === 'string') { result.state = state } if (typeof metadata.priority === 'string') { result.priority = metadata.priority } if (typeof metadata.category === 'string') { result.category = metadata.category } const author = metadata.author if (typeof author === 'string') { result.author = author } const lastUpdated = parseTagDate(metadata.lastUpdated) if (lastUpdated) { result.lastUpdated = lastUpdated } return result }, }