simstudioai--sim/apps/sim/connectors/servicenow/servicenow.ts

import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { validateServiceNowInstanceUrl } from '@/lib/core/security/input-validation'
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
import { DEFAULT_MAX_ITEMS, servicenowConnectorMeta } from '@/connectors/servicenow/meta'
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
import { htmlToPlainText, parseTagDate } from '@/connectors/utils'

const logger = createLogger('ServiceNowConnector')

const PAGE_SIZE = 100

/**
 * ServiceNow sys_id whitelist: 32-character lowercase hex strings.
 *
 * The encoded query language uses `^` as the AND separator and `^OR` as the
 * OR separator with no escape syntax, so any user-supplied value interpolated
 * into a `sysparm_query` clause must be validated up front. Path-based
 * fetches (`/api/now/table/{table}/{sys_id}`) likewise treat the sys_id as a
 * URL path segment and must be constrained to safe characters.
 */
const SYS_ID_PATTERN = /^[a-f0-9]{32}$/i
const NUMERIC_ID_PATTERN = /^\d+$/
/**
 * Reject characters that have meaning in a ServiceNow encoded query
 * (`^` is the operator separator; control chars and quotes can break the
 * URL). All other Unicode characters — including accented letters used in
 * categories like "Général" or "Ação" — are allowed.
 */
const KB_CATEGORY_DISALLOWED = /[\^"'`\u0000-\u001f\u007f]/
const VALID_WORKFLOW_STATES = new Set(['published', 'draft', 'review', 'retired', 'outdated'])

interface ServiceNowRecord {
  sys_id: string
  sys_updated_on?: string
  sys_created_on?: string
  sys_created_by?: string
  sys_updated_by?: string
}

interface KBArticle extends ServiceNowRecord {
  short_description?: string
  text?: string
  wiki?: string
  workflow_state?: string
  kb_category?: string | { display_value?: string }
  kb_knowledge_base?: string | { display_value?: string }
  number?: string
  author?: string | { display_value?: string }
}

interface Incident extends ServiceNowRecord {
  number?: string
  short_description?: string
  description?: string
  state?: string
  priority?: string
  category?: string
  assigned_to?: string | { display_value?: string }
  opened_by?: string | { display_value?: string }
  close_notes?: string
  comments_and_work_notes?: string
  work_notes?: string
  resolution_notes?: string
}

/**
 * Normalizes and validates the ServiceNow instance URL.
 *
 * Prepends https:// if the scheme is missing, strips trailing slashes, then
 * enforces a ServiceNow-owned domain allowlist to prevent SSRF — the instance
 * URL is user-controlled and was previously fetched server-side with no
 * validation.
 */
function resolveServiceNowInstanceUrl(rawUrl: string): string {
  let url = (rawUrl ?? '').trim().replace(/\/+$/, '')
  if (url && !url.startsWith('https://') && !url.startsWith('http://')) {
    url = `https://${url}`
  }
  const validation = validateServiceNowInstanceUrl(url)
  if (!validation.isValid) {
    throw new Error(validation.error || 'Invalid instance URL')
  }
  return validation.sanitized ?? url
}

/**
 * Builds Basic Auth header from username and API key/password.
 */
function buildAuthHeader(accessToken: string, sourceConfig: Record<string, unknown>): string {
  const username = sourceConfig.username as string
  const encoded = Buffer.from(`${username}:${accessToken}`).toString('base64')
  return `Basic ${encoded}`
}

/**
 * Calls the ServiceNow Table API.
 */
async function serviceNowApiGet(
  instanceUrl: string,
  tableName: string,
  authHeader: string,
  params: Record<string, string>,
  retryOptions?: Parameters<typeof fetchWithRetry>[2]
): Promise<{ result: Record<string, unknown>[]; nextOffset?: number; totalCount?: number }> {
  const queryParams = new URLSearchParams(params)
  const url = `${instanceUrl}/api/now/table/${tableName}?${queryParams.toString()}`

  const response = await fetchWithRetry(
    url,
    {
      method: 'GET',
      headers: {
        Authorization: authHeader,
        Accept: 'application/json',
        'Content-Type': 'application/json',
      },
    },
    retryOptions
  )

  if (!response.ok) {
    const errorText = await response.text().catch(() => 'Unknown error')
    throw new Error(`ServiceNow API error (${response.status}): ${errorText}`)
  }

  const data = (await response.json()) as { result: Record<string, unknown>[] }

  const totalCountHeader = response.headers.get('X-Total-Count')
  const totalCount = totalCountHeader ? Number(totalCountHeader) : undefined

  const offset = Number(params.sysparm_offset || '0')
  const limit = Number(params.sysparm_limit || String(PAGE_SIZE))
  const resultCount = data.result?.length ?? 0

  const nextOffset = resultCount >= limit ? offset + limit : undefined

  return {
    result: data.result || [],
    nextOffset,
    totalCount,
  }
}

/**
 * Fetches a single ServiceNow record by sys_id via the path-based Table API
 * endpoint (`GET /api/now/table/{tableName}/{sys_id}`), which returns a
 * `{ result: <record> }` object rather than the array shape returned by the
 * list endpoint. Returns `null` when the record is not found (404).
 */
async function serviceNowApiGetById(
  instanceUrl: string,
  tableName: string,
  sysId: string,
  authHeader: string,
  params: Record<string, string>,
  retryOptions?: Parameters<typeof fetchWithRetry>[2]
): Promise<Record<string, unknown> | null> {
  const queryParams = new URLSearchParams(params)
  const queryString = queryParams.toString()
  const url = `${instanceUrl}/api/now/table/${tableName}/${sysId}${queryString ? `?${queryString}` : ''}`

  const response = await fetchWithRetry(
    url,
    {
      method: 'GET',
      headers: {
        Authorization: authHeader,
        Accept: 'application/json',
        'Content-Type': 'application/json',
      },
    },
    retryOptions
  )

  if (response.status === 404) {
    return null
  }

  if (!response.ok) {
    const errorText = await response.text().catch(() => 'Unknown error')
    throw new Error(`ServiceNow API error (${response.status}): ${errorText}`)
  }

  const data = (await response.json()) as { result?: Record<string, unknown> }
  return data.result ?? null
}

function isServiceNowRecord(record: unknown): record is ServiceNowRecord & Record<string, unknown> {
  return (
    typeof record === 'object' &&
    record !== null &&
    !Array.isArray(record) &&
    typeof (record as Record<string, unknown>).sys_id === 'string' &&
    ((record as Record<string, unknown>).sys_id as string).length > 0
  )
}

/**
 * Extracts a display value from a field that may be a string or a reference object.
 * When sysparm_display_value=true, fields are plain strings.
 * When sysparm_display_value=all, fields are objects with display_value/value.
 * This helper normalises both shapes.
 */
function displayValue(field: unknown): string | undefined {
  if (typeof field === 'string') return field || undefined
  if (field && typeof field === 'object') {
    const obj = field as Record<string, unknown>
    if ('display_value' in obj && typeof obj.display_value === 'string') {
      return obj.display_value || undefined
    }
    if ('value' in obj && typeof obj.value === 'string') {
      return obj.value || undefined
    }
  }
  return undefined
}

/**
 * Extracts the raw value from a field that may be a string or an object
 * returned by sysparm_display_value=all. Prefers `value` over `display_value`
 * so that coded values (e.g. state "1") are preserved for mapping functions.
 */
function rawValue(field: unknown): string | undefined {
  if (typeof field === 'string') return field || undefined
  if (field && typeof field === 'object') {
    const obj = field as Record<string, unknown>
    if ('value' in obj && typeof obj.value === 'string') {
      return obj.value || undefined
    }
    if ('display_value' in obj && typeof obj.display_value === 'string') {
      return obj.display_value || undefined
    }
  }
  return undefined
}

/**
 * Maps ServiceNow state codes to human-readable labels for incidents.
 */
function incidentStateLabel(state: string | undefined): string {
  const stateMap: Record<string, string> = {
    '1': 'New',
    '2': 'In Progress',
    '3': 'On Hold',
    '6': 'Resolved',
    '7': 'Closed',
    '8': 'Canceled',
  }
  return state ? stateMap[state] || state : 'Unknown'
}

/**
 * Maps ServiceNow priority codes to human-readable labels.
 */
function priorityLabel(priority: string | undefined): string {
  const priorityMap: Record<string, string> = {
    '1': 'Critical',
    '2': 'High',
    '3': 'Moderate',
    '4': 'Low',
    '5': 'Planning',
  }
  return priority ? priorityMap[priority] || priority : 'Unknown'
}

/**
 * Converts a KB article record to an ExternalDocument.
 */
function kbArticleToDocument(article: KBArticle, instanceUrl: string): ExternalDocument {
  const title = rawValue(article.short_description) || rawValue(article.number) || article.sys_id
  /**
   * Wiki-template KB articles populate `wiki` with the body and leave
   * `text` empty; HTML-template articles do the opposite. Falling back
   * to `wiki` keeps both layouts indexable.
   */
  const articleText = rawValue(article.text) || rawValue(article.wiki) || ''
  const content = htmlToPlainText(articleText)
  const sysId = rawValue(article.sys_id) || article.sys_id
  const updatedOn = rawValue(article.sys_updated_on) || ''
  const contentHash = `servicenow:${sysId}:${updatedOn}`
  const sourceUrl = `${instanceUrl}/kb_view.do?sys_kb_id=${sysId}`

  return {
    externalId: sysId,
    title,
    content,
    mimeType: 'text/plain',
    sourceUrl,
    contentHash,
    metadata: {
      type: 'kb_article',
      number: rawValue(article.number),
      workflowState: rawValue(article.workflow_state),
      category: displayValue(article.kb_category),
      knowledgeBase: displayValue(article.kb_knowledge_base),
      author: displayValue(article.author) || rawValue(article.sys_created_by),
      lastUpdated: rawValue(article.sys_updated_on),
      createdOn: rawValue(article.sys_created_on),
    },
  }
}

/**
 * Converts an incident record to an ExternalDocument.
 */
function incidentToDocument(incident: Incident, instanceUrl: string): ExternalDocument {
  const number = rawValue(incident.number)
  const shortDesc = rawValue(incident.short_description)
  const title = number ? `${number}: ${shortDesc || 'Untitled'}` : shortDesc || incident.sys_id

  const parts: string[] = []
  if (shortDesc) {
    parts.push(`Summary: ${shortDesc}`)
  }
  const description = rawValue(incident.description)
  if (description) {
    parts.push(`Description: ${htmlToPlainText(description)}`)
  }
  const state = rawValue(incident.state)
  const priority = rawValue(incident.priority)
  parts.push(`State: ${incidentStateLabel(state)}`)
  parts.push(`Priority: ${priorityLabel(priority)}`)
  const category = rawValue(incident.category)
  if (category) {
    parts.push(`Category: ${category}`)
  }
  if (displayValue(incident.assigned_to)) {
    parts.push(`Assigned To: ${displayValue(incident.assigned_to)}`)
  }
  if (displayValue(incident.opened_by)) {
    parts.push(`Opened By: ${displayValue(incident.opened_by)}`)
  }
  const resolutionNotes = rawValue(incident.resolution_notes)
  if (resolutionNotes) {
    parts.push(`Resolution Notes: ${htmlToPlainText(resolutionNotes)}`)
  }
  const closeNotes = rawValue(incident.close_notes)
  if (closeNotes) {
    parts.push(`Close Notes: ${htmlToPlainText(closeNotes)}`)
  }

  const content = parts.join('\n')
  const sysId = rawValue(incident.sys_id) || incident.sys_id
  const updatedOn = rawValue(incident.sys_updated_on) || ''
  const contentHash = `servicenow:${sysId}:${updatedOn}`
  const sourceUrl = `${instanceUrl}/incident.do?sys_id=${sysId}`

  return {
    externalId: sysId,
    title,
    content,
    mimeType: 'text/plain',
    sourceUrl,
    contentHash,
    metadata: {
      type: 'incident',
      number,
      state: incidentStateLabel(state),
      priority: priorityLabel(priority),
      category,
      assignedTo: displayValue(incident.assigned_to),
      openedBy: displayValue(incident.opened_by),
      author: displayValue(incident.opened_by) || rawValue(incident.sys_created_by),
      lastUpdated: rawValue(incident.sys_updated_on),
      createdOn: rawValue(incident.sys_created_on),
    },
  }
}

/**
 * Builds the sysparm_query filter string for KB articles.
 */
function buildKBQuery(sourceConfig: Record<string, unknown>): string {
  const parts: string[] = []

  const workflowState = sourceConfig.workflowState as string | undefined
  if (workflowState && workflowState !== 'all') {
    if (VALID_WORKFLOW_STATES.has(workflowState)) {
      parts.push(`workflow_state=${workflowState}`)
    } else {
      logger.warn('Skipping workflowState filter: value is not in the allowed set', {
        workflowState,
      })
    }
  }

  const kbCategory = sourceConfig.kbCategory as string | undefined
  const trimmedCategory = kbCategory?.trim()
  if (trimmedCategory) {
    if (!KB_CATEGORY_DISALLOWED.test(trimmedCategory)) {
      parts.push(`kb_category.label=${trimmedCategory}`)
    } else {
      logger.warn('Skipping kbCategory filter: value contains disallowed characters', {
        kbCategory: trimmedCategory,
      })
    }
  }

  parts.push('ORDERBYDESCsys_updated_on')
  return parts.join('^')
}

/**
 * Builds the sysparm_query filter string for incidents.
 */
function buildIncidentQuery(sourceConfig: Record<string, unknown>): string {
  const parts: string[] = []

  const incidentState = sourceConfig.incidentState as string | undefined
  if (incidentState && incidentState !== 'all') {
    if (NUMERIC_ID_PATTERN.test(incidentState)) {
      parts.push(`state=${incidentState}`)
    } else {
      logger.warn('Skipping incidentState filter: value is not a numeric ID', { incidentState })
    }
  }

  const incidentPriority = sourceConfig.incidentPriority as string | undefined
  if (incidentPriority && incidentPriority !== 'all') {
    if (NUMERIC_ID_PATTERN.test(incidentPriority)) {
      parts.push(`priority=${incidentPriority}`)
    } else {
      logger.warn('Skipping incidentPriority filter: value is not a numeric ID', {
        incidentPriority,
      })
    }
  }

  parts.push('ORDERBYDESCsys_updated_on')
  return parts.join('^')
}

export const servicenowConnector: ConnectorConfig = {
  ...servicenowConnectorMeta,

  listDocuments: async (
    accessToken: string,
    sourceConfig: Record<string, unknown>,
    cursor?: string,
    _syncContext?: Record<string, unknown>
  ): Promise<ExternalDocumentList> => {
    const instanceUrl = resolveServiceNowInstanceUrl(sourceConfig.instanceUrl as string)
    const contentType = (sourceConfig.contentType as string) || 'kb_knowledge'
    const maxItems = sourceConfig.maxItems ? Number(sourceConfig.maxItems) : DEFAULT_MAX_ITEMS
    const authHeader = buildAuthHeader(accessToken, sourceConfig)

    const offset = cursor ? Number(cursor) : 0
    const remaining = maxItems - offset
    if (remaining <= 0) {
      return { documents: [], hasMore: false }
    }

    const limit = Math.min(PAGE_SIZE, remaining)
    const isKB = contentType === 'kb_knowledge'
    const tableName = isKB ? 'kb_knowledge' : 'incident'
    const query = isKB ? buildKBQuery(sourceConfig) : buildIncidentQuery(sourceConfig)

    const fields = isKB
      ? 'sys_id,short_description,text,wiki,workflow_state,kb_category,kb_knowledge_base,number,author,sys_created_by,sys_updated_by,sys_updated_on,sys_created_on'
      : 'sys_id,number,short_description,description,state,priority,category,assigned_to,opened_by,close_notes,resolution_notes,sys_created_by,sys_updated_by,sys_updated_on,sys_created_on'

    const params: Record<string, string> = {
      sysparm_limit: String(limit),
      sysparm_offset: String(offset),
      sysparm_query: query,
      sysparm_fields: fields,
      sysparm_display_value: 'all',
    }

    logger.info('Fetching ServiceNow records', {
      table: tableName,
      offset,
      limit,
      query,
    })

    const { result, nextOffset } = await serviceNowApiGet(
      instanceUrl,
      tableName,
      authHeader,
      params
    )

    const documents: ExternalDocument[] = []
    for (const record of result) {
      if (!isServiceNowRecord(record)) {
        logger.warn('Skipping ServiceNow record without sys_id', { table: tableName })
        continue
      }

      const doc = isKB
        ? kbArticleToDocument(record, instanceUrl)
        : incidentToDocument(record, instanceUrl)

      if (doc.content.trim()) {
        documents.push(doc)
      }
    }

    const hasMore = nextOffset !== undefined && nextOffset < maxItems
    const nextCursor = hasMore ? String(nextOffset) : undefined

    logger.info('Fetched ServiceNow documents', {
      count: documents.length,
      hasMore,
      nextCursor,
    })

    return {
      documents,
      nextCursor,
      hasMore,
    }
  },

  getDocument: async (
    accessToken: string,
    sourceConfig: Record<string, unknown>,
    externalId: string
  ): Promise<ExternalDocument | null> => {
    const contentType = (sourceConfig.contentType as string) || 'kb_knowledge'
    const authHeader = buildAuthHeader(accessToken, sourceConfig)
    const isKB = contentType === 'kb_knowledge'
    const tableName = isKB ? 'kb_knowledge' : 'incident'

    if (!SYS_ID_PATTERN.test(externalId)) {
      logger.warn('Rejecting ServiceNow getDocument with invalid sys_id', {
        externalId,
        table: tableName,
      })
      return null
    }

    const fields = isKB
      ? 'sys_id,short_description,text,wiki,workflow_state,kb_category,kb_knowledge_base,number,author,sys_created_by,sys_updated_by,sys_updated_on,sys_created_on'
      : 'sys_id,number,short_description,description,state,priority,category,assigned_to,opened_by,close_notes,resolution_notes,sys_created_by,sys_updated_by,sys_updated_on,sys_created_on'

    const instanceUrl = resolveServiceNowInstanceUrl(sourceConfig.instanceUrl as string)

    try {
      const record = await serviceNowApiGetById(instanceUrl, tableName, externalId, authHeader, {
        sysparm_fields: fields,
        sysparm_display_value: 'all',
      })

      if (!record || !isServiceNowRecord(record)) {
        return null
      }

      const doc = isKB
        ? kbArticleToDocument(record, instanceUrl)
        : incidentToDocument(record, instanceUrl)

      return doc.content.trim() ? doc : null
    } catch (error) {
      logger.warn('Failed to get ServiceNow document', {
        externalId,
        table: tableName,
        error: toError(error).message,
      })
      return null
    }
  },

  validateConfig: async (
    accessToken: string,
    sourceConfig: Record<string, unknown>
  ): Promise<{ valid: boolean; error?: string }> => {
    const instanceUrl = sourceConfig.instanceUrl as string | undefined
    const username = sourceConfig.username as string | undefined
    const contentType = sourceConfig.contentType as string | undefined
    const maxItems = sourceConfig.maxItems as string | undefined

    if (!instanceUrl?.trim()) {
      return { valid: false, error: 'Instance URL is required' }
    }

    if (!username?.trim()) {
      return { valid: false, error: 'Username is required' }
    }

    if (!contentType) {
      return { valid: false, error: 'Content type is required' }
    }

    if (maxItems && (Number.isNaN(Number(maxItems)) || Number(maxItems) <= 0)) {
      return { valid: false, error: 'Max items must be a positive number' }
    }

    let normalizedUrl: string
    try {
      normalizedUrl = resolveServiceNowInstanceUrl(instanceUrl)
    } catch (error) {
      return { valid: false, error: toError(error).message }
    }

    const authHeader = buildAuthHeader(accessToken, sourceConfig)
    const tableName = contentType === 'kb_knowledge' ? 'kb_knowledge' : 'incident'

    try {
      await serviceNowApiGet(
        normalizedUrl,
        tableName,
        authHeader,
        {
          sysparm_limit: '1',
          sysparm_offset: '0',
        },
        VALIDATE_RETRY_OPTIONS
      )
      return { valid: true }
    } catch (error) {
      return { valid: false, error: toError(error).message || 'Failed to connect to ServiceNow' }
    }
  },

  mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
    const result: Record<string, unknown> = {}

    if (typeof metadata.type === 'string') {
      result.type = metadata.type === 'kb_article' ? 'KB Article' : 'Incident'
    }

    const state = metadata.state ?? metadata.workflowState
    if (typeof state === 'string') {
      result.state = state
    }

    if (typeof metadata.priority === 'string') {
      result.priority = metadata.priority
    }

    if (typeof metadata.category === 'string') {
      result.category = metadata.category
    }

    const author = metadata.author
    if (typeof author === 'string') {
      result.author = author
    }

    const lastUpdated = parseTagDate(metadata.lastUpdated)
    if (lastUpdated) {
      result.lastUpdated = lastUpdated
    }

    return result
  },
}