import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { htmlToPlainText, joinTagArray, parseTagDate } from '@/connectors/utils' import { DEFAULT_MAX_POSTS, wordpressConnectorMeta } from '@/connectors/wordpress/meta' const logger = createLogger('WordPressConnector') const WP_API_BASE = 'https://public-api.wordpress.com/rest/v1.1/sites' /** * Strips protocol prefix and trailing slashes from a site URL so the * WordPress.com API receives a bare domain (e.g. "mysite.wordpress.com"). */ function normalizeSiteUrl(raw: string): string { return raw.replace(/^https?:\/\//, '').replace(/\/+$/, '') } const POSTS_PER_PAGE = 20 interface WordPressPost { ID: number title: string content: string URL: string modified: string type: string author: { name: string } categories: Record tags: Record } interface WordPressPostsResponse { found: number posts: WordPressPost[] } interface ListCursor { offset: number } /** * Extracts category names from a WordPress categories object. */ function extractCategoryNames(categories: Record): string[] { return Object.values(categories).map((c) => c.name) } /** * Extracts tag names from a WordPress tags object. */ function extractTagNames(tags: Record): string[] { return Object.values(tags).map((t) => t.name) } /** * Converts a WordPress post to an ExternalDocument. */ function postToDocument(post: WordPressPost): ExternalDocument { const plainText = htmlToPlainText(post.content) const fullContent = `# ${post.title}\n\n${plainText}` const contentHash = `wordpress:${post.ID}:${post.modified || ''}` const categories = extractCategoryNames(post.categories) const tags = extractTagNames(post.tags) return { externalId: String(post.ID), title: post.title || 'Untitled', content: fullContent, mimeType: 'text/plain', sourceUrl: post.URL, contentHash, metadata: { author: post.author?.name, lastModified: post.modified, postType: post.type, categories, tags, }, } } /** * Resolves the postType config value to the WordPress API type parameter. */ function resolvePostType(postType?: string): string { switch (postType) { case 'Posts': return 'post' case 'Pages': return 'page' default: return 'any' } } export const wordpressConnector: ConnectorConfig = { ...wordpressConnectorMeta, listDocuments: async ( accessToken: string, sourceConfig: Record, cursor?: string, syncContext?: Record ): Promise => { const rawSiteUrl = (sourceConfig.siteUrl as string)?.trim() if (!rawSiteUrl) { throw new Error('Site URL is required') } const siteUrl = normalizeSiteUrl(rawSiteUrl) const maxPosts = sourceConfig.maxPosts ? Number(sourceConfig.maxPosts) : DEFAULT_MAX_POSTS const type = resolvePostType(sourceConfig.postType as string | undefined) const parsed: ListCursor = cursor ? JSON.parse(cursor) : { offset: 0 } const totalDocsFetched = (syncContext?.totalDocsFetched as number) ?? 0 const remaining = maxPosts > 0 ? maxPosts - totalDocsFetched : POSTS_PER_PAGE if (remaining <= 0) { return { documents: [], hasMore: false } } const pageSize = Math.min(POSTS_PER_PAGE, remaining) const url = `${WP_API_BASE}/${encodeURIComponent(siteUrl)}/posts?number=${pageSize}&offset=${parsed.offset}&type=${type}` logger.info('Fetching WordPress posts', { siteUrl, offset: parsed.offset, type, pageSize }) const response = await fetchWithRetry(url, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json', }, }) if (!response.ok) { const errorText = await response.text().catch(() => '') throw new Error(`WordPress API error: ${response.status} ${errorText}`) } const data = (await response.json()) as WordPressPostsResponse const posts = data.posts || [] const documents = posts.map(postToDocument) const totalFetched = totalDocsFetched + documents.length if (syncContext) syncContext.totalDocsFetched = totalFetched const hitLimit = maxPosts > 0 && totalFetched >= maxPosts const newOffset = parsed.offset + posts.length const hasMore = !hitLimit && newOffset < data.found return { documents, hasMore, nextCursor: hasMore ? JSON.stringify({ offset: newOffset }) : undefined, } }, getDocument: async ( accessToken: string, sourceConfig: Record, externalId: string ): Promise => { const rawSiteUrl = (sourceConfig.siteUrl as string)?.trim() if (!rawSiteUrl) { throw new Error('Site URL is required') } const siteUrl = normalizeSiteUrl(rawSiteUrl) const url = `${WP_API_BASE}/${encodeURIComponent(siteUrl)}/posts/${externalId}` try { const response = await fetchWithRetry(url, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json', }, }) if (!response.ok) { if (response.status === 404) return null throw new Error(`WordPress API error: ${response.status}`) } const post = (await response.json()) as WordPressPost return postToDocument(post) } catch (error) { logger.warn('Failed to get WordPress document', { externalId, error: toError(error).message, }) return null } }, validateConfig: async ( accessToken: string, sourceConfig: Record ): Promise<{ valid: boolean; error?: string }> => { const rawSiteUrl = (sourceConfig.siteUrl as string)?.trim() const maxPosts = sourceConfig.maxPosts as string | undefined if (!rawSiteUrl) { return { valid: false, error: 'Site URL is required' } } const siteUrl = normalizeSiteUrl(rawSiteUrl) if (maxPosts && (Number.isNaN(Number(maxPosts)) || Number(maxPosts) <= 0)) { return { valid: false, error: 'Max posts must be a positive number' } } try { const url = `${WP_API_BASE}/${encodeURIComponent(siteUrl)}` const response = await fetchWithRetry( url, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json', }, }, VALIDATE_RETRY_OPTIONS ) if (!response.ok) { if (response.status === 404) { return { valid: false, error: `Site not found: ${siteUrl}` } } return { valid: false, error: `WordPress API error: ${response.status}` } } return { valid: true } } catch (error) { const message = getErrorMessage(error, 'Failed to validate configuration') return { valid: false, error: message } } }, mapTags: (metadata: Record): Record => { const result: Record = {} if (typeof metadata.author === 'string') { result.author = metadata.author } const lastModified = parseTagDate(metadata.lastModified) if (lastModified) { result.lastModified = lastModified } if (typeof metadata.postType === 'string') { result.postType = metadata.postType } const categories = joinTagArray(metadata.categories) if (categories) { result.categories = categories } const tags = joinTagArray(metadata.tags) if (tags) { result.tags = tags } return result }, }