wehub-resource-sync
428 lines
13 KiB
TypeScript
428 lines
13 KiB
TypeScript
import { Client } from '@botpress/client'
|
|
import { getBigCommerceClient, BigCommerceClient } from '../client'
|
|
import { getProductImageUrl, BigCommerceProductImage, stripHtmlTags } from '../index'
|
|
import { PRODUCT_TABLE_SCHEMA, PRODUCTS_TABLE_NAME as PRODUCT_TABLE } from '../schemas/products'
|
|
import * as bp from '.botpress'
|
|
|
|
type BigCommerceProduct = {
|
|
id: number
|
|
name: string
|
|
sku?: string
|
|
price?: number
|
|
sale_price?: number
|
|
retail_price?: number
|
|
weight?: number
|
|
type?: string
|
|
inventory_level?: number
|
|
inventory_tracking?: string
|
|
brand_id?: number
|
|
categories?: number[]
|
|
availability?: string
|
|
condition?: string
|
|
is_visible?: boolean
|
|
sort_order?: number
|
|
description?: string
|
|
images?: Array<BigCommerceProductImage>
|
|
custom_url?: { url: string }
|
|
total_sold?: number
|
|
}
|
|
|
|
type SyncConfig = {
|
|
storeHash: string
|
|
accessToken: string
|
|
tableName: string
|
|
batchSize?: number
|
|
productsPerPage?: number
|
|
}
|
|
|
|
const DEFAULT_BATCH_SIZE = 50
|
|
const DEFAULT_PRODUCTS_PER_PAGE = 250
|
|
|
|
function transformProductToTableRow(
|
|
product: BigCommerceProduct,
|
|
categoryById: Record<number, string>,
|
|
brandById: Record<number, string>
|
|
) {
|
|
const categoryNames =
|
|
product.categories?.map((categoryId: number) => categoryById[categoryId] || categoryId.toString()) || []
|
|
|
|
const categories = categoryNames.join(',')
|
|
const brandName = product.brand_id ? brandById[product.brand_id] || product.brand_id.toString() : ''
|
|
const imageUrl = product.images && product.images.length > 0 ? getProductImageUrl(product.images) : ''
|
|
|
|
return {
|
|
product_id: product.id,
|
|
name: product.name,
|
|
sku: product.sku,
|
|
price: product.price,
|
|
sale_price: product.sale_price,
|
|
retail_price: product.retail_price,
|
|
total_sold: product.total_sold || 0,
|
|
weight: product.weight,
|
|
type: product.type,
|
|
inventory_level: product.inventory_level,
|
|
inventory_tracking: product.inventory_tracking,
|
|
brand_name: brandName,
|
|
categories,
|
|
availability: product.availability,
|
|
condition: product.condition,
|
|
is_visible: product.is_visible,
|
|
sort_order: product.sort_order,
|
|
description: stripHtmlTags(product.description)?.substring(0, 1000) || '',
|
|
image_url: imageUrl,
|
|
url: product.custom_url?.url || '',
|
|
}
|
|
}
|
|
|
|
async function saveProductsToTable(
|
|
client: Client,
|
|
tableName: string,
|
|
products: BigCommerceProduct[],
|
|
categoryById: Record<number, string>,
|
|
brandById: Record<number, string>,
|
|
batchSize: number = DEFAULT_BATCH_SIZE
|
|
) {
|
|
try {
|
|
const tableRows = products.map((product) => transformProductToTableRow(product, categoryById, brandById))
|
|
|
|
const totalRows = tableRows.length
|
|
let processedRows = 0
|
|
|
|
while (processedRows < totalRows) {
|
|
const batch = tableRows.slice(processedRows, processedRows + batchSize)
|
|
|
|
await client.createTableRows({
|
|
table: tableName,
|
|
rows: batch,
|
|
})
|
|
|
|
processedRows += batch.length
|
|
}
|
|
|
|
return totalRows
|
|
} catch (error) {
|
|
console.error('Failed to save products to table:', error)
|
|
throw error
|
|
}
|
|
}
|
|
|
|
async function loadCategoriesAndBrands(bigCommerceClient: BigCommerceClient) {
|
|
const categoryById: Record<number, string> = {}
|
|
const brandById: Record<number, string> = {}
|
|
|
|
try {
|
|
const categoriesResponse = await bigCommerceClient.getCategories()
|
|
if (categoriesResponse && categoriesResponse.data) {
|
|
categoriesResponse.data.forEach((category: { id: number; name: string }) => {
|
|
categoryById[category.id] = category.name
|
|
})
|
|
}
|
|
|
|
const brandsResponse = await bigCommerceClient.getBrands()
|
|
if (brandsResponse && brandsResponse.data) {
|
|
brandsResponse.data.forEach((brand: { id: number; name: string }) => {
|
|
brandById[brand.id] = brand.name
|
|
})
|
|
}
|
|
} catch (error) {
|
|
console.error('Error loading categories/brands:', error)
|
|
}
|
|
|
|
return { categoryById, brandById }
|
|
}
|
|
|
|
async function processPages({
|
|
bigCommerceClient,
|
|
botpressClient,
|
|
tableName,
|
|
categoryById,
|
|
brandById,
|
|
startPage,
|
|
totalPages,
|
|
batchSize,
|
|
productsPerPage,
|
|
logger,
|
|
}: {
|
|
bigCommerceClient: BigCommerceClient
|
|
botpressClient: Client
|
|
tableName: string
|
|
categoryById: Record<number, string>
|
|
brandById: Record<number, string>
|
|
startPage: number
|
|
totalPages: number
|
|
batchSize: number
|
|
productsPerPage: number
|
|
logger: bp.Logger
|
|
}): Promise<{ processedItems: number; lastProcessedPage: number; errors: string[] }> {
|
|
const startTime = Date.now()
|
|
let processedItems = 0
|
|
let lastProcessedPage = startPage - 1
|
|
const errors: string[] = []
|
|
const totalPagesToProcess = totalPages - startPage + 1
|
|
|
|
logger.forBot().info(`Processing pages from ${startPage} to ${totalPages} (${totalPagesToProcess} pages total)`)
|
|
|
|
for (let page = startPage; page <= totalPages; page++) {
|
|
const pagesProcessed = page - startPage
|
|
const progress = Math.round((pagesProcessed / totalPagesToProcess) * 100)
|
|
|
|
try {
|
|
logger.forBot().info(`Processing page ${page}/${totalPages} (${progress}% complete)...`)
|
|
|
|
const pageResult = await bigCommerceClient.getProducts({ page, limit: productsPerPage })
|
|
|
|
if (!pageResult.data || pageResult.data.length === 0) {
|
|
logger.forBot().warn(`No products found on page ${page}`)
|
|
continue
|
|
}
|
|
|
|
// Save products to table
|
|
await saveProductsToTable(botpressClient, tableName, pageResult.data, categoryById, brandById, batchSize)
|
|
|
|
processedItems += pageResult.data.length
|
|
lastProcessedPage = page
|
|
|
|
logger.forBot().info(`Completed page ${page}: ${pageResult.data.length} products`)
|
|
|
|
await new Promise((resolve) => setTimeout(resolve, 50))
|
|
} catch (error) {
|
|
const errorMessage = `Error processing page ${page}: ${error instanceof Error ? error.message : 'Unknown error'}`
|
|
logger.forBot().error(errorMessage)
|
|
errors.push(errorMessage)
|
|
continue
|
|
}
|
|
}
|
|
|
|
const totalElapsed = Date.now() - startTime
|
|
const pagesCompleted = lastProcessedPage - startPage + 1
|
|
const finalProgress = Math.round((pagesCompleted / totalPagesToProcess) * 100)
|
|
|
|
logger
|
|
.forBot()
|
|
.info(
|
|
`Processing completed: ${pagesCompleted}/${totalPagesToProcess} pages (${finalProgress}%) ` +
|
|
`in ${Math.round(totalElapsed / 1000)}s. Processed ${processedItems} products.`
|
|
)
|
|
|
|
return { processedItems, lastProcessedPage, errors }
|
|
}
|
|
|
|
function initializeClients(ctx: { storeHash: string; accessToken: string }, client: bp.Client) {
|
|
const bigCommerceClient = getBigCommerceClient(ctx)
|
|
const getVanillaClient = (client: bp.Client): Client => client._inner
|
|
const botpressVanillaClient = getVanillaClient(client)
|
|
|
|
return { bigCommerceClient, botpressVanillaClient }
|
|
}
|
|
|
|
export async function executeBackgroundSync({
|
|
ctx,
|
|
input,
|
|
logger,
|
|
client,
|
|
}: {
|
|
ctx: { configuration: { storeHash: string; accessToken: string } }
|
|
input: {
|
|
startPage: number
|
|
totalPages: number
|
|
tableName: string
|
|
batchSize: number
|
|
productsPerPage: number
|
|
categoryById: Record<number, string>
|
|
brandById: Record<number, string>
|
|
}
|
|
logger: bp.Logger
|
|
client: bp.Client
|
|
}) {
|
|
const { startPage, totalPages, tableName, batchSize, productsPerPage, categoryById, brandById } = input
|
|
const { bigCommerceClient, botpressVanillaClient } = initializeClients(ctx.configuration, client)
|
|
|
|
logger.forBot().info('Starting background processing...')
|
|
|
|
const result = await processPages({
|
|
bigCommerceClient,
|
|
botpressClient: botpressVanillaClient,
|
|
tableName,
|
|
categoryById,
|
|
brandById,
|
|
startPage,
|
|
totalPages,
|
|
batchSize,
|
|
productsPerPage,
|
|
logger,
|
|
})
|
|
|
|
return {
|
|
success: true,
|
|
processedItems: result.processedItems,
|
|
lastProcessedPage: result.lastProcessedPage,
|
|
totalPages,
|
|
errors: result.errors,
|
|
finalProgress: Math.round(((result.lastProcessedPage - startPage + 1) / (totalPages - startPage + 1)) * 100),
|
|
totalElapsed: Math.round((Date.now() - Date.now()) / 1000),
|
|
}
|
|
}
|
|
|
|
const syncProducts: bp.IntegrationProps['actions']['syncProducts'] = async (props) => {
|
|
const { client, logger } = props
|
|
const ctx = props.ctx.configuration
|
|
|
|
const { bigCommerceClient, botpressVanillaClient } = initializeClients(ctx, client)
|
|
|
|
const config: SyncConfig = {
|
|
storeHash: ctx.storeHash,
|
|
accessToken: ctx.accessToken,
|
|
tableName: PRODUCT_TABLE,
|
|
batchSize: DEFAULT_BATCH_SIZE,
|
|
productsPerPage: DEFAULT_PRODUCTS_PER_PAGE,
|
|
}
|
|
|
|
try {
|
|
logger.forBot().info('Starting BigCommerce product sync')
|
|
|
|
await botpressVanillaClient.getOrCreateTable({
|
|
table: config.tableName,
|
|
schema: PRODUCT_TABLE_SCHEMA,
|
|
})
|
|
|
|
try {
|
|
logger.forBot().info('Clearing existing products table...')
|
|
await botpressVanillaClient.deleteTable({
|
|
table: config.tableName,
|
|
})
|
|
|
|
await botpressVanillaClient.getOrCreateTable({
|
|
table: config.tableName,
|
|
schema: PRODUCT_TABLE_SCHEMA,
|
|
})
|
|
} catch (error) {
|
|
logger.forBot().warn('Error clearing products table:', error)
|
|
}
|
|
|
|
// Step 1: Load categories and brands
|
|
logger.forBot().info('Loading categories and brands...')
|
|
const { categoryById, brandById } = await loadCategoriesAndBrands(bigCommerceClient)
|
|
logger
|
|
.forBot()
|
|
.info(`Loaded ${Object.keys(categoryById).length} categories and ${Object.keys(brandById).length} brands`)
|
|
|
|
// Step 2: Get first page synchronously
|
|
logger.forBot().info('Fetching first page synchronously...')
|
|
|
|
const firstPageResponse = await bigCommerceClient.getProducts({ page: 1, limit: config.productsPerPage! })
|
|
const totalPages: number = firstPageResponse.meta?.pagination?.total_pages ?? 1
|
|
const firstPageCount = firstPageResponse.data?.length ?? 0
|
|
|
|
if (firstPageCount === 0) {
|
|
logger.forBot().warn('No products found in BigCommerce store')
|
|
return {
|
|
success: true,
|
|
message: 'No products found in BigCommerce store',
|
|
productsCount: 0,
|
|
firstPageProcessed: 0,
|
|
totalPages: 0,
|
|
backgroundProcessing: false,
|
|
}
|
|
}
|
|
|
|
// Step 3: Process first page using unified function
|
|
logger.forBot().info(`Processing first page (${firstPageCount} products)...`)
|
|
await processPages({
|
|
bigCommerceClient,
|
|
botpressClient: botpressVanillaClient,
|
|
tableName: config.tableName,
|
|
categoryById,
|
|
brandById,
|
|
startPage: 1,
|
|
totalPages: 1,
|
|
batchSize: DEFAULT_BATCH_SIZE,
|
|
productsPerPage: DEFAULT_PRODUCTS_PER_PAGE,
|
|
logger,
|
|
})
|
|
|
|
logger.forBot().info(`First page processed. Total pages: ${totalPages}, First page items: ${firstPageCount}`)
|
|
|
|
// Step 4: If there are more pages, send webhook to trigger background processing
|
|
if (totalPages > 1) {
|
|
logger.forBot().info('First page processed. Sending webhook to trigger background processing...')
|
|
|
|
const webhookUrl = `https://webhook.botpress.cloud/${props.ctx.webhookId}`
|
|
|
|
const payload = {
|
|
event: 'background-sync-triggered',
|
|
data: {
|
|
startPage: 2,
|
|
totalPages,
|
|
tableName: config.tableName,
|
|
batchSize: DEFAULT_BATCH_SIZE,
|
|
productsPerPage: DEFAULT_PRODUCTS_PER_PAGE,
|
|
storeHash: config.storeHash,
|
|
accessToken: config.accessToken,
|
|
categoryById,
|
|
brandById,
|
|
},
|
|
}
|
|
|
|
try {
|
|
fetch(webhookUrl, {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
},
|
|
body: JSON.stringify(payload),
|
|
}).catch((error) => {
|
|
logger.forBot().error('Failed to send background processing webhook:', error)
|
|
})
|
|
|
|
logger.forBot().info('Background processing webhook sent successfully')
|
|
|
|
return {
|
|
success: true,
|
|
firstPageProcessed: firstPageCount,
|
|
totalPages,
|
|
productsCount: firstPageCount,
|
|
backgroundProcessing: true,
|
|
lastProcessedPage: 1,
|
|
backgroundErrors: [],
|
|
message: `First page processed synchronously. Background processing webhook sent for remaining ${totalPages - 1} pages.`,
|
|
}
|
|
} catch (error) {
|
|
logger.forBot().error('Failed to send background processing webhook:', error)
|
|
return {
|
|
success: false,
|
|
firstPageProcessed: firstPageCount,
|
|
totalPages,
|
|
productsCount: firstPageCount,
|
|
backgroundProcessing: false,
|
|
lastProcessedPage: 1,
|
|
backgroundErrors: [
|
|
`Failed to send background processing webhook: ${error instanceof Error ? error.message : String(error)}`,
|
|
],
|
|
message: 'First page processed but failed to send background processing webhook.',
|
|
}
|
|
}
|
|
} else {
|
|
return {
|
|
success: true,
|
|
firstPageProcessed: firstPageCount,
|
|
totalPages: 1,
|
|
productsCount: firstPageCount,
|
|
backgroundProcessing: false,
|
|
message: 'All products synced synchronously (single page).',
|
|
}
|
|
}
|
|
} catch (error) {
|
|
logger.forBot().error('Sync failed:', error)
|
|
return {
|
|
success: false,
|
|
message: `Error syncing BigCommerce products: ${error instanceof Error ? error.message : String(error)}`,
|
|
productsCount: 0,
|
|
firstPageProcessed: 0,
|
|
totalPages: 0,
|
|
backgroundProcessing: false,
|
|
}
|
|
}
|
|
}
|
|
|
|
export default syncProducts
|