chore: import upstream snapshot with attribution
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:20:55 +08:00
commit d25d482dc2
13754 changed files with 4996608 additions and 0 deletions
@@ -0,0 +1,265 @@
/**
* @vitest-environment node
*/
import { dbChainMock, dbChainMockFns, resetDbChainMock } from '@sim/testing'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type { DbOrTx } from '@/lib/db/types'
import { listSurfacedBackgroundWork } from '@/ee/workspace-forking/lib/background-work/store'
const executor = dbChainMock.db as unknown as DbOrTx
/** Shape produced by the drizzle-orm mock's `sql` tagged template. */
interface MockSqlFragment {
strings: readonly string[]
values: unknown[]
}
interface MockCondition {
type: string
conditions?: MockCondition[]
left?: unknown
right?: unknown
column?: unknown
values?: unknown[]
}
/** Resolves the first `.where(...)` (the children lookup) to the given fork ids. */
function mockChildrenLookup(childIds: string[]) {
dbChainMockFns.where.mockImplementationOnce(
() => Promise.resolve(childIds.map((id) => ({ id }))) as never
)
}
/** Builds an opaque cursor the way the store encodes one (base64 JSON). */
function encodeCursor(data: { updatedAt: string; id: string }): string {
return Buffer.from(JSON.stringify(data)).toString('base64')
}
function decodeCursor(cursor: string): { updatedAt: string; id: string } {
return JSON.parse(Buffer.from(cursor, 'base64').toString())
}
describe('listSurfacedBackgroundWork', () => {
beforeEach(() => {
vi.clearAllMocks()
resetDbChainMock()
})
it('returns the surfaced rows ordered by recency with the id tiebreaker', async () => {
mockChildrenLookup([])
const rows = [
{ id: 'job-1', updatedAt: new Date('2026-07-01T10:00:00.000Z') },
{ id: 'job-2', updatedAt: new Date('2026-07-01T09:00:00.000Z') },
]
dbChainMockFns.limit.mockResolvedValueOnce(rows as never)
const result = await listSurfacedBackgroundWork(executor, 'ws-1')
expect(result.rows).toEqual(rows)
expect(dbChainMockFns.orderBy).toHaveBeenCalledWith(
{ type: 'desc', column: 'updatedAt' },
{ type: 'desc', column: 'id' }
)
// Over-fetches one row past the default page size to detect another page.
expect(dbChainMockFns.limit).toHaveBeenCalledWith(51)
})
it('returns a null cursor when the page is not full', async () => {
mockChildrenLookup([])
dbChainMockFns.limit.mockResolvedValueOnce([
{ id: 'job-1', updatedAt: new Date('2026-07-01T10:00:00.000Z') },
] as never)
const result = await listSurfacedBackgroundWork(executor, 'ws-1', { limit: 2 })
expect(result.rows).toHaveLength(1)
expect(result.nextCursor).toBeNull()
})
it('returns a null cursor for an empty page', async () => {
mockChildrenLookup([])
dbChainMockFns.limit.mockResolvedValueOnce([] as never)
const result = await listSurfacedBackgroundWork(executor, 'ws-1')
expect(result).toEqual({ rows: [], nextCursor: null })
})
it('trims the over-fetched row and encodes the next cursor from the last returned row', async () => {
mockChildrenLookup([])
const rows = [
{ id: 'job-1', updatedAt: new Date('2026-07-01T10:00:00.000Z') },
{ id: 'job-2', updatedAt: new Date('2026-07-01T09:00:00.000Z') },
{ id: 'job-3', updatedAt: new Date('2026-07-01T08:00:00.000Z') },
]
dbChainMockFns.limit.mockResolvedValueOnce(rows as never)
const result = await listSurfacedBackgroundWork(executor, 'ws-1', { limit: 2 })
expect(dbChainMockFns.limit).toHaveBeenCalledWith(3)
expect(result.rows).toEqual(rows.slice(0, 2))
expect(result.nextCursor).not.toBeNull()
expect(decodeCursor(result.nextCursor as string)).toEqual({
updatedAt: '2026-07-01T09:00:00.000Z',
id: 'job-2',
})
})
it('applies the cursor as a keyset condition with the id tiebreaker', async () => {
mockChildrenLookup([])
const cursor = encodeCursor({ updatedAt: '2026-07-01T09:00:00.000Z', id: 'job-2' })
await listSurfacedBackgroundWork(executor, 'ws-1', { cursor })
const rowsWhere = dbChainMockFns.where.mock.calls[1][0] as MockCondition
expect(rowsWhere.type).toBe('and')
expect(rowsWhere.conditions).toHaveLength(3)
const cursorDate = new Date('2026-07-01T09:00:00.000Z')
const keyset = (rowsWhere.conditions as MockCondition[])[2]
expect(keyset).toEqual({
type: 'or',
conditions: [
{ type: 'lt', left: 'updatedAt', right: cursorDate },
{
type: 'and',
conditions: [
{ type: 'eq', left: 'updatedAt', right: cursorDate },
{ type: 'lt', left: 'id', right: 'job-2' },
],
},
],
})
})
it('pages through rows with identical updatedAt via the id tiebreaker', async () => {
// Two rows share one timestamp; page 1 ends on the higher id. The next
// cursor must carry that id so the second page matches the remaining row
// through the eq(updatedAt) + lt(id) arm instead of skipping it.
const sharedAt = new Date('2026-07-01T09:00:00.000Z')
mockChildrenLookup([])
dbChainMockFns.limit.mockResolvedValueOnce([
{ id: 'job-b', updatedAt: sharedAt },
{ id: 'job-a', updatedAt: sharedAt },
] as never)
const firstPage = await listSurfacedBackgroundWork(executor, 'ws-1', { limit: 1 })
expect(firstPage.rows).toEqual([{ id: 'job-b', updatedAt: sharedAt }])
expect(decodeCursor(firstPage.nextCursor as string)).toEqual({
updatedAt: sharedAt.toISOString(),
id: 'job-b',
})
mockChildrenLookup([])
dbChainMockFns.limit.mockResolvedValueOnce([{ id: 'job-a', updatedAt: sharedAt }] as never)
const secondPage = await listSurfacedBackgroundWork(executor, 'ws-1', {
cursor: firstPage.nextCursor as string,
limit: 1,
})
const rowsWhere = dbChainMockFns.where.mock.calls[3][0] as MockCondition
const keyset = (rowsWhere.conditions as MockCondition[])[2]
expect(keyset.conditions?.[1]).toEqual({
type: 'and',
conditions: [
{ type: 'eq', left: 'updatedAt', right: sharedAt },
{ type: 'lt', left: 'id', right: 'job-b' },
],
})
expect(secondPage.rows).toEqual([{ id: 'job-a', updatedAt: sharedAt }])
expect(secondPage.nextCursor).toBeNull()
})
it('ignores an undecodable cursor and serves the first page', async () => {
mockChildrenLookup([])
await listSurfacedBackgroundWork(executor, 'ws-1', { cursor: 'not-base64-json' })
const rowsWhere = dbChainMockFns.where.mock.calls[1][0] as MockCondition
expect(rowsWhere.conditions).toHaveLength(2)
})
it('clamps the requested limit to the server-side cap', async () => {
mockChildrenLookup([])
await listSurfacedBackgroundWork(executor, 'ws-1', { limit: 5000 })
expect(dbChainMockFns.limit).toHaveBeenCalledWith(101)
})
it('looks up live forks of the workspace for the child-keyed clause', async () => {
mockChildrenLookup([])
await listSurfacedBackgroundWork(executor, 'ws-1')
const childrenWhere = dbChainMockFns.where.mock.calls[0][0] as MockCondition
expect(childrenWhere).toEqual({
type: 'and',
conditions: [
{ type: 'eq', left: 'forkedFromWorkspaceId', right: 'ws-1' },
{ type: 'isNull', column: 'archivedAt' },
],
})
})
it('matches rows keyed to the workspace, to it as fork child, and to it as edge partner', async () => {
mockChildrenLookup([])
await listSurfacedBackgroundWork(executor, 'ws-1')
const rowsWhere = dbChainMockFns.where.mock.calls[1][0] as MockCondition
expect(rowsWhere.type).toBe('and')
expect(rowsWhere.conditions).toHaveLength(2)
const [involves, statuses] = rowsWhere.conditions as [MockCondition, MockCondition]
expect(involves.type).toBe('or')
const orConditions = involves.conditions as unknown as [
MockCondition,
MockSqlFragment,
MockSqlFragment,
]
expect(orConditions).toHaveLength(3)
expect(orConditions[0]).toEqual({ type: 'eq', left: 'workspaceId', right: 'ws-1' })
const childIdClause = orConditions[1]
expect(childIdClause.strings.join('?')).toContain("->> 'childWorkspaceId' =")
expect(childIdClause.values).toEqual(['metadata', 'ws-1'])
const otherIdClause = orConditions[2]
expect(otherIdClause.strings.join('?')).toContain("->> 'otherWorkspaceId' =")
expect(otherIdClause.values).toEqual(['metadata', 'ws-1'])
expect(statuses).toEqual({
type: 'inArray',
column: 'status',
values: ['pending', 'processing', 'completed', 'completed_with_warnings', 'failed'],
})
})
it('also matches sync/rollback rows keyed to the forks of the workspace (pre-otherWorkspaceId history)', async () => {
mockChildrenLookup(['fork-1', 'fork-2'])
await listSurfacedBackgroundWork(executor, 'ws-1')
const rowsWhere = dbChainMockFns.where.mock.calls[1][0] as MockCondition
const involves = (rowsWhere.conditions as MockCondition[])[0]
expect(involves.conditions).toHaveLength(4)
const childKeyedClause = (involves.conditions as MockCondition[])[3]
expect(childKeyedClause).toEqual({
type: 'and',
conditions: [
{ type: 'inArray', column: 'workspaceId', values: ['fork-1', 'fork-2'] },
{ type: 'inArray', column: 'kind', values: ['fork_sync', 'fork_rollback'] },
],
})
})
it('omits the child-keyed clause when the workspace has no forks', async () => {
mockChildrenLookup([])
await listSurfacedBackgroundWork(executor, 'ws-1')
const rowsWhere = dbChainMockFns.where.mock.calls[1][0] as MockCondition
const involves = (rowsWhere.conditions as MockCondition[])[0]
expect(involves.conditions).toHaveLength(3)
})
})
@@ -0,0 +1,348 @@
import { backgroundWorkStatus, workspace } from '@sim/db/schema'
import { createLogger } from '@sim/logger'
import { generateId } from '@sim/utils/id'
import { and, desc, eq, inArray, isNull, lt, lte, or, type SQL, sql } from 'drizzle-orm'
import type { DbOrTx } from '@/lib/db/types'
const logger = createLogger('ForkBackgroundWork')
export type BackgroundWorkKind =
| 'deployment_side_effects'
| 'fork_content_copy'
| 'fork_sync'
| 'fork_rollback'
export type BackgroundWorkStatusValue =
| 'pending'
| 'processing'
| 'completed'
| 'completed_with_warnings'
| 'failed'
/** Statuses surfaced as recent jobs - active, completed, or a recent issue. */
const SURFACED_STATUSES: BackgroundWorkStatusValue[] = [
'pending',
'processing',
'completed',
'completed_with_warnings',
'failed',
]
/** Default page size for the workspace's Activity tab (mirrors the audit log's). */
const BACKGROUND_WORK_PAGE_SIZE = 50
/** Server-side cap on the requested page size (mirrors the audit log's). */
const BACKGROUND_WORK_PAGE_SIZE_MAX = 100
/**
* An active (pending/processing) row older than this is treated as abandoned: the
* worker crashed or restarted before {@link finishBackgroundWork}, and a hard crash
* (Trigger CRASHED / process death) fires no in-task hook. The outbox-processor cron
* sweeps these via {@link reapStaleBackgroundWork}, marking them `failed` so the UI
* surfaces the failure instead of spinning forever.
*/
const STALE_ACTIVE_MS = 30 * 60 * 1000
/** Terminal rows older than this are pruned by the cron so the audit trail stays bounded. */
const RETENTION_MS = 30 * 24 * 60 * 60 * 1000
export interface BackgroundWorkRow {
id: string
workspaceId: string
workflowId: string | null
kind: BackgroundWorkKind
status: BackgroundWorkStatusValue
message: string | null
error: string | null
metadata: unknown
startedAt: Date
completedAt: Date | null
updatedAt: Date
}
/**
* Begin tracking a unit of background work, returning its id. Any prior row for the
* same scope (workspace + workflow + kind) is removed first so exactly one status per
* scope is live - a fresh run supersedes a stale completed/failed one.
*/
export async function startBackgroundWork(
executor: DbOrTx,
params: {
workspaceId: string
workflowId?: string | null
kind: BackgroundWorkKind
message?: string
metadata?: unknown
/**
* Replace any prior row for the same scope (workspace + workflow + kind) so exactly
* one status per scope is live (default). Pass `false` to keep an append-only audit
* trail - e.g. fork jobs, where each fork is a distinct historical entry.
*/
supersede?: boolean
}
): Promise<string> {
const { workspaceId, workflowId = null, kind, message, metadata, supersede = true } = params
if (supersede) {
await executor
.delete(backgroundWorkStatus)
.where(
and(
eq(backgroundWorkStatus.workspaceId, workspaceId),
eq(backgroundWorkStatus.kind, kind),
workflowId == null
? isNull(backgroundWorkStatus.workflowId)
: eq(backgroundWorkStatus.workflowId, workflowId)
)
)
}
const id = generateId()
const now = new Date()
await executor.insert(backgroundWorkStatus).values({
id,
workspaceId,
workflowId,
kind,
status: 'processing',
message: message ?? null,
metadata: metadata ?? null,
startedAt: now,
updatedAt: now,
})
return id
}
/**
* Record a synchronous operation directly as a terminal audit entry (no `processing`
* phase). Append-only - used by sync/rollback, which complete in-request, so they show
* up in the same workspace audit log as fork jobs.
*/
export async function recordBackgroundWork(
executor: DbOrTx,
params: {
workspaceId: string
kind: BackgroundWorkKind
status: Extract<BackgroundWorkStatusValue, 'completed' | 'completed_with_warnings' | 'failed'>
message?: string
error?: string
metadata?: unknown
}
): Promise<void> {
const now = new Date()
await executor.insert(backgroundWorkStatus).values({
id: generateId(),
workspaceId: params.workspaceId,
workflowId: null,
kind: params.kind,
status: params.status,
message: params.message ?? null,
error: params.error ?? null,
metadata: params.metadata ?? null,
startedAt: now,
completedAt: now,
updatedAt: now,
})
}
/** Mark a tracked unit of work terminal (completed / completed_with_warnings / failed). */
export async function finishBackgroundWork(
executor: DbOrTx,
id: string,
params: {
status: Extract<BackgroundWorkStatusValue, 'completed' | 'completed_with_warnings' | 'failed'>
message?: string
error?: string
metadata?: unknown
}
): Promise<void> {
const now = new Date()
// Merge metadata so terminal counts (copied/failed) augment what start recorded
// (child name, per-kind plan) instead of replacing it.
let metadata: unknown
if (params.metadata !== undefined) {
const [existing] = await executor
.select({ metadata: backgroundWorkStatus.metadata })
.from(backgroundWorkStatus)
.where(eq(backgroundWorkStatus.id, id))
.limit(1)
metadata = { ...toMetadataRecord(existing?.metadata), ...toMetadataRecord(params.metadata) }
}
await executor
.update(backgroundWorkStatus)
.set({
status: params.status,
message: params.message ?? null,
error: params.error ?? null,
...(params.metadata !== undefined ? { metadata } : {}),
completedAt: now,
updatedAt: now,
})
.where(eq(backgroundWorkStatus.id, id))
}
/** Coerce an unknown jsonb metadata value to a plain record for safe merging. */
function toMetadataRecord(value: unknown): Record<string, unknown> {
return value && typeof value === 'object' && !Array.isArray(value)
? (value as Record<string, unknown>)
: {}
}
/** Keyset position of the last row of a page: `updatedAt` plus the `id` tiebreaker. */
interface BackgroundWorkCursorData {
updatedAt: string
id: string
}
/** Encodes the keyset position as an opaque base64 cursor (mirrors the audit log's). */
function encodeCursor(data: BackgroundWorkCursorData): string {
return Buffer.from(JSON.stringify(data)).toString('base64')
}
function decodeCursor(cursor: string): BackgroundWorkCursorData | null {
try {
return JSON.parse(Buffer.from(cursor, 'base64').toString())
} catch {
return null
}
}
/**
* Keyset condition for rows strictly after the cursor position in
* `updatedAt DESC, id DESC` order: older `updatedAt`, or the same `updatedAt`
* (which is not unique) with a smaller `id`. Null for an invalid cursor, which
* degrades to the first page rather than erroring.
*/
function buildCursorCondition(cursor: string): SQL<unknown> | null {
const cursorData = decodeCursor(cursor)
if (!cursorData?.updatedAt || !cursorData.id) return null
const cursorDate = new Date(cursorData.updatedAt)
if (Number.isNaN(cursorDate.getTime())) return null
return or(
lt(backgroundWorkStatus.updatedAt, cursorDate),
and(eq(backgroundWorkStatus.updatedAt, cursorDate), lt(backgroundWorkStatus.id, cursorData.id))
)!
}
export interface BackgroundWorkPage {
rows: BackgroundWorkRow[]
/** Cursor for the next page; null when this page is the last. */
nextCursor: string | null
}
/**
* Recent background-work jobs involving a workspace - the durable audit record the
* Activity view renders, most-recent first and keyset-paginated (`updatedAt DESC,
* id DESC`, cursor + limit mirroring the audit log's `queryAuditLogs`). Fork jobs
* are append-only (one row per fork), so this is the workspace's fork history;
* older rows are pruned by the cron.
*
* Every fork event is recorded ONCE, keyed to the workspace it was initiated from
* (fork-create → the parent; sync/rollback/sync-copy → the workspace whose page ran
* it), so "involving" matches both sides of each edge without double-writing rows:
*
* - rows keyed to this workspace (its own forks, syncs it ran, its rollbacks);
* - rows whose `metadata.childWorkspaceId` is this workspace (its own creation,
* recorded on the parent);
* - rows whose `metadata.otherWorkspaceId` is this workspace (the other side of a
* sync/rollback/sync-copy edge);
* - sync/rollback rows keyed to one of this workspace's forks - a fork's only sync
* edge is its parent, so these are guaranteed edge events (covers rows written
* before `metadata.otherWorkspaceId` existed).
*/
export async function listSurfacedBackgroundWork(
executor: DbOrTx,
workspaceId: string,
options?: { cursor?: string; limit?: number }
): Promise<BackgroundWorkPage> {
const limit = Math.min(
Math.max(options?.limit ?? BACKGROUND_WORK_PAGE_SIZE, 1),
BACKGROUND_WORK_PAGE_SIZE_MAX
)
const childRows = await executor
.select({ id: workspace.id })
.from(workspace)
.where(and(eq(workspace.forkedFromWorkspaceId, workspaceId), isNull(workspace.archivedAt)))
const childWorkspaceIds = childRows.map((row) => row.id)
const involvesWorkspace = or(
eq(backgroundWorkStatus.workspaceId, workspaceId),
sql`${backgroundWorkStatus.metadata} ->> 'childWorkspaceId' = ${workspaceId}`,
sql`${backgroundWorkStatus.metadata} ->> 'otherWorkspaceId' = ${workspaceId}`,
...(childWorkspaceIds.length > 0
? [
and(
inArray(backgroundWorkStatus.workspaceId, childWorkspaceIds),
inArray(backgroundWorkStatus.kind, ['fork_sync', 'fork_rollback'])
),
]
: [])
)
const conditions = [involvesWorkspace, inArray(backgroundWorkStatus.status, SURFACED_STATUSES)]
if (options?.cursor) {
const cursorCondition = buildCursorCondition(options.cursor)
if (cursorCondition) conditions.push(cursorCondition)
}
// Over-fetch by one row to learn whether another page exists (audit-log pattern).
const rows = (await executor
.select()
.from(backgroundWorkStatus)
.where(and(...conditions))
.orderBy(desc(backgroundWorkStatus.updatedAt), desc(backgroundWorkStatus.id))
.limit(limit + 1)) as BackgroundWorkRow[]
const hasMore = rows.length > limit
const page = rows.slice(0, limit)
const last = page[page.length - 1]
const nextCursor =
hasMore && last ? encodeCursor({ updatedAt: last.updatedAt.toISOString(), id: last.id }) : null
return { rows: page, nextCursor }
}
/**
* Fail background-work rows stuck in an active state past {@link STALE_ACTIVE_MS}: the
* worker crashed or restarted before writing a terminal status, and a hard crash has
* no in-task hook to recover from. Marks them `failed` so the UI shows the failure
* rather than an indefinitely-spinning banner. Touches ONLY `background_work_status`.
* Returns the count reaped; invoked from the outbox-processor cron.
*/
export async function reapStaleBackgroundWork(executor: DbOrTx): Promise<number> {
const now = new Date()
const cutoff = new Date(now.getTime() - STALE_ACTIVE_MS)
const reaped = await executor
.update(backgroundWorkStatus)
.set({
status: 'failed',
error: 'Background work did not finish in time (the worker may have restarted).',
completedAt: now,
updatedAt: now,
})
.where(
and(
inArray(backgroundWorkStatus.status, ['pending', 'processing']),
lte(backgroundWorkStatus.startedAt, cutoff)
)
)
.returning({ id: backgroundWorkStatus.id })
// Retention: the append-only fork audit trail would otherwise grow forever, so drop
// terminal rows past the retention window. The Activity tab paginates separately.
await executor
.delete(backgroundWorkStatus)
.where(
and(
inArray(backgroundWorkStatus.status, ['completed', 'completed_with_warnings', 'failed']),
lte(backgroundWorkStatus.updatedAt, new Date(now.getTime() - RETENTION_MS))
)
)
if (reaped.length > 0) {
logger.warn('Reaped stale background-work rows', {
count: reaped.length,
thresholdMs: STALE_ACTIVE_MS,
})
}
return reaped.length
}