chore: import upstream snapshot with attribution
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,433 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { knowledgeBase, workflow, workflowBlocks, workflowDeploymentVersion } from '@sim/db/schema'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const dbMock = vi.hoisted(() => {
|
||||
const reads = new Map<unknown, unknown[][]>()
|
||||
const updates: Array<{ table: unknown; values: Record<string, unknown> }> = []
|
||||
const deletes: Array<{ table: unknown }> = []
|
||||
|
||||
const nextPage = (table: unknown): unknown[] => {
|
||||
const pages = reads.get(table)
|
||||
return pages && pages.length > 0 ? (pages.shift() as unknown[]) : []
|
||||
}
|
||||
|
||||
// A drizzle-style read builder bound to one table: `.where`/`.orderBy`/`.limit` chain back to
|
||||
// the same builder, and awaiting it (at `.where()` or `.limit()`) shifts that table's next page.
|
||||
const makeReadBuilder = (table: unknown) => {
|
||||
const builder = {
|
||||
where: () => builder,
|
||||
orderBy: () => builder,
|
||||
limit: () => builder,
|
||||
then: (onFulfilled: (rows: unknown[]) => unknown, onRejected?: (error: unknown) => unknown) =>
|
||||
Promise.resolve(nextPage(table)).then(onFulfilled, onRejected),
|
||||
}
|
||||
return builder
|
||||
}
|
||||
|
||||
const db = {
|
||||
select: () => ({ from: (table: unknown) => makeReadBuilder(table) }),
|
||||
update: (table: unknown) => ({
|
||||
set: (values: Record<string, unknown>) => ({
|
||||
where: () => {
|
||||
updates.push({ table, values })
|
||||
return Promise.resolve([])
|
||||
},
|
||||
}),
|
||||
}),
|
||||
delete: (table: unknown) => ({
|
||||
where: () => {
|
||||
deletes.push({ table })
|
||||
return Promise.resolve([])
|
||||
},
|
||||
}),
|
||||
}
|
||||
|
||||
return {
|
||||
db,
|
||||
updates,
|
||||
deletes,
|
||||
queueRead: (table: unknown, ...pages: unknown[][]) => reads.set(table, pages),
|
||||
reset: () => {
|
||||
reads.clear()
|
||||
updates.length = 0
|
||||
deletes.length = 0
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
const { mockInvalidateDeployedStateCache } = vi.hoisted(() => ({
|
||||
mockInvalidateDeployedStateCache: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@sim/db', () => ({
|
||||
db: dbMock.db,
|
||||
dbReplica: dbMock.db,
|
||||
runOutsideTransactionContext: <T>(fn: () => T): T => fn(),
|
||||
instrumentPoolClient: <T>(client: T): T => client,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/workflows/persistence/utils', () => ({
|
||||
invalidateDeployedStateCache: mockInvalidateDeployedStateCache,
|
||||
CREDENTIAL_SUBBLOCK_IDS: new Set(['credential', 'manualCredential', 'triggerCredentials']),
|
||||
}))
|
||||
|
||||
// The reference indexer resolves a tool's params via the tool registry; stub it so loading the
|
||||
// remap module never pulls the full registry (this file only exercises top-level selectors).
|
||||
vi.mock('@/tools/params', () => ({
|
||||
getToolIdForOperation: () => undefined,
|
||||
getToolParametersConfig: () => null,
|
||||
getSubBlocksForToolInput: (
|
||||
_toolId: string,
|
||||
_type: string,
|
||||
_values: unknown,
|
||||
_modes: unknown,
|
||||
provided?: { subBlocks?: SubBlockConfig[] }
|
||||
) => ({ subBlocks: provided?.subBlocks ?? [] }),
|
||||
formatParameterLabel: (label: string) => label,
|
||||
}))
|
||||
|
||||
import { getBlock } from '@/blocks/registry'
|
||||
import type { BlockConfig, SubBlockConfig } from '@/blocks/types'
|
||||
import {
|
||||
clearFailedForkResourceReferences,
|
||||
clearFailedReferencesInDeploymentVersions,
|
||||
clearFailedReferencesInWorkflows,
|
||||
rewriteDeploymentVersionState,
|
||||
} from '@/ee/workspace-forking/lib/copy/cleanup-failed'
|
||||
import type { ForkCopyResolver } from '@/ee/workspace-forking/lib/remap/fork-bootstrap'
|
||||
import type { ForkRemapKind } from '@/ee/workspace-forking/lib/remap/remap-references'
|
||||
|
||||
const blockWith = (subBlocks: SubBlockConfig[]): BlockConfig =>
|
||||
({ name: 'Knowledge', description: '', subBlocks, outputs: {} }) as unknown as BlockConfig
|
||||
|
||||
/** A KB block whose `documentId` (document-selector) hangs off `knowledgeBaseId` (kb-selector). */
|
||||
const kbBlockConfig = () =>
|
||||
blockWith([
|
||||
{ id: 'knowledgeBaseId', title: 'KB', type: 'knowledge-base-selector' },
|
||||
{ id: 'documentId', title: 'Doc', type: 'document-selector', dependsOn: ['knowledgeBaseId'] },
|
||||
])
|
||||
|
||||
/** An agent block whose `tools` tool-input holds a KB tool with a nested `knowledgeBaseId` param. */
|
||||
const agentToolConfig = (type: string): BlockConfig => {
|
||||
if (type === 'agent') return blockWith([{ id: 'tools', title: 'Tools', type: 'tool-input' }])
|
||||
if (type === 'kbtool')
|
||||
return blockWith([{ id: 'knowledgeBaseId', title: 'KB', type: 'knowledge-base-selector' }])
|
||||
return undefined as unknown as BlockConfig
|
||||
}
|
||||
|
||||
/** A deployment-version state whose agent block references `kbId` inside a tool-input tool param. */
|
||||
const agentVersionState = (kbId: string) => ({
|
||||
blocks: {
|
||||
'agent-1': {
|
||||
id: 'agent-1',
|
||||
type: 'agent',
|
||||
name: 'Agent',
|
||||
subBlocks: {
|
||||
tools: {
|
||||
id: 'tools',
|
||||
type: 'tool-input',
|
||||
value: [{ type: 'kbtool', toolId: 'kbtool_search', params: { knowledgeBaseId: kbId } }],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
edges: [],
|
||||
loops: {},
|
||||
parallels: {},
|
||||
})
|
||||
|
||||
type AgentStateBlocks = {
|
||||
blocks: {
|
||||
'agent-1': { subBlocks: { tools: { value: Array<{ params: { knowledgeBaseId: string } }> } } }
|
||||
}
|
||||
}
|
||||
const nestedKbValue = (state: unknown) =>
|
||||
(state as AgentStateBlocks).blocks['agent-1'].subBlocks.tools.value[0].params.knowledgeBaseId
|
||||
|
||||
/** A serialized deployment-version state whose single block points its KB selector at `kbId`. */
|
||||
const versionState = (kbId: string) => ({
|
||||
blocks: {
|
||||
'block-1': {
|
||||
id: 'block-1',
|
||||
type: 'knowledge',
|
||||
name: 'KB Block',
|
||||
subBlocks: {
|
||||
knowledgeBaseId: { id: 'knowledgeBaseId', type: 'knowledge-base-selector', value: kbId },
|
||||
documentId: { id: 'documentId', type: 'document-selector', value: 'doc-keep' },
|
||||
},
|
||||
},
|
||||
},
|
||||
edges: [],
|
||||
loops: {},
|
||||
parallels: {},
|
||||
})
|
||||
|
||||
const draftBlockRow = (kbId: string) => ({
|
||||
id: 'b-1',
|
||||
workflowId: 'wf-1',
|
||||
type: 'knowledge',
|
||||
subBlocks: {
|
||||
knowledgeBaseId: { id: 'knowledgeBaseId', type: 'knowledge-base-selector', value: kbId },
|
||||
documentId: { id: 'documentId', type: 'document-selector', value: 'doc-keep' },
|
||||
},
|
||||
})
|
||||
|
||||
/** A draft block whose `file-upload` subblock points at a copied workspace-file storage key. */
|
||||
const fileBlockRow = (fileKey: string) => ({
|
||||
id: 'b-file',
|
||||
workflowId: 'wf-1',
|
||||
type: 'agent',
|
||||
subBlocks: {
|
||||
file: { id: 'file', type: 'file-upload', value: { key: fileKey, name: 'a.png' } },
|
||||
},
|
||||
})
|
||||
|
||||
const failedKbResolver: ForkCopyResolver = (kind, id) =>
|
||||
kind === 'knowledge-base' && id === 'failed-kb' ? null : id
|
||||
|
||||
const failedByKind = () =>
|
||||
new Map<ForkRemapKind, Set<string>>([['knowledge-base', new Set(['failed-kb'])]])
|
||||
|
||||
type StateBlocks = { blocks: Record<string, { subBlocks: Record<string, { value: unknown }> }> }
|
||||
const kbValue = (state: unknown) =>
|
||||
(state as StateBlocks).blocks['block-1'].subBlocks.knowledgeBaseId.value
|
||||
const docValue = (state: unknown) =>
|
||||
(state as StateBlocks).blocks['block-1'].subBlocks.documentId.value
|
||||
|
||||
describe('cleanup-failed', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
dbMock.reset()
|
||||
vi.mocked(getBlock).mockReturnValue(kbBlockConfig())
|
||||
})
|
||||
|
||||
describe('rewriteDeploymentVersionState', () => {
|
||||
it('clears a block ref that resolves to a failed id and its dependents', () => {
|
||||
const result = rewriteDeploymentVersionState(versionState('failed-kb'), failedKbResolver)
|
||||
expect(result.changed).toBe(true)
|
||||
expect(kbValue(result.state)).toBe('')
|
||||
// documentId hangs off knowledgeBaseId, so the cleared parent clears it too.
|
||||
expect(docValue(result.state)).toBe('')
|
||||
})
|
||||
|
||||
it('leaves a state that references no failed id untouched (same reference, not changed)', () => {
|
||||
const input = versionState('other-kb')
|
||||
const result = rewriteDeploymentVersionState(input, failedKbResolver)
|
||||
expect(result.changed).toBe(false)
|
||||
expect(result.state).toBe(input)
|
||||
})
|
||||
|
||||
it('is tolerant of a malformed state shape', () => {
|
||||
const input = { not: 'a workflow state' }
|
||||
const result = rewriteDeploymentVersionState(input, failedKbResolver)
|
||||
expect(result.changed).toBe(false)
|
||||
expect(result.state).toBe(input)
|
||||
})
|
||||
|
||||
// The deployed-version sweep must clear EVERY subblock variety the draft sweep does -
|
||||
// including a failed id nested in an agent block's `tool-input` tool params, not only
|
||||
// top-level selectors - via the shared remapForkSubBlocks/clearFailedSubBlockReferences.
|
||||
it('clears a failed id nested in an agent tool-input param inside a deployed version', () => {
|
||||
vi.mocked(getBlock).mockImplementation((type) => agentToolConfig(type))
|
||||
const result = rewriteDeploymentVersionState(agentVersionState('failed-kb'), failedKbResolver)
|
||||
expect(result.changed).toBe(true)
|
||||
expect(nestedKbValue(result.state)).toBe('')
|
||||
})
|
||||
|
||||
it('leaves an agent tool-input param that references no failed id untouched', () => {
|
||||
vi.mocked(getBlock).mockImplementation((type) => agentToolConfig(type))
|
||||
const input = agentVersionState('other-kb')
|
||||
const result = rewriteDeploymentVersionState(input, failedKbResolver)
|
||||
expect(result.changed).toBe(false)
|
||||
expect(result.state).toBe(input)
|
||||
})
|
||||
})
|
||||
|
||||
describe('clearFailedReferencesInWorkflows', () => {
|
||||
it('sweeps the draft blocks and returns the affected workflow ids', async () => {
|
||||
dbMock.queueRead(workflow, [{ id: 'wf-1' }])
|
||||
dbMock.queueRead(workflowBlocks, [draftBlockRow('failed-kb')])
|
||||
|
||||
const affected = await clearFailedReferencesInWorkflows('child-ws', failedByKind(), 'test')
|
||||
|
||||
expect([...affected]).toEqual(['wf-1'])
|
||||
expect(dbMock.updates).toHaveLength(1)
|
||||
expect(dbMock.updates[0].table).toBe(workflowBlocks)
|
||||
const cleared = dbMock.updates[0].values.subBlocks as Record<string, { value: unknown }>
|
||||
expect(cleared.knowledgeBaseId.value).toBe('')
|
||||
expect(cleared.documentId.value).toBe('')
|
||||
})
|
||||
|
||||
it('returns an empty set and writes nothing when no block references a failed id', async () => {
|
||||
dbMock.queueRead(workflow, [{ id: 'wf-1' }])
|
||||
dbMock.queueRead(workflowBlocks, [draftBlockRow('other-kb')])
|
||||
|
||||
const affected = await clearFailedReferencesInWorkflows('child-ws', failedByKind(), 'test')
|
||||
|
||||
expect(affected.size).toBe(0)
|
||||
expect(dbMock.updates).toHaveLength(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('clearFailedReferencesInDeploymentVersions', () => {
|
||||
it('rewrites a version referencing a failed id and invalidates its deployed-state cache', async () => {
|
||||
dbMock.queueRead(workflowDeploymentVersion, [
|
||||
{ id: 'dv-1', version: 5, state: versionState('failed-kb') },
|
||||
])
|
||||
|
||||
await clearFailedReferencesInDeploymentVersions(new Set(['wf-1']), failedByKind(), 'test')
|
||||
|
||||
expect(dbMock.updates).toHaveLength(1)
|
||||
expect(dbMock.updates[0].table).toBe(workflowDeploymentVersion)
|
||||
expect(kbValue(dbMock.updates[0].values.state)).toBe('')
|
||||
expect(docValue(dbMock.updates[0].values.state)).toBe('')
|
||||
expect(mockInvalidateDeployedStateCache).toHaveBeenCalledTimes(1)
|
||||
expect(mockInvalidateDeployedStateCache).toHaveBeenCalledWith('dv-1')
|
||||
})
|
||||
|
||||
it('leaves a version that does not reference a failed id unwritten and uncached', async () => {
|
||||
dbMock.queueRead(workflowDeploymentVersion, [
|
||||
{ id: 'dv-old', version: 3, state: versionState('other-kb') },
|
||||
])
|
||||
|
||||
await clearFailedReferencesInDeploymentVersions(new Set(['wf-1']), failedByKind(), 'test')
|
||||
|
||||
expect(dbMock.updates).toHaveLength(0)
|
||||
expect(mockInvalidateDeployedStateCache).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('writes only the changed version when a workflow mixes referencing and non-referencing versions', async () => {
|
||||
dbMock.queueRead(workflowDeploymentVersion, [
|
||||
{ id: 'dv-active', version: 5, state: versionState('failed-kb') },
|
||||
{ id: 'dv-old', version: 4, state: versionState('other-kb') },
|
||||
])
|
||||
|
||||
await clearFailedReferencesInDeploymentVersions(new Set(['wf-1']), failedByKind(), 'test')
|
||||
|
||||
expect(dbMock.updates).toHaveLength(1)
|
||||
expect(mockInvalidateDeployedStateCache).toHaveBeenCalledTimes(1)
|
||||
expect(mockInvalidateDeployedStateCache).toHaveBeenCalledWith('dv-active')
|
||||
})
|
||||
|
||||
it('does nothing when no workflows were affected', async () => {
|
||||
await clearFailedReferencesInDeploymentVersions(new Set(), failedByKind(), 'test')
|
||||
expect(dbMock.updates).toHaveLength(0)
|
||||
expect(mockInvalidateDeployedStateCache).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('clearFailedForkResourceReferences', () => {
|
||||
it('threads the draft sweep into the deployed sweep, then drops the placeholder', async () => {
|
||||
dbMock.queueRead(workflow, [{ id: 'wf-1' }])
|
||||
dbMock.queueRead(workflowBlocks, [draftBlockRow('failed-kb')])
|
||||
dbMock.queueRead(workflowDeploymentVersion, [
|
||||
{ id: 'dv-active', version: 5, state: versionState('failed-kb') },
|
||||
{ id: 'dv-old', version: 4, state: versionState('other-kb') },
|
||||
])
|
||||
|
||||
const cleaned = await clearFailedForkResourceReferences({
|
||||
childWorkspaceId: 'child-ws',
|
||||
failures: [{ kind: 'knowledge-base', childId: 'failed-kb', documentChildIds: [] }],
|
||||
requestId: 'test',
|
||||
})
|
||||
|
||||
expect(cleaned).toEqual({ cleared: 1, clearingFailed: false })
|
||||
// One draft block update + one deployed version update (only the referencing version).
|
||||
const updatedTables = dbMock.updates.map((u) => u.table)
|
||||
expect(updatedTables).toEqual([workflowBlocks, workflowDeploymentVersion])
|
||||
expect(mockInvalidateDeployedStateCache).toHaveBeenCalledTimes(1)
|
||||
expect(mockInvalidateDeployedStateCache).toHaveBeenCalledWith('dv-active')
|
||||
// The orphaned KB placeholder is dropped after both sweeps.
|
||||
expect(dbMock.deletes).toHaveLength(1)
|
||||
expect(dbMock.deletes[0].table).toBe(knowledgeBase)
|
||||
})
|
||||
|
||||
it('still drops the placeholder when no workflow referenced the failed resource', async () => {
|
||||
dbMock.queueRead(workflow, [{ id: 'wf-1' }])
|
||||
dbMock.queueRead(workflowBlocks, [draftBlockRow('other-kb')])
|
||||
|
||||
const cleaned = await clearFailedForkResourceReferences({
|
||||
childWorkspaceId: 'child-ws',
|
||||
failures: [{ kind: 'knowledge-base', childId: 'failed-kb', documentChildIds: [] }],
|
||||
requestId: 'test',
|
||||
})
|
||||
|
||||
expect(cleaned).toEqual({ cleared: 1, clearingFailed: false })
|
||||
// No draft block referenced the failed id AND no deployed targets were threaded, so the
|
||||
// deployed sweep is skipped entirely.
|
||||
expect(dbMock.updates).toHaveLength(0)
|
||||
expect(mockInvalidateDeployedStateCache).not.toHaveBeenCalled()
|
||||
expect(dbMock.deletes).toHaveLength(1)
|
||||
expect(dbMock.deletes[0].table).toBe(knowledgeBase)
|
||||
})
|
||||
|
||||
it('sweeps a deployed target version even when no draft referenced the failed id', async () => {
|
||||
// Draft is clean (other-kb), but a deployed target version still points at the dropped
|
||||
// placeholder - the deployed-target scope (not draft divergence) catches it.
|
||||
dbMock.queueRead(workflow, [{ id: 'wf-1' }])
|
||||
dbMock.queueRead(workflowBlocks, [draftBlockRow('other-kb')])
|
||||
dbMock.queueRead(workflowDeploymentVersion, [
|
||||
{ id: 'dv-1', version: 5, state: versionState('failed-kb') },
|
||||
])
|
||||
|
||||
const cleaned = await clearFailedForkResourceReferences({
|
||||
childWorkspaceId: 'child-ws',
|
||||
failures: [{ kind: 'knowledge-base', childId: 'failed-kb', documentChildIds: [] }],
|
||||
deployedTargetWorkflowIds: ['wf-deployed'],
|
||||
requestId: 'test',
|
||||
})
|
||||
|
||||
expect(cleaned).toEqual({ cleared: 1, clearingFailed: false })
|
||||
expect(dbMock.updates.map((u) => u.table)).toContain(workflowDeploymentVersion)
|
||||
expect(mockInvalidateDeployedStateCache).toHaveBeenCalledWith('dv-1')
|
||||
// Clearing succeeded, so the placeholder is dropped.
|
||||
expect(dbMock.deletes[0].table).toBe(knowledgeBase)
|
||||
})
|
||||
|
||||
it('clears a file-upload reference to a failed copied blob and drops no row', async () => {
|
||||
dbMock.queueRead(workflow, [{ id: 'wf-1' }])
|
||||
dbMock.queueRead(workflowBlocks, [fileBlockRow('workspace/child/failed.png')])
|
||||
|
||||
const cleaned = await clearFailedForkResourceReferences({
|
||||
childWorkspaceId: 'child-ws',
|
||||
failures: [{ kind: 'file', childKey: 'workspace/child/failed.png' }],
|
||||
requestId: 'test',
|
||||
})
|
||||
|
||||
expect(cleaned).toEqual({ cleared: 1, clearingFailed: false })
|
||||
expect(dbMock.updates).toHaveLength(1)
|
||||
expect(dbMock.updates[0].table).toBe(workflowBlocks)
|
||||
const cleared = dbMock.updates[0].values.subBlocks as Record<string, { value: unknown }>
|
||||
expect(cleared.file.value).toBe('')
|
||||
// A failed file has no placeholder row to drop (the metadata row stays re-uploadable).
|
||||
expect(dbMock.deletes).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('reports cleared:0 + clearingFailed and skips the placeholder drop when a clear phase throws', async () => {
|
||||
// A clear-phase failure must not drop the placeholder: that would turn an empty placeholder
|
||||
// into a dangling reference to a deleted row. Make the draft block UPDATE throw.
|
||||
dbMock.queueRead(workflow, [{ id: 'wf-1' }])
|
||||
dbMock.queueRead(workflowBlocks, [draftBlockRow('failed-kb')])
|
||||
const originalUpdate = dbMock.db.update
|
||||
dbMock.db.update = () => {
|
||||
throw new Error('update failed')
|
||||
}
|
||||
try {
|
||||
const cleaned = await clearFailedForkResourceReferences({
|
||||
childWorkspaceId: 'child-ws',
|
||||
failures: [{ kind: 'knowledge-base', childId: 'failed-kb', documentChildIds: [] }],
|
||||
requestId: 'test',
|
||||
})
|
||||
// The count must NOT overstate: nothing was cleared and the flag marks cleanup incomplete.
|
||||
expect(cleaned).toEqual({ cleared: 0, clearingFailed: true })
|
||||
} finally {
|
||||
dbMock.db.update = originalUpdate
|
||||
}
|
||||
// The drop is skipped, so the placeholder row survives (no delete issued).
|
||||
expect(dbMock.deletes).toHaveLength(0)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,364 @@
|
||||
import { db } from '@sim/db'
|
||||
import {
|
||||
document,
|
||||
knowledgeBase,
|
||||
userTableDefinitions,
|
||||
workflow,
|
||||
workflowBlocks,
|
||||
workflowDeploymentVersion,
|
||||
} from '@sim/db/schema'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { and, asc, eq, gt, inArray } from 'drizzle-orm'
|
||||
import { isRecord, type SubBlockRecord } from '@/lib/workflows/persistence/remap-internal-ids'
|
||||
import { invalidateDeployedStateCache } from '@/lib/workflows/persistence/utils'
|
||||
import type { ForkFailedResource } from '@/ee/workspace-forking/lib/copy/copy-resources'
|
||||
import type { ForkCopyResolver } from '@/ee/workspace-forking/lib/remap/fork-bootstrap'
|
||||
import {
|
||||
clearDependentsOnRemap,
|
||||
type ForkRemapKind,
|
||||
remapForkSubBlocks,
|
||||
} from '@/ee/workspace-forking/lib/remap/remap-references'
|
||||
|
||||
const logger = createLogger('WorkspaceForkCleanupFailed')
|
||||
|
||||
/** Child workflow ids loaded per page so the block sweep never materializes the whole workspace. */
|
||||
const WORKFLOW_PAGE = 200
|
||||
|
||||
/** Deployment versions loaded per page so a workflow with many versions never loads all at once. */
|
||||
const DEPLOYMENT_VERSION_PAGE = 100
|
||||
|
||||
/** Identity-or-clear resolver: a failed id resolves to null (cleared), any other id to itself. */
|
||||
function buildFailedResolver(failedByKind: Map<ForkRemapKind, Set<string>>): ForkCopyResolver {
|
||||
return (kind, id) => (failedByKind.get(kind)?.has(id) ? null : id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the identity-or-clear resolver to one block's subBlocks: a value (top-level selector or
|
||||
* nested tool param) that resolves to a failed id is cleared, and its `dependsOn` children with it;
|
||||
* everything else is left untouched. Returns the rewritten record plus whether anything changed.
|
||||
* Shared by the draft block sweep and the deployment-version state sweep so both clear identically.
|
||||
*/
|
||||
function clearFailedSubBlockReferences(
|
||||
subBlocks: SubBlockRecord,
|
||||
blockType: string,
|
||||
resolve: ForkCopyResolver
|
||||
): { subBlocks: SubBlockRecord; changed: boolean } {
|
||||
const result = remapForkSubBlocks(subBlocks, resolve, 'create')
|
||||
// remappedKeys is non-empty only when a failed id was actually cleared, so a block that
|
||||
// referenced nothing failed is reported unchanged without a write.
|
||||
if (result.remappedKeys.size === 0) return { subBlocks, changed: false }
|
||||
return {
|
||||
subBlocks: clearDependentsOnRemap(result.subBlocks, blockType, result.remappedKeys),
|
||||
changed: true,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up after a resource whose post-commit content fill failed: clear every subblock
|
||||
* reference in the child workspace's workflows that points at the failed resource (so no
|
||||
* subblock keeps a dead id), then drop the orphaned placeholder rows (a KB cascade-drops its
|
||||
* documents + embeddings; a table cascade-drops its rows). In-content references inside copied
|
||||
* skill/markdown bodies are intentionally left as graceful broken links rather than mutated.
|
||||
*
|
||||
* The deployed-version sweep covers the draft-affected workflows UNION this sync's deployed
|
||||
* target workflows (`deployedTargetWorkflowIds`): a deployed version can reference the dropped
|
||||
* placeholder even when the draft no longer does (the user edited the empty-looking block in the
|
||||
* fill window), so scoping to draft divergence alone would miss it.
|
||||
*
|
||||
* Best-effort and isolated: a failure cleaning one resource is logged and the rest continue, so
|
||||
* a cleanup error never aborts the others. The placeholder drop is SKIPPED when a reference-clear
|
||||
* phase threw - dropping it then would turn an empty placeholder into a dangling reference to a
|
||||
* deleted row; leaving it keeps the reference resolvable (to empty content) until a later retry.
|
||||
*
|
||||
* Returns `{ cleared, clearingFailed }` for the fork activity metadata. `clearingFailed` is true
|
||||
* when a reference-clear phase threw - placeholders were then NOT dropped - and `cleared` is 0 in
|
||||
* that case, so the report never claims references it did not actually clear. On success `cleared`
|
||||
* is the count of failed resources whose references were cleared.
|
||||
*
|
||||
* Storage accounting: this cleanup never decrements storage usage because it never removes
|
||||
* anything that was counted. Copied file blobs are the only counted copies (incremented in
|
||||
* `executeForkFileBlobCopies` only after the blob lands), and a failed file's blob never
|
||||
* landed - its metadata row is intentionally left re-uploadable, and nothing was charged. The
|
||||
* dropped table/KB/document placeholders are DB rows the upload path never counts, and any KB
|
||||
* blobs copied before their KB failed are left in storage (rows only are dropped here) but
|
||||
* uncounted - mirroring the KB upload path, which never counts KB blobs.
|
||||
*/
|
||||
export async function clearFailedForkResourceReferences(params: {
|
||||
childWorkspaceId: string
|
||||
failures: ForkFailedResource[]
|
||||
/** Target workflows this sync deployed; their deployed versions are swept regardless of draft. */
|
||||
deployedTargetWorkflowIds?: string[]
|
||||
requestId?: string
|
||||
}): Promise<{ cleared: number; clearingFailed: boolean }> {
|
||||
const { childWorkspaceId, failures, requestId = 'unknown' } = params
|
||||
if (failures.length === 0) return { cleared: 0, clearingFailed: false }
|
||||
|
||||
const failedByKind = new Map<ForkRemapKind, Set<string>>()
|
||||
const markFailed = (kind: ForkRemapKind, id: string) => {
|
||||
const set = failedByKind.get(kind)
|
||||
if (set) set.add(id)
|
||||
else failedByKind.set(kind, new Set([id]))
|
||||
}
|
||||
const tableIds: string[] = []
|
||||
const kbIds: string[] = []
|
||||
// Standalone documents copied into an already-existing target KB (the doc-into-mapped-KB sync
|
||||
// path) - dropped individually, since their KB is not ours to remove.
|
||||
const docIds: string[] = []
|
||||
for (const failure of failures) {
|
||||
if (failure.kind === 'table') {
|
||||
markFailed('table', failure.childId)
|
||||
tableIds.push(failure.childId)
|
||||
} else if (failure.kind === 'knowledge-document') {
|
||||
markFailed('knowledge-document', failure.childId)
|
||||
docIds.push(failure.childId)
|
||||
} else if (failure.kind === 'file') {
|
||||
// A failed file blob: clear `file-upload` references to its copied storage key. No row to
|
||||
// drop - the metadata row is left in place so the user can re-upload the missing blob.
|
||||
markFailed('file', failure.childKey)
|
||||
} else {
|
||||
markFailed('knowledge-base', failure.childId)
|
||||
for (const docId of failure.documentChildIds) markFailed('knowledge-document', docId)
|
||||
kbIds.push(failure.childId)
|
||||
}
|
||||
}
|
||||
|
||||
// Whether BOTH reference-clear phases completed without throwing. The placeholder drop below is
|
||||
// gated on this: if clearing threw, a workflow (draft or deployed version) may still reference
|
||||
// the failed id, so dropping its placeholder would create a dangling reference to a deleted row.
|
||||
let clearingSucceeded = true
|
||||
|
||||
let affectedWorkflowIds: Set<string> = new Set()
|
||||
try {
|
||||
affectedWorkflowIds = await clearFailedReferencesInWorkflows(
|
||||
childWorkspaceId,
|
||||
failedByKind,
|
||||
requestId
|
||||
)
|
||||
} catch (error) {
|
||||
clearingSucceeded = false
|
||||
logger.error(`[${requestId}] Failed to clear references for failed fork resources`, {
|
||||
childWorkspaceId,
|
||||
error: getErrorMessage(error),
|
||||
})
|
||||
}
|
||||
|
||||
// The same dead id also lives in DEPLOYED version states (the active one re-cut from the
|
||||
// placeholder-referencing draft at this sync's redeploy, plus any newer version from a rare
|
||||
// race), not only the draft just swept above. Sweep those too so "Load deployment" can't
|
||||
// re-poison the cleaned draft with a dropped id, and the execute path never runs against content
|
||||
// that no longer exists. Scope is the draft-affected workflows UNION this sync's deployed target
|
||||
// workflows: a deployed version may reference the placeholder even when the draft no longer does
|
||||
// (edited in the fill window), so draft divergence alone is not a reliable scope. Isolated: a
|
||||
// failure here never aborts a sibling resource's cleanup.
|
||||
const deployedSweepIds = new Set(affectedWorkflowIds)
|
||||
for (const id of params.deployedTargetWorkflowIds ?? []) deployedSweepIds.add(id)
|
||||
try {
|
||||
await clearFailedReferencesInDeploymentVersions(deployedSweepIds, failedByKind, requestId)
|
||||
} catch (error) {
|
||||
clearingSucceeded = false
|
||||
logger.error(`[${requestId}] Failed to clear references in fork deployment versions`, {
|
||||
childWorkspaceId,
|
||||
error: getErrorMessage(error),
|
||||
})
|
||||
}
|
||||
|
||||
// Drop the orphaned placeholders. The KB delete cascades its documents + embeddings; the
|
||||
// table delete cascades its rows; a standalone document delete cascades its embeddings. Done
|
||||
// after the refs are cleared so a drop failure can't strand a workflow still pointing at the
|
||||
// (now content-less) resource - and ONLY when clearing succeeded, so a clear failure never
|
||||
// turns an empty placeholder into a dangling reference to a deleted row.
|
||||
if (!clearingSucceeded) {
|
||||
logger.warn(
|
||||
`[${requestId}] Skipping fork resource placeholder drop after a reference-clear failure`,
|
||||
{
|
||||
childWorkspaceId,
|
||||
tables: tableIds.length,
|
||||
knowledgeBases: kbIds.length,
|
||||
documents: docIds.length,
|
||||
}
|
||||
)
|
||||
return { cleared: 0, clearingFailed: true }
|
||||
}
|
||||
try {
|
||||
if (tableIds.length > 0) {
|
||||
await db.delete(userTableDefinitions).where(inArray(userTableDefinitions.id, tableIds))
|
||||
}
|
||||
if (kbIds.length > 0) {
|
||||
await db.delete(knowledgeBase).where(inArray(knowledgeBase.id, kbIds))
|
||||
}
|
||||
if (docIds.length > 0) {
|
||||
await db.delete(document).where(inArray(document.id, docIds))
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Failed to drop orphaned fork resource placeholders`, {
|
||||
childWorkspaceId,
|
||||
error: getErrorMessage(error),
|
||||
})
|
||||
}
|
||||
|
||||
return { cleared: failures.length, clearingFailed: false }
|
||||
}
|
||||
|
||||
/**
|
||||
* Sweep the child workspace's workflow blocks and clear any subblock value (top-level selector
|
||||
* or nested tool param) that resolves to a failed child resource id. Reuses the create-mode
|
||||
* remap with an identity-or-clear resolver: a non-failed id resolves to itself (left unchanged),
|
||||
* a failed id resolves to null and is cleared, and its `dependsOn` children are cleared too.
|
||||
* Returns the ids of the workflows whose blocks actually had a reference cleared, so the deployed
|
||||
* version sweep can scope itself to exactly those workflows.
|
||||
*/
|
||||
export async function clearFailedReferencesInWorkflows(
|
||||
childWorkspaceId: string,
|
||||
failedByKind: Map<ForkRemapKind, Set<string>>,
|
||||
requestId: string
|
||||
): Promise<Set<string>> {
|
||||
const resolve = buildFailedResolver(failedByKind)
|
||||
const affectedWorkflowIds = new Set<string>()
|
||||
|
||||
let afterWorkflowId: string | null = null
|
||||
for (;;) {
|
||||
const workflowRows = await db
|
||||
.select({ id: workflow.id })
|
||||
.from(workflow)
|
||||
.where(
|
||||
afterWorkflowId === null
|
||||
? eq(workflow.workspaceId, childWorkspaceId)
|
||||
: and(eq(workflow.workspaceId, childWorkspaceId), gt(workflow.id, afterWorkflowId))
|
||||
)
|
||||
.orderBy(asc(workflow.id))
|
||||
.limit(WORKFLOW_PAGE)
|
||||
if (workflowRows.length === 0) break
|
||||
|
||||
const workflowIds = workflowRows.map((row) => row.id)
|
||||
const blocks = await db
|
||||
.select({
|
||||
id: workflowBlocks.id,
|
||||
workflowId: workflowBlocks.workflowId,
|
||||
type: workflowBlocks.type,
|
||||
subBlocks: workflowBlocks.subBlocks,
|
||||
})
|
||||
.from(workflowBlocks)
|
||||
.where(inArray(workflowBlocks.workflowId, workflowIds))
|
||||
|
||||
for (const block of blocks) {
|
||||
const current = (block.subBlocks ?? {}) as SubBlockRecord
|
||||
const { subBlocks: cleared, changed } = clearFailedSubBlockReferences(
|
||||
current,
|
||||
block.type,
|
||||
resolve
|
||||
)
|
||||
if (!changed) continue
|
||||
await db
|
||||
.update(workflowBlocks)
|
||||
.set({ subBlocks: cleared })
|
||||
.where(eq(workflowBlocks.id, block.id))
|
||||
affectedWorkflowIds.add(block.workflowId)
|
||||
}
|
||||
|
||||
if (workflowRows.length < WORKFLOW_PAGE) break
|
||||
afterWorkflowId = workflowIds[workflowIds.length - 1]
|
||||
}
|
||||
|
||||
return affectedWorkflowIds
|
||||
}
|
||||
|
||||
/**
|
||||
* Rewrite a deployment version's serialized state in memory, clearing every block subblock that
|
||||
* resolves to a failed id (and its `dependsOn` children). Returns `changed: false` with the
|
||||
* original state when no block referenced a failed id - so a version cut before this sync, which
|
||||
* cannot contain the new placeholder id, is a no-op and never written back. Tolerant of a
|
||||
* malformed/legacy state shape (anything that is not `{ blocks: {...} }` is left untouched).
|
||||
*/
|
||||
export function rewriteDeploymentVersionState(
|
||||
state: unknown,
|
||||
resolve: ForkCopyResolver
|
||||
): { state: unknown; changed: boolean } {
|
||||
if (!isRecord(state) || !isRecord(state.blocks)) return { state, changed: false }
|
||||
|
||||
let nextBlocks: Record<string, unknown> | null = null
|
||||
for (const [blockId, block] of Object.entries(state.blocks)) {
|
||||
if (!isRecord(block)) continue
|
||||
const blockType = typeof block.type === 'string' ? block.type : undefined
|
||||
if (!blockType || !isRecord(block.subBlocks)) continue
|
||||
const { subBlocks: cleared, changed } = clearFailedSubBlockReferences(
|
||||
block.subBlocks as SubBlockRecord,
|
||||
blockType,
|
||||
resolve
|
||||
)
|
||||
if (!changed) continue
|
||||
nextBlocks ??= { ...state.blocks }
|
||||
nextBlocks[blockId] = { ...block, subBlocks: cleared }
|
||||
}
|
||||
|
||||
if (!nextBlocks) return { state, changed: false }
|
||||
return { state: { ...state, blocks: nextBlocks }, changed: true }
|
||||
}
|
||||
|
||||
/**
|
||||
* Rewrite the DEPLOYED version states of the workflows whose draft blocks were just swept. The dead
|
||||
* placeholder id lives in any version re-cut from the (placeholder-referencing) draft at this sync's
|
||||
* redeploy - in practice the active one - so it must be cleared there too, not only in the draft.
|
||||
* Every version of each affected workflow is examined, but only versions whose state actually
|
||||
* changes are written: an older version predates the sync and cannot contain the new id, so it is a
|
||||
* no-op. After a version is rewritten its cached deployed state is evicted so execute/serve rebuilds
|
||||
* from the cleaned snapshot. Bounded work (no long transaction): per-version short UPDATEs, versions
|
||||
* keyset-paginated, and a per-workflow failure is logged without aborting the other workflows.
|
||||
*/
|
||||
export async function clearFailedReferencesInDeploymentVersions(
|
||||
workflowIds: ReadonlySet<string>,
|
||||
failedByKind: Map<ForkRemapKind, Set<string>>,
|
||||
requestId: string
|
||||
): Promise<void> {
|
||||
if (workflowIds.size === 0) return
|
||||
const resolve = buildFailedResolver(failedByKind)
|
||||
|
||||
for (const workflowId of workflowIds) {
|
||||
try {
|
||||
let afterVersion: number | null = null
|
||||
for (;;) {
|
||||
const versions = await db
|
||||
.select({
|
||||
id: workflowDeploymentVersion.id,
|
||||
version: workflowDeploymentVersion.version,
|
||||
state: workflowDeploymentVersion.state,
|
||||
})
|
||||
.from(workflowDeploymentVersion)
|
||||
.where(
|
||||
afterVersion === null
|
||||
? eq(workflowDeploymentVersion.workflowId, workflowId)
|
||||
: and(
|
||||
eq(workflowDeploymentVersion.workflowId, workflowId),
|
||||
gt(workflowDeploymentVersion.version, afterVersion)
|
||||
)
|
||||
)
|
||||
.orderBy(asc(workflowDeploymentVersion.version))
|
||||
.limit(DEPLOYMENT_VERSION_PAGE)
|
||||
if (versions.length === 0) break
|
||||
|
||||
for (const version of versions) {
|
||||
const { state: nextState, changed } = rewriteDeploymentVersionState(
|
||||
version.state,
|
||||
resolve
|
||||
)
|
||||
if (!changed) continue
|
||||
await db
|
||||
.update(workflowDeploymentVersion)
|
||||
.set({ state: nextState })
|
||||
.where(eq(workflowDeploymentVersion.id, version.id))
|
||||
// Evict the post-migration deployed state cached by this immutable version id so the
|
||||
// execute/serve path rebuilds from the cleaned snapshot.
|
||||
invalidateDeployedStateCache(version.id)
|
||||
}
|
||||
|
||||
if (versions.length < DEPLOYMENT_VERSION_PAGE) break
|
||||
afterVersion = versions[versions.length - 1].version
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Failed to clear references in deployment versions`, {
|
||||
workflowId,
|
||||
error: getErrorMessage(error),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
hasForkContentToCopy,
|
||||
serializeContentRefMaps,
|
||||
} from '@/ee/workspace-forking/lib/copy/content-copy-runner'
|
||||
import type { BlobCopyTask } from '@/ee/workspace-forking/lib/copy/copy-files'
|
||||
import type { ForkContentPlan } from '@/ee/workspace-forking/lib/copy/copy-resources'
|
||||
|
||||
describe('serializeContentRefMaps', () => {
|
||||
it('converts each map to a record and drops empty maps', () => {
|
||||
const result = serializeContentRefMaps({
|
||||
workspaceId: { from: 'src', to: 'dst' },
|
||||
fileKeys: new Map([['k1', 'k2']]),
|
||||
fileIds: new Map(),
|
||||
workflows: new Map([['wf-src', 'wf-dst']]),
|
||||
knowledgeBases: new Map([['kb-src', 'kb-dst']]),
|
||||
skills: new Map([['sk-src', 'sk-dst']]),
|
||||
})
|
||||
|
||||
expect(result.workspaceId).toEqual({ from: 'src', to: 'dst' })
|
||||
expect(result.fileKeys).toEqual({ k1: 'k2' })
|
||||
// An empty map is omitted rather than serialized to `{}`.
|
||||
expect(result.fileIds).toBeUndefined()
|
||||
expect(result.workflows).toEqual({ 'wf-src': 'wf-dst' })
|
||||
expect(result.knowledgeBases).toEqual({ 'kb-src': 'kb-dst' })
|
||||
expect(result.skills).toEqual({ 'sk-src': 'sk-dst' })
|
||||
// Maps not supplied stay undefined.
|
||||
expect(result.tables).toBeUndefined()
|
||||
expect(result.folders).toBeUndefined()
|
||||
})
|
||||
|
||||
it('passes an undefined workspaceId through unchanged', () => {
|
||||
const result = serializeContentRefMaps({})
|
||||
expect(result.workspaceId).toBeUndefined()
|
||||
expect(result.fileKeys).toBeUndefined()
|
||||
expect(result.skills).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('hasForkContentToCopy', () => {
|
||||
const emptyPlan = (): ForkContentPlan => ({
|
||||
sourceWorkspaceId: 'src',
|
||||
childWorkspaceId: 'child',
|
||||
userId: 'u',
|
||||
tables: [],
|
||||
knowledgeBases: [],
|
||||
skills: [],
|
||||
documents: [],
|
||||
})
|
||||
// The helper only inspects array lengths, so a single placeholder entry per kind is enough.
|
||||
const oneSkill = [{}] as unknown as ForkContentPlan['skills']
|
||||
const oneDoc = [{}] as unknown as ForkContentPlan['documents']
|
||||
const oneTable = [{}] as unknown as ForkContentPlan['tables']
|
||||
const oneKb = [{}] as unknown as ForkContentPlan['knowledgeBases']
|
||||
const oneBlob = [{}] as unknown as BlobCopyTask[]
|
||||
const noBlobs: BlobCopyTask[] = []
|
||||
|
||||
it('is true when skills are non-empty (the create-fork skill-only fix)', () => {
|
||||
expect(hasForkContentToCopy({ ...emptyPlan(), skills: oneSkill }, noBlobs)).toBe(true)
|
||||
})
|
||||
|
||||
it('is true when documents are non-empty', () => {
|
||||
expect(hasForkContentToCopy({ ...emptyPlan(), documents: oneDoc }, noBlobs)).toBe(true)
|
||||
})
|
||||
|
||||
it('is true when tables are non-empty', () => {
|
||||
expect(hasForkContentToCopy({ ...emptyPlan(), tables: oneTable }, noBlobs)).toBe(true)
|
||||
})
|
||||
|
||||
it('is true when knowledgeBases are non-empty', () => {
|
||||
expect(hasForkContentToCopy({ ...emptyPlan(), knowledgeBases: oneKb }, noBlobs)).toBe(true)
|
||||
})
|
||||
|
||||
it('is true when there are blob tasks', () => {
|
||||
expect(hasForkContentToCopy(emptyPlan(), oneBlob)).toBe(true)
|
||||
})
|
||||
|
||||
it('is false for an all-empty plan with no blob tasks', () => {
|
||||
expect(hasForkContentToCopy(emptyPlan(), noBlobs)).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,214 @@
|
||||
import { db } from '@sim/db'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { isTriggerDevEnabled } from '@/lib/core/config/env-flags'
|
||||
import { runDetached } from '@/lib/core/utils/background'
|
||||
import { finishBackgroundWork } from '@/ee/workspace-forking/lib/background-work/store'
|
||||
import { clearFailedForkResourceReferences } from '@/ee/workspace-forking/lib/copy/cleanup-failed'
|
||||
import type { BlobCopyTask } from '@/ee/workspace-forking/lib/copy/copy-files'
|
||||
import { executeForkFileBlobCopies } from '@/ee/workspace-forking/lib/copy/copy-files'
|
||||
import type {
|
||||
ForkContentPlan,
|
||||
ForkFailedResource,
|
||||
} from '@/ee/workspace-forking/lib/copy/copy-resources'
|
||||
import { copyForkResourceContent } from '@/ee/workspace-forking/lib/copy/copy-resources'
|
||||
import type { ForkContentRefMaps } from '@/ee/workspace-forking/lib/remap/remap-content-refs'
|
||||
|
||||
const logger = createLogger('WorkspaceForkContentCopy')
|
||||
|
||||
/**
|
||||
* Whether a fork/sync has any heavy content to copy after the commit: table rows, KB documents,
|
||||
* copied skill bodies, standalone documents, or file blobs. The single gate for scheduling the
|
||||
* background content fill - shared by fork-create and promote so the two can't diverge. A
|
||||
* skill-only (or documents-only) copy must still schedule it, otherwise the in-content `sim:` /
|
||||
* serve-link rewrite in {@link runForkContentCopy} never runs and copied bodies keep source links.
|
||||
*/
|
||||
export function hasForkContentToCopy(
|
||||
contentPlan: ForkContentPlan,
|
||||
blobTasks: BlobCopyTask[]
|
||||
): boolean {
|
||||
return (
|
||||
contentPlan.tables.length > 0 ||
|
||||
contentPlan.knowledgeBases.length > 0 ||
|
||||
contentPlan.skills.length > 0 ||
|
||||
contentPlan.documents.length > 0 ||
|
||||
blobTasks.length > 0
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* JSON-serializable form of {@link ForkContentRefMaps} (Maps become Records) so the in-content
|
||||
* reference maps survive the Trigger.dev payload boundary. Rehydrated to Maps in the runner.
|
||||
*/
|
||||
export interface SerializableForkContentRefMaps {
|
||||
workspaceId?: { from: string; to: string }
|
||||
fileKeys?: Record<string, string>
|
||||
fileIds?: Record<string, string>
|
||||
workflows?: Record<string, string>
|
||||
knowledgeBases?: Record<string, string>
|
||||
tables?: Record<string, string>
|
||||
skills?: Record<string, string>
|
||||
folders?: Record<string, string>
|
||||
}
|
||||
|
||||
/**
|
||||
* Serializable payload for the post-fork heavy-content copy. Runs either as a
|
||||
* Trigger.dev task (`background/fork-content-copy`) or inline via `runDetached`
|
||||
* when Trigger.dev is disabled - both call {@link runForkContentCopy}.
|
||||
*/
|
||||
export interface ForkContentCopyPayload {
|
||||
contentPlan: ForkContentPlan
|
||||
blobTasks: BlobCopyTask[]
|
||||
/** In-content reference maps for rewriting copied markdown blobs (serialized form). */
|
||||
contentRefMaps?: SerializableForkContentRefMaps
|
||||
/**
|
||||
* `background_work_status` row to finish when the copy ends, so the source workspace's
|
||||
* Manage Forks -> Activity entry resolves (completed / warning / error). Started right
|
||||
* after the fork commits so it's visible immediately.
|
||||
*/
|
||||
statusId?: string
|
||||
/**
|
||||
* Target workflow ids this sync deployed (promote's deploy loop). When a copied resource's
|
||||
* fill fails, its dropped placeholder must be cleared from these workflows' DEPLOYED version
|
||||
* states too - a deployed version can reference the placeholder even when the draft no longer
|
||||
* does (edited in the fill window), so the cleanup unions these with the draft-affected set
|
||||
* rather than relying on draft divergence. Empty/omitted for fork-create (child is undeployed).
|
||||
*/
|
||||
deployedTargetWorkflowIds?: string[]
|
||||
requestId?: string
|
||||
}
|
||||
|
||||
const toRefMap = (record?: Record<string, string>): Map<string, string> | undefined =>
|
||||
record ? new Map(Object.entries(record)) : undefined
|
||||
|
||||
const fromRefMap = (map?: ReadonlyMap<string, string>): Record<string, string> | undefined =>
|
||||
map && map.size > 0 ? Object.fromEntries(map) : undefined
|
||||
|
||||
/**
|
||||
* Convert the Map-based {@link ForkContentRefMaps} to its JSON-serializable form for the
|
||||
* Trigger.dev payload. Empty maps are dropped (omitted). Single source of truth for the
|
||||
* Map->Record direction, paired with {@link deserializeContentRefMaps}.
|
||||
*/
|
||||
export function serializeContentRefMaps(maps: ForkContentRefMaps): SerializableForkContentRefMaps {
|
||||
return {
|
||||
workspaceId: maps.workspaceId,
|
||||
fileKeys: fromRefMap(maps.fileKeys),
|
||||
fileIds: fromRefMap(maps.fileIds),
|
||||
workflows: fromRefMap(maps.workflows),
|
||||
knowledgeBases: fromRefMap(maps.knowledgeBases),
|
||||
tables: fromRefMap(maps.tables),
|
||||
skills: fromRefMap(maps.skills),
|
||||
folders: fromRefMap(maps.folders),
|
||||
}
|
||||
}
|
||||
|
||||
/** Rehydrate the serialized content-ref maps to the Map-based {@link ForkContentRefMaps}. */
|
||||
function deserializeContentRefMaps(
|
||||
serialized?: SerializableForkContentRefMaps
|
||||
): ForkContentRefMaps | undefined {
|
||||
if (!serialized) return undefined
|
||||
return {
|
||||
workspaceId: serialized.workspaceId,
|
||||
fileKeys: toRefMap(serialized.fileKeys),
|
||||
fileIds: toRefMap(serialized.fileIds),
|
||||
workflows: toRefMap(serialized.workflows),
|
||||
knowledgeBases: toRefMap(serialized.knowledgeBases),
|
||||
tables: toRefMap(serialized.tables),
|
||||
skills: toRefMap(serialized.skills),
|
||||
folders: toRefMap(serialized.folders),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy the heavy fork content after the fork transaction has committed: table
|
||||
* rows, KB documents + embeddings (keyset-paginated), and file blobs. Best-effort
|
||||
* and idempotency-unsafe (per-row inserts use fresh ids), so it must run at most
|
||||
* once - never blindly retried. Per-resource failures are counted (not thrown), so
|
||||
* the run finishes `completed_with_warnings` rather than failing the whole copy.
|
||||
*/
|
||||
export async function runForkContentCopy(payload: ForkContentCopyPayload): Promise<void> {
|
||||
const { contentPlan, blobTasks, statusId, requestId } = payload
|
||||
try {
|
||||
const contentRefMaps = deserializeContentRefMaps(payload.contentRefMaps)
|
||||
const resourceCounts = await copyForkResourceContent({ contentPlan, contentRefMaps, requestId })
|
||||
const fileCounts = await executeForkFileBlobCopies(blobTasks, requestId, contentRefMaps)
|
||||
// A resource whose content fill failed leaves a dangling reference: a table/KB/doc placeholder
|
||||
// its workflows still point at, or a `file-upload` whose copied blob is missing. Clear those
|
||||
// references (draft + deployed versions) and drop the table/KB/doc placeholder so nothing
|
||||
// dangles; a failed file leaves its metadata row (re-uploadable) but has its refs cleared.
|
||||
const fileFailures: ForkFailedResource[] = fileCounts.failedTargetKeys.map((childKey) => ({
|
||||
kind: 'file',
|
||||
childKey,
|
||||
}))
|
||||
const { cleared, clearingFailed } = await clearFailedForkResourceReferences({
|
||||
childWorkspaceId: contentPlan.childWorkspaceId,
|
||||
failures: [...resourceCounts.failures, ...fileFailures],
|
||||
deployedTargetWorkflowIds: payload.deployedTargetWorkflowIds,
|
||||
requestId,
|
||||
})
|
||||
const copied = resourceCounts.copied + fileCounts.copied
|
||||
const failed = resourceCounts.failed + fileCounts.failed
|
||||
if (statusId) {
|
||||
await finishBackgroundWork(db, statusId, {
|
||||
status: failed > 0 ? 'completed_with_warnings' : 'completed',
|
||||
message:
|
||||
failed > 0
|
||||
? `Copied ${copied} item${copied === 1 ? '' : 's'}; ${failed} could not be copied`
|
||||
: `Copied ${copied} item${copied === 1 ? '' : 's'}`,
|
||||
metadata: {
|
||||
copied,
|
||||
failed,
|
||||
clearedReferences: cleared,
|
||||
...(clearingFailed ? { clearingFailed: true } : {}),
|
||||
},
|
||||
})
|
||||
}
|
||||
} catch (error) {
|
||||
if (statusId) {
|
||||
await finishBackgroundWork(db, statusId, {
|
||||
status: 'failed',
|
||||
error: getErrorMessage(error, 'Background resource copy failed'),
|
||||
}).catch(() => {})
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedule the post-commit heavy-content copy off the request path. Uses the Trigger.dev task
|
||||
* when enabled (so it survives an app deploy), else `runDetached` inline best-effort. Shared by
|
||||
* both fork and sync - only the `detachedLabel` (and so the inline job's name) differs. Never
|
||||
* throws: a scheduling failure is logged and, when a status row exists, marks it failed, so a
|
||||
* committed fork/sync is never turned into a 500 by a background-scheduling hiccup.
|
||||
*/
|
||||
export async function scheduleForkContentCopy(
|
||||
payload: ForkContentCopyPayload,
|
||||
options: { detachedLabel: string; requestId?: string }
|
||||
): Promise<void> {
|
||||
const { detachedLabel, requestId = 'unknown' } = options
|
||||
try {
|
||||
if (isTriggerDevEnabled) {
|
||||
const [{ forkContentCopyTask }, { tasks }, { resolveTriggerRegion }] = await Promise.all([
|
||||
import('@/background/fork-content-copy'),
|
||||
import('@trigger.dev/sdk'),
|
||||
import('@/lib/core/async-jobs/region'),
|
||||
])
|
||||
await tasks.trigger<typeof forkContentCopyTask>('fork-content-copy', payload, {
|
||||
region: await resolveTriggerRegion(),
|
||||
})
|
||||
} else {
|
||||
runDetached(detachedLabel, () => runForkContentCopy(payload))
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Failed to schedule fork content copy`, {
|
||||
detachedLabel,
|
||||
error: getErrorMessage(error),
|
||||
})
|
||||
if (payload.statusId) {
|
||||
await finishBackgroundWork(db, payload.statusId, {
|
||||
status: 'failed',
|
||||
error: getErrorMessage(error, 'Could not start the background copy'),
|
||||
}).catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import type { DbOrTx } from '@/lib/db/types'
|
||||
import { copyForkChatDeployments } from '@/ee/workspace-forking/lib/copy/copy-chats'
|
||||
|
||||
/**
|
||||
* Sequenced select mock: each `.select().from().where()` resolves the next queued result.
|
||||
* The chat copy issues (1) source chats, (2) target chat rows, then per identifier attempt
|
||||
* (3+) the taken-identifier check; inserts are captured.
|
||||
*/
|
||||
function makeTx(selectResults: unknown[][]) {
|
||||
const inserted: Array<Record<string, unknown>> = []
|
||||
let call = 0
|
||||
const tx = {
|
||||
select: () => ({
|
||||
from: () => ({ where: () => Promise.resolve(selectResults[call++] ?? []) }),
|
||||
}),
|
||||
insert: () => ({
|
||||
values: (values: Array<Record<string, unknown>>) => {
|
||||
inserted.push(...values)
|
||||
return Promise.resolve()
|
||||
},
|
||||
}),
|
||||
}
|
||||
return { tx: tx as unknown as DbOrTx, inserted }
|
||||
}
|
||||
|
||||
const sourceChat = (overrides: Record<string, unknown> = {}) => ({
|
||||
id: 'chat-src',
|
||||
workflowId: 'wf-src',
|
||||
userId: 'src-user',
|
||||
identifier: 'original-chat',
|
||||
title: 'Support Chat',
|
||||
description: 'desc',
|
||||
isActive: true,
|
||||
customizations: { welcomeMessage: 'hi' },
|
||||
authType: 'password',
|
||||
password: 'hashed-secret',
|
||||
allowedEmails: ['a@b.com'],
|
||||
outputConfigs: [{ blockId: 'block-src', path: 'content' }],
|
||||
archivedAt: null,
|
||||
createdAt: new Date('2026-01-01'),
|
||||
updatedAt: new Date('2026-01-01'),
|
||||
...overrides,
|
||||
})
|
||||
|
||||
const pair = {
|
||||
sourceWorkflowId: 'wf-src',
|
||||
targetWorkflowId: 'wf-tgt',
|
||||
workflowName: 'Support Flow',
|
||||
}
|
||||
|
||||
describe('copyForkChatDeployments', () => {
|
||||
it('copies a live source chat with a generated identifier and remapped output block ids', async () => {
|
||||
const { tx, inserted } = makeTx([
|
||||
[sourceChat()],
|
||||
[], // target has no chat rows
|
||||
[], // no identifier collisions
|
||||
])
|
||||
const result = await copyForkChatDeployments({
|
||||
tx,
|
||||
pairs: [pair],
|
||||
targetWorkspaceName: 'My Team WS',
|
||||
userId: 'user-1',
|
||||
now: new Date('2026-07-07'),
|
||||
resolveBlockId: (targetWorkflowId, sourceBlockId) =>
|
||||
`${targetWorkflowId}:${sourceBlockId}:mapped`,
|
||||
})
|
||||
|
||||
expect(result.created).toBe(1)
|
||||
expect(inserted).toHaveLength(1)
|
||||
const copy = inserted[0]
|
||||
expect(copy.id).not.toBe('chat-src')
|
||||
expect(copy.workflowId).toBe('wf-tgt')
|
||||
expect(copy.userId).toBe('user-1')
|
||||
// `{workspace}-{workflow}-{randomnum}` in the identifier charset.
|
||||
expect(copy.identifier).toMatch(/^my-team-ws-support-flow-\d{6}$/)
|
||||
// Config copies verbatim - auth (hashed password) included.
|
||||
expect(copy.title).toBe('Support Chat')
|
||||
expect(copy.authType).toBe('password')
|
||||
expect(copy.password).toBe('hashed-secret')
|
||||
expect(copy.allowedEmails).toEqual(['a@b.com'])
|
||||
// Output configs bind to the target's block ids via the sync's block resolver.
|
||||
expect(copy.outputConfigs).toEqual([{ blockId: 'wf-tgt:block-src:mapped', path: 'content' }])
|
||||
expect(copy.archivedAt).toBeNull()
|
||||
})
|
||||
|
||||
it('leaves a target that already has ANY chat row untouched (live or archived - never resurrects)', async () => {
|
||||
const { tx, inserted } = makeTx([
|
||||
[sourceChat()],
|
||||
[{ workflowId: 'wf-tgt' }], // target already has a chat row
|
||||
])
|
||||
const result = await copyForkChatDeployments({
|
||||
tx,
|
||||
pairs: [pair],
|
||||
targetWorkspaceName: 'WS',
|
||||
userId: 'user-1',
|
||||
now: new Date(),
|
||||
resolveBlockId: (_wf, blockId) => blockId,
|
||||
})
|
||||
expect(result.created).toBe(0)
|
||||
expect(inserted).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('never emits duplicate identifiers within a batch (same workspace + workflow slug)', async () => {
|
||||
const twoChats = makeTx([
|
||||
[sourceChat(), sourceChat({ id: 'chat-src-2', identifier: 'other' })],
|
||||
[], // target has no chat rows
|
||||
[], // no live-identifier collisions
|
||||
])
|
||||
const result = await copyForkChatDeployments({
|
||||
tx: twoChats.tx,
|
||||
pairs: [pair],
|
||||
targetWorkspaceName: 'WS',
|
||||
userId: 'user-1',
|
||||
now: new Date(),
|
||||
resolveBlockId: (_wf, blockId) => blockId,
|
||||
})
|
||||
expect(result.created).toBe(2)
|
||||
const identifiers = twoChats.inserted.map((row) => row.identifier)
|
||||
expect(new Set(identifiers).size).toBe(2)
|
||||
})
|
||||
|
||||
it('no-ops with no pairs or no live source chats', async () => {
|
||||
const empty = makeTx([[]])
|
||||
expect(
|
||||
(
|
||||
await copyForkChatDeployments({
|
||||
tx: empty.tx,
|
||||
pairs: [pair],
|
||||
targetWorkspaceName: 'WS',
|
||||
userId: 'user-1',
|
||||
now: new Date(),
|
||||
resolveBlockId: (_wf, blockId) => blockId,
|
||||
})
|
||||
).created
|
||||
).toBe(0)
|
||||
expect(
|
||||
(
|
||||
await copyForkChatDeployments({
|
||||
tx: empty.tx,
|
||||
pairs: [],
|
||||
targetWorkspaceName: 'WS',
|
||||
userId: 'user-1',
|
||||
now: new Date(),
|
||||
resolveBlockId: (_wf, blockId) => blockId,
|
||||
})
|
||||
).created
|
||||
).toBe(0)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,190 @@
|
||||
import { chat } from '@sim/db/schema'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { generateId, generateShortId } from '@sim/utils/id'
|
||||
import { randomInt } from '@sim/utils/random'
|
||||
import { and, inArray, isNull } from 'drizzle-orm'
|
||||
import type { DbOrTx } from '@/lib/db/types'
|
||||
import { isRecord } from '@/lib/workflows/persistence/remap-internal-ids'
|
||||
|
||||
const logger = createLogger('WorkspaceForkCopyChats')
|
||||
|
||||
/** Attempts at a random chat identifier before falling back to a long random suffix. */
|
||||
const IDENTIFIER_ATTEMPTS = 5
|
||||
|
||||
export interface ForkChatCopyPair {
|
||||
sourceWorkflowId: string
|
||||
targetWorkflowId: string
|
||||
/** The target workflow's display name, for the generated chat identifier. */
|
||||
workflowName: string
|
||||
}
|
||||
|
||||
/** Lowercase a display name into the chat identifier charset (`[a-z0-9-]`), bounded. */
|
||||
function slugifyForIdentifier(value: string): string {
|
||||
const slug = value
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '')
|
||||
.slice(0, 24)
|
||||
.replace(/-+$/g, '')
|
||||
return slug || 'chat'
|
||||
}
|
||||
|
||||
/**
|
||||
* `{workspace}-{workflow}-{randomnum}` in the identifier charset. Six digits: concurrent forks
|
||||
* of one parent share the workspace/workflow slugs and can't see each other's uncommitted rows,
|
||||
* so the number is the only collision guard across simultaneous transactions.
|
||||
*/
|
||||
function buildIdentifierCandidate(workspaceSlug: string, workflowName: string): string {
|
||||
const random = randomInt(100000, 1000000)
|
||||
return `${workspaceSlug}-${slugifyForIdentifier(workflowName)}-${random}`
|
||||
}
|
||||
|
||||
/** Remap each output config's `blockId` onto the target workflow's block ids. */
|
||||
function remapChatOutputConfigs(
|
||||
value: unknown,
|
||||
targetWorkflowId: string,
|
||||
resolveBlockId: (targetWorkflowId: string, sourceBlockId: string) => string
|
||||
): unknown {
|
||||
if (!Array.isArray(value)) return value
|
||||
return value.map((entry) => {
|
||||
if (!isRecord(entry) || typeof entry.blockId !== 'string') return entry
|
||||
return { ...entry, blockId: resolveBlockId(targetWorkflowId, entry.blockId) }
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Carry chat deployments onto the target side of a fork or sync: each LIVE source chat whose
|
||||
* target workflow has NO chat row at all (live or archived) is copied with a freshly generated
|
||||
* identifier - `{target-workspace}-{workflow}-{randomnum}` - so the copy serves at its own URL
|
||||
* immediately once the workflow deploys. Config copies verbatim (title, customizations, auth
|
||||
* incl. the hashed password and allowed emails); `outputConfigs` block ids are remapped through
|
||||
* the SAME block-id resolver the workflow write used, so the outputs bind to the target's
|
||||
* blocks.
|
||||
*
|
||||
* Targets with ANY existing chat row are left completely untouched ("maintained"): an already
|
||||
* carried-over chat keeps its identifier and config on every subsequent sync, and a chat the
|
||||
* target side deliberately archived is never resurrected. Bounded by the synced workflow count;
|
||||
* identifiers are collision-checked against live chats and fall back to a long random suffix.
|
||||
*/
|
||||
export async function copyForkChatDeployments(params: {
|
||||
tx: DbOrTx
|
||||
pairs: ForkChatCopyPair[]
|
||||
/** The TARGET workspace's display name, the identifier's first segment. */
|
||||
targetWorkspaceName: string
|
||||
userId: string
|
||||
now: Date
|
||||
resolveBlockId: (targetWorkflowId: string, sourceBlockId: string) => string
|
||||
requestId?: string
|
||||
}): Promise<{ created: number }> {
|
||||
const { tx, pairs, targetWorkspaceName, userId, now, resolveBlockId, requestId } = params
|
||||
if (pairs.length === 0) return { created: 0 }
|
||||
|
||||
const pairBySource = new Map(pairs.map((pair) => [pair.sourceWorkflowId, pair]))
|
||||
const sourceChats = await tx
|
||||
.select()
|
||||
.from(chat)
|
||||
.where(and(inArray(chat.workflowId, [...pairBySource.keys()]), isNull(chat.archivedAt)))
|
||||
if (sourceChats.length === 0) return { created: 0 }
|
||||
|
||||
// A target workflow with ANY chat row (live or archived) keeps it: live means the carry-over
|
||||
// already happened (or the target made its own); archived means the target deliberately
|
||||
// retired it - recreating would resurrect against their intent.
|
||||
const candidateTargetIds = [
|
||||
...new Set(
|
||||
sourceChats
|
||||
.map((row) => pairBySource.get(row.workflowId)?.targetWorkflowId)
|
||||
.filter((id): id is string => Boolean(id))
|
||||
),
|
||||
]
|
||||
const existingTargetRows = await tx
|
||||
.select({ workflowId: chat.workflowId })
|
||||
.from(chat)
|
||||
.where(inArray(chat.workflowId, candidateTargetIds))
|
||||
const targetsWithChat = new Set(existingTargetRows.map((row) => row.workflowId))
|
||||
|
||||
const toCopy = sourceChats.filter((row) => {
|
||||
const pair = pairBySource.get(row.workflowId)
|
||||
return pair && !targetsWithChat.has(pair.targetWorkflowId)
|
||||
})
|
||||
if (toCopy.length === 0) return { created: 0 }
|
||||
|
||||
// Generate identifiers, retrying collisions against LIVE chats (the unique index is partial
|
||||
// on `archived_at IS NULL`, so archived identifiers are reusable) and against this batch.
|
||||
const workspaceSlug = slugifyForIdentifier(targetWorkspaceName)
|
||||
const identifierByChatId = new Map<string, string>()
|
||||
let pending = toCopy.map((row) => ({
|
||||
chatId: row.id,
|
||||
workflowName: pairBySource.get(row.workflowId)?.workflowName ?? 'chat',
|
||||
}))
|
||||
for (let attempt = 0; attempt < IDENTIFIER_ATTEMPTS && pending.length > 0; attempt++) {
|
||||
const claimed = new Set(identifierByChatId.values())
|
||||
const candidates = pending.map((entry) => {
|
||||
let candidate = buildIdentifierCandidate(workspaceSlug, entry.workflowName)
|
||||
while (claimed.has(candidate)) {
|
||||
candidate = buildIdentifierCandidate(workspaceSlug, entry.workflowName)
|
||||
}
|
||||
claimed.add(candidate)
|
||||
return { ...entry, candidate }
|
||||
})
|
||||
const taken = new Set(
|
||||
(
|
||||
await tx
|
||||
.select({ identifier: chat.identifier })
|
||||
.from(chat)
|
||||
.where(
|
||||
and(
|
||||
inArray(
|
||||
chat.identifier,
|
||||
candidates.map((entry) => entry.candidate)
|
||||
),
|
||||
isNull(chat.archivedAt)
|
||||
)
|
||||
)
|
||||
).map((row) => row.identifier)
|
||||
)
|
||||
pending = []
|
||||
for (const entry of candidates) {
|
||||
if (taken.has(entry.candidate)) pending.push(entry)
|
||||
else identifierByChatId.set(entry.chatId, entry.candidate)
|
||||
}
|
||||
}
|
||||
for (const entry of pending) {
|
||||
// Exhausted the friendly attempts: a long random suffix is effectively collision-free
|
||||
// (the global unique index still backstops it).
|
||||
identifierByChatId.set(
|
||||
entry.chatId,
|
||||
`${workspaceSlug}-${slugifyForIdentifier(entry.workflowName)}-${generateShortId(10)
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]/g, '0')}`
|
||||
)
|
||||
}
|
||||
|
||||
const inserts: (typeof chat.$inferInsert)[] = []
|
||||
for (const row of toCopy) {
|
||||
const pair = pairBySource.get(row.workflowId)
|
||||
const identifier = identifierByChatId.get(row.id)
|
||||
if (!pair || !identifier) continue
|
||||
inserts.push({
|
||||
...row,
|
||||
id: generateId(),
|
||||
workflowId: pair.targetWorkflowId,
|
||||
userId,
|
||||
identifier,
|
||||
outputConfigs: remapChatOutputConfigs(
|
||||
row.outputConfigs,
|
||||
pair.targetWorkflowId,
|
||||
resolveBlockId
|
||||
),
|
||||
archivedAt: null,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
}
|
||||
if (inserts.length > 0) {
|
||||
await tx.insert(chat).values(inserts)
|
||||
logger.info(`[${requestId ?? 'unknown'}] Carried ${inserts.length} chat deployment(s)`, {
|
||||
identifiers: inserts.map((row) => row.identifier),
|
||||
})
|
||||
}
|
||||
return { created: inserts.length }
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { storageServiceMock, storageServiceMockFns } from '@sim/testing'
|
||||
import { omit } from '@sim/utils/object'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const { mockIncrementStorageUsage } = vi.hoisted(() => ({
|
||||
mockIncrementStorageUsage: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock)
|
||||
vi.mock('@/lib/billing/storage', () => ({
|
||||
incrementStorageUsage: mockIncrementStorageUsage,
|
||||
}))
|
||||
vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({
|
||||
generateWorkspaceFileKey: vi.fn(
|
||||
(workspaceId: string, fileName: string) => `workspace/${workspaceId}/generated-${fileName}`
|
||||
),
|
||||
}))
|
||||
|
||||
import type { DbOrTx } from '@/lib/db/types'
|
||||
import {
|
||||
type BlobCopyTask,
|
||||
executeForkFileBlobCopies,
|
||||
planForkFileCopies,
|
||||
} from '@/ee/workspace-forking/lib/copy/copy-files'
|
||||
|
||||
function makeTask(overrides: Partial<BlobCopyTask> = {}): BlobCopyTask {
|
||||
return {
|
||||
sourceKey: 'workspace/src-ws/source-a.txt',
|
||||
targetKey: 'workspace/child-ws/target-a.txt',
|
||||
context: 'workspace',
|
||||
fileName: 'a.txt',
|
||||
contentType: 'text/plain',
|
||||
size: 100,
|
||||
userId: 'user-1',
|
||||
workspaceId: 'child-ws',
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
describe('executeForkFileBlobCopies storage accounting', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
storageServiceMockFns.mockHeadObject.mockResolvedValue(null)
|
||||
storageServiceMockFns.mockDownloadFile.mockResolvedValue(Buffer.from('blob-bytes'))
|
||||
storageServiceMockFns.mockUploadFile.mockResolvedValue({ key: 'workspace/child-ws/target' })
|
||||
mockIncrementStorageUsage.mockResolvedValue(undefined)
|
||||
})
|
||||
|
||||
it('charges the initiating user exactly once per landed blob, by the metadata row size', async () => {
|
||||
const tasks = [
|
||||
makeTask({ targetKey: 'workspace/child-ws/t1', size: 100 }),
|
||||
makeTask({ targetKey: 'workspace/child-ws/t2', size: 200, fileName: 'b.txt' }),
|
||||
]
|
||||
|
||||
const result = await executeForkFileBlobCopies(tasks, 'test')
|
||||
|
||||
expect(result).toEqual({ copied: 2, failed: 0, failedTargetKeys: [] })
|
||||
expect(mockIncrementStorageUsage).toHaveBeenCalledTimes(2)
|
||||
expect(mockIncrementStorageUsage).toHaveBeenNthCalledWith(1, 'user-1', 100, 'child-ws')
|
||||
expect(mockIncrementStorageUsage).toHaveBeenNthCalledWith(2, 'user-1', 200, 'child-ws')
|
||||
})
|
||||
|
||||
it('skips an already-existing target blob without re-copying or re-charging (replayed run)', async () => {
|
||||
storageServiceMockFns.mockHeadObject.mockResolvedValue({ size: 100 })
|
||||
|
||||
const result = await executeForkFileBlobCopies([makeTask()], 'test')
|
||||
|
||||
expect(result).toEqual({ copied: 1, failed: 0, failedTargetKeys: [] })
|
||||
expect(storageServiceMockFns.mockDownloadFile).not.toHaveBeenCalled()
|
||||
expect(storageServiceMockFns.mockUploadFile).not.toHaveBeenCalled()
|
||||
expect(mockIncrementStorageUsage).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('never charges a failed copy (the blob did not land)', async () => {
|
||||
storageServiceMockFns.mockDownloadFile.mockRejectedValue(new Error('source gone'))
|
||||
|
||||
const result = await executeForkFileBlobCopies([makeTask()], 'test')
|
||||
|
||||
expect(result).toEqual({
|
||||
copied: 0,
|
||||
failed: 1,
|
||||
failedTargetKeys: ['workspace/child-ws/target-a.txt'],
|
||||
})
|
||||
expect(mockIncrementStorageUsage).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('treats a tracking failure as best-effort - the copy still counts as landed', async () => {
|
||||
mockIncrementStorageUsage.mockRejectedValue(new Error('billing hiccup'))
|
||||
|
||||
const result = await executeForkFileBlobCopies([makeTask()], 'test')
|
||||
|
||||
expect(result).toEqual({ copied: 1, failed: 0, failedTargetKeys: [] })
|
||||
expect(storageServiceMockFns.mockUploadFile).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('skips the charge for a legacy payload enqueued before size existed', async () => {
|
||||
// Simulates a Trigger.dev payload serialized by a pre-`size` deploy (rolling upgrade).
|
||||
const legacyTask = omit(makeTask(), ['size']) as BlobCopyTask
|
||||
|
||||
const result = await executeForkFileBlobCopies([legacyTask], 'test')
|
||||
|
||||
expect(result.copied).toBe(1)
|
||||
expect(mockIncrementStorageUsage).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('planForkFileCopies', () => {
|
||||
it('carries the source metadata size onto each blob task and the child row', async () => {
|
||||
const sourceMeta = {
|
||||
id: 'wf_src1',
|
||||
key: 'workspace/src-ws/1-abc-a.txt',
|
||||
userId: 'uploader-1',
|
||||
workspaceId: 'src-ws',
|
||||
folderId: 'folder-1',
|
||||
context: 'workspace',
|
||||
chatId: null,
|
||||
originalName: 'a.txt',
|
||||
displayName: null,
|
||||
contentType: 'text/plain',
|
||||
size: 4321,
|
||||
deletedAt: null,
|
||||
uploadedAt: new Date('2026-01-01'),
|
||||
updatedAt: new Date('2026-01-01'),
|
||||
}
|
||||
const inserted: Array<Record<string, unknown>> = []
|
||||
const tx = {
|
||||
select: vi.fn(() => ({ from: () => ({ where: () => Promise.resolve([sourceMeta]) }) })),
|
||||
insert: vi.fn(() => ({
|
||||
values: (row: Record<string, unknown>) => {
|
||||
inserted.push(row)
|
||||
return Promise.resolve()
|
||||
},
|
||||
})),
|
||||
} as unknown as DbOrTx
|
||||
|
||||
const result = await planForkFileCopies({
|
||||
tx,
|
||||
sourceWorkspaceId: 'src-ws',
|
||||
childWorkspaceId: 'child-ws',
|
||||
userId: 'user-1',
|
||||
fileIds: ['wf_src1'],
|
||||
now: new Date('2026-02-01'),
|
||||
})
|
||||
|
||||
expect(result.blobTasks).toHaveLength(1)
|
||||
expect(result.blobTasks[0]).toMatchObject({
|
||||
sourceKey: 'workspace/src-ws/1-abc-a.txt',
|
||||
targetKey: 'workspace/child-ws/generated-a.txt',
|
||||
size: 4321,
|
||||
userId: 'user-1',
|
||||
workspaceId: 'child-ws',
|
||||
})
|
||||
expect(inserted[0]).toMatchObject({ size: 4321, workspaceId: 'child-ws' })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,242 @@
|
||||
import { workspaceFiles } from '@sim/db/schema'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { generateId } from '@sim/utils/id'
|
||||
import { and, eq, inArray, isNull, or } from 'drizzle-orm'
|
||||
import { incrementStorageUsage } from '@/lib/billing/storage'
|
||||
import type { DbOrTx } from '@/lib/db/types'
|
||||
import { generateWorkspaceFileKey } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
|
||||
import { downloadFile, headObject, uploadFile } from '@/lib/uploads/core/storage-service'
|
||||
import type { StorageContext } from '@/lib/uploads/shared/types'
|
||||
import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation'
|
||||
import {
|
||||
type ForkContentRefMaps,
|
||||
rewriteForkContentRefs,
|
||||
} from '@/ee/workspace-forking/lib/remap/remap-content-refs'
|
||||
|
||||
const logger = createLogger('WorkspaceForkCopyFiles')
|
||||
|
||||
const MARKDOWN_CONTENT_TYPES = new Set(['text/markdown', 'text/x-markdown'])
|
||||
|
||||
/** Whether a copied blob is markdown text whose in-content references should be rewritten. */
|
||||
function isMarkdownBlob(task: Pick<BlobCopyTask, 'contentType' | 'fileName'>): boolean {
|
||||
if (MARKDOWN_CONTENT_TYPES.has(task.contentType)) return true
|
||||
const name = task.fileName.toLowerCase()
|
||||
return name.endsWith('.md') || name.endsWith('.mdx') || name.endsWith('.markdown')
|
||||
}
|
||||
|
||||
export interface BlobCopyTask {
|
||||
sourceKey: string
|
||||
targetKey: string
|
||||
context: StorageContext
|
||||
fileName: string
|
||||
contentType: string
|
||||
/**
|
||||
* Byte size from the source metadata row - the child `workspace_files` row was inserted
|
||||
* with this same size, so the storage-usage increment after a successful blob copy
|
||||
* charges exactly the bytes the row advertises (matching the upload path, where the
|
||||
* incremented bytes always equal the row's `size`).
|
||||
*/
|
||||
size: number
|
||||
userId: string
|
||||
workspaceId: string
|
||||
}
|
||||
|
||||
export interface PlanForkFileCopiesResult {
|
||||
/**
|
||||
* source storage key -> child storage key. `file-upload` subblocks reference
|
||||
* files by storage key (not `workspace_files.id`), so the fork remap keys on the
|
||||
* storage key. At sync time this map is persisted in the fork resource map
|
||||
* (`resourceType: 'file'`, keyed by storage key) so a re-sync resolves the copy
|
||||
* instead of re-copying; at create-fork time it is not (the child is brand new).
|
||||
*/
|
||||
keyMap: Map<string, string>
|
||||
/**
|
||||
* source `workspace_files.id` -> child id. Used to rewrite in-content file references
|
||||
* that key on the file id (`sim:file/<id>`, `/api/files/view/<id>`, the in-app files
|
||||
* path) inside copied skill/markdown content; not persisted in the fork resource map.
|
||||
*/
|
||||
idMap: Map<string, string>
|
||||
/** Blob duplications to run after the fork transaction commits. */
|
||||
blobTasks: BlobCopyTask[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert child `workspace_files` metadata rows for the selected files (new id +
|
||||
* new storage key) and return the source→child storage-key map plus the blob
|
||||
* copies to run post-commit. The metadata row must exist before the blob upload
|
||||
* (its idempotent metadata insert reuses the row), and both must run after the
|
||||
* child workspace row exists (FK). Runs in the fork transaction; blob I/O is
|
||||
* deferred to {@link executeForkFileBlobCopies}.
|
||||
*
|
||||
* Files are selected EITHER by `workspace_files.id` (the fork modal's picker lists files
|
||||
* by id) OR by storage `key` (sync references key files by their storage key, not id). At
|
||||
* least one of the two must be non-empty; both may be supplied (their matched rows union).
|
||||
*/
|
||||
export async function planForkFileCopies(params: {
|
||||
tx: DbOrTx
|
||||
sourceWorkspaceId: string
|
||||
childWorkspaceId: string
|
||||
userId: string
|
||||
fileIds?: string[]
|
||||
fileKeys?: string[]
|
||||
now: Date
|
||||
}): Promise<PlanForkFileCopiesResult> {
|
||||
const { tx, sourceWorkspaceId, childWorkspaceId, userId, now } = params
|
||||
const fileIds = params.fileIds ?? []
|
||||
const fileKeys = params.fileKeys ?? []
|
||||
const keyMap = new Map<string, string>()
|
||||
const idMap = new Map<string, string>()
|
||||
const blobTasks: BlobCopyTask[] = []
|
||||
if (fileIds.length === 0 && fileKeys.length === 0) return { keyMap, idMap, blobTasks }
|
||||
|
||||
// Match by id and/or storage key (OR'd) so either selection shape resolves to the same
|
||||
// source rows. Batch the metadata read (one query for all selected files): non-deleted,
|
||||
// scoped to the source workspace, and restricted to durable `workspace` files. Only
|
||||
// workspace files are forkable - chat/copilot/mothership uploads are session-scoped and
|
||||
// their chat-bound unique index can't be duplicated - so any non-workspace id/key passed
|
||||
// here is ignored rather than copied.
|
||||
const selectors = [
|
||||
fileIds.length > 0 ? inArray(workspaceFiles.id, fileIds) : undefined,
|
||||
fileKeys.length > 0 ? inArray(workspaceFiles.key, fileKeys) : undefined,
|
||||
].filter((clause): clause is NonNullable<typeof clause> => clause !== undefined)
|
||||
const metas = await tx
|
||||
.select()
|
||||
.from(workspaceFiles)
|
||||
.where(
|
||||
and(
|
||||
selectors.length === 1 ? selectors[0] : or(...selectors),
|
||||
eq(workspaceFiles.workspaceId, sourceWorkspaceId),
|
||||
eq(workspaceFiles.context, 'workspace'),
|
||||
isNull(workspaceFiles.deletedAt)
|
||||
)
|
||||
)
|
||||
|
||||
for (const meta of metas) {
|
||||
const childFileId = generateId()
|
||||
// Use the canonical workspace-file key (`workspace/{id}/...`) so the file-serve
|
||||
// API can infer the storage context; a bare `{id}/...` key has no context prefix.
|
||||
const targetKey = generateWorkspaceFileKey(childWorkspaceId, meta.originalName)
|
||||
await tx.insert(workspaceFiles).values({
|
||||
...meta,
|
||||
id: childFileId,
|
||||
key: targetKey,
|
||||
workspaceId: childWorkspaceId,
|
||||
userId,
|
||||
folderId: null,
|
||||
deletedAt: null,
|
||||
uploadedAt: now,
|
||||
})
|
||||
keyMap.set(meta.key, targetKey)
|
||||
idMap.set(meta.id, childFileId)
|
||||
blobTasks.push({
|
||||
sourceKey: meta.key,
|
||||
targetKey,
|
||||
context: meta.context as StorageContext,
|
||||
fileName: meta.originalName,
|
||||
contentType: meta.contentType,
|
||||
size: meta.size,
|
||||
userId,
|
||||
workspaceId: childWorkspaceId,
|
||||
})
|
||||
}
|
||||
|
||||
return { keyMap, idMap, blobTasks }
|
||||
}
|
||||
|
||||
/**
|
||||
* Duplicate each planned file blob to its new key. `uploadFile`'s metadata insert
|
||||
* is idempotent on the key (the row was already created in the transaction), so
|
||||
* this only copies bytes. Markdown blobs additionally have their in-content references
|
||||
* (`sim:` links, embedded file/image URLs) rewritten through `contentRefMaps` so they
|
||||
* point at the copied resources (unmapped targets are left as graceful broken links).
|
||||
* Best-effort: a content-rewrite failure falls back to copying the raw bytes. A failed
|
||||
* blob's child storage key is returned in `failedTargetKeys` so the caller can clear the
|
||||
* `file-upload` references pointing at the now-missing object (the metadata row is left in
|
||||
* place, so the user can still re-upload the blob).
|
||||
*
|
||||
* Storage accounting: each blob that actually lands increments the initiating user's
|
||||
* storage usage by the metadata row's size - the copied bytes are charged exactly as if
|
||||
* the file had been uploaded to the target workspace. The increment cannot double-count:
|
||||
* the content-copy job is at-most-once by config (`maxAttempts: 1`), each task increments
|
||||
* only after its own successful upload, and the target-existence skip below means a
|
||||
* manually replayed run neither re-copies nor re-charges a blob a prior attempt landed.
|
||||
* Like the upload path, a tracking failure is logged and never fails the copy - and is
|
||||
* never retried, so a landed blob whose increment failed stays uncounted (a manual replay
|
||||
* skips it without charging). Accepted trade-off, matching the platform's upload paths:
|
||||
* storage may undercount, but a user is never charged twice or for bytes that didn't land.
|
||||
*/
|
||||
export async function executeForkFileBlobCopies(
|
||||
blobTasks: BlobCopyTask[],
|
||||
requestId = 'unknown',
|
||||
contentRefMaps?: ForkContentRefMaps
|
||||
): Promise<{ copied: number; failed: number; failedTargetKeys: string[] }> {
|
||||
let copied = 0
|
||||
const failedTargetKeys: string[] = []
|
||||
for (const task of blobTasks) {
|
||||
try {
|
||||
// Replay guard: target keys are freshly generated per fork/sync, so an existing
|
||||
// object can only mean an earlier attempt already landed this exact copy. Skip
|
||||
// without incrementing - a replay must never double-charge, so if the prior
|
||||
// attempt's best-effort increment failed those bytes stay uncounted (the same
|
||||
// accepted undercount as a tracking failure on the upload path). `headObject`
|
||||
// returns null on local storage, where the copy is simply repeated (same bytes
|
||||
// to the same key).
|
||||
const existing = await headObject(task.targetKey, task.context)
|
||||
if (existing) {
|
||||
copied += 1
|
||||
continue
|
||||
}
|
||||
const buffer = await downloadFile({
|
||||
key: task.sourceKey,
|
||||
context: task.context,
|
||||
maxBytes: MAX_FILE_SIZE,
|
||||
})
|
||||
let body: Buffer = buffer
|
||||
if (contentRefMaps && isMarkdownBlob(task)) {
|
||||
try {
|
||||
const text = buffer.toString('utf8')
|
||||
const rewritten = rewriteForkContentRefs(text, contentRefMaps)
|
||||
if (rewritten !== text) body = Buffer.from(rewritten, 'utf8')
|
||||
} catch (error) {
|
||||
logger.warn(`[${requestId}] Failed to rewrite markdown blob content; copying raw bytes`, {
|
||||
targetKey: task.targetKey,
|
||||
error: getErrorMessage(error),
|
||||
})
|
||||
}
|
||||
}
|
||||
await uploadFile({
|
||||
file: body,
|
||||
fileName: task.fileName,
|
||||
contentType: task.contentType,
|
||||
context: task.context,
|
||||
customKey: task.targetKey,
|
||||
preserveKey: true,
|
||||
metadata: {
|
||||
userId: task.userId,
|
||||
workspaceId: task.workspaceId,
|
||||
originalName: task.fileName,
|
||||
},
|
||||
})
|
||||
copied += 1
|
||||
// The typeof guard covers payloads enqueued before `size` existed (rolling deploy).
|
||||
if (typeof task.size === 'number' && task.size > 0) {
|
||||
try {
|
||||
await incrementStorageUsage(task.userId, task.size, task.workspaceId)
|
||||
} catch (storageError) {
|
||||
logger.error(`[${requestId}] Failed to update storage tracking for copied file blob`, {
|
||||
targetKey: task.targetKey,
|
||||
error: getErrorMessage(storageError),
|
||||
})
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
failedTargetKeys.push(task.targetKey)
|
||||
logger.warn(`[${requestId}] Failed to copy file blob during fork`, {
|
||||
targetKey: task.targetKey,
|
||||
error: getErrorMessage(error),
|
||||
})
|
||||
}
|
||||
}
|
||||
return { copied, failed: failedTargetKeys.length, failedTargetKeys }
|
||||
}
|
||||
@@ -0,0 +1,666 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import {
|
||||
dbChainMock,
|
||||
dbChainMockFns,
|
||||
resetDbChainMock,
|
||||
storageServiceMock,
|
||||
storageServiceMockFns,
|
||||
} from '@sim/testing'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const { mockIncrementStorageUsage } = vi.hoisted(() => ({
|
||||
mockIncrementStorageUsage: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@sim/db', () => dbChainMock)
|
||||
vi.mock('@/lib/uploads/core/storage-service', () => storageServiceMock)
|
||||
vi.mock('@/lib/billing/storage', () => ({
|
||||
incrementStorageUsage: mockIncrementStorageUsage,
|
||||
}))
|
||||
|
||||
import type { DbOrTx } from '@/lib/db/types'
|
||||
import {
|
||||
copyForkResourceContainers,
|
||||
copyForkResourceContent,
|
||||
type ForkContentPlan,
|
||||
planForkMappedKbDocumentCopies,
|
||||
} from '@/ee/workspace-forking/lib/copy/copy-resources'
|
||||
import type { ForkReferenceResolver } from '@/ee/workspace-forking/lib/remap/remap-references'
|
||||
|
||||
function basePlan(overrides: Partial<ForkContentPlan> = {}): ForkContentPlan {
|
||||
return {
|
||||
sourceWorkspaceId: 'src-ws',
|
||||
childWorkspaceId: 'child-ws',
|
||||
userId: 'user-1',
|
||||
tables: [],
|
||||
knowledgeBases: [],
|
||||
skills: [],
|
||||
documents: [],
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
const sourceDoc = {
|
||||
id: 'doc-1',
|
||||
knowledgeBaseId: 'src-kb',
|
||||
storageKey: 'kb/source-key',
|
||||
fileUrl: '/api/files/serve/kb%2Fsource-key',
|
||||
filename: 'report.pdf',
|
||||
mimeType: 'application/pdf',
|
||||
}
|
||||
|
||||
describe('copyForkResourceContent', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
resetDbChainMock()
|
||||
storageServiceMockFns.mockDownloadFile.mockResolvedValue(Buffer.from('blob-bytes'))
|
||||
storageServiceMockFns.mockUploadFile.mockResolvedValue({
|
||||
key: 'kb/child-key',
|
||||
path: '/api/files/serve/kb/child-key',
|
||||
})
|
||||
})
|
||||
|
||||
it('rewrites in-workspace resource URLs nested in copied table cell data', async () => {
|
||||
dbChainMockFns.limit.mockResolvedValueOnce([
|
||||
{
|
||||
id: 'r1',
|
||||
tableId: 'src-tbl',
|
||||
workspaceId: 'src-ws',
|
||||
data: {
|
||||
kb: '/workspace/src-ws/knowledge/kb-1',
|
||||
nested: { wf: '/workspace/src-ws/w/wf-1' },
|
||||
plain: 'no url here',
|
||||
},
|
||||
},
|
||||
])
|
||||
|
||||
const result = await copyForkResourceContent({
|
||||
contentPlan: basePlan({ tables: [{ sourceId: 'src-tbl', childId: 'child-tbl' }] }),
|
||||
contentRefMaps: {
|
||||
workspaceId: { from: 'src-ws', to: 'child-ws' },
|
||||
knowledgeBases: new Map([['kb-1', 'kb-2']]),
|
||||
workflows: new Map([['wf-1', 'wf-2']]),
|
||||
},
|
||||
requestId: 'test',
|
||||
})
|
||||
|
||||
expect(result.failed).toBe(0)
|
||||
// The first insert is the table-rows copy (no KBs/docs/skills in this plan).
|
||||
const inserted = dbChainMockFns.values.mock.calls[0][0] as Array<{
|
||||
data: { kb: string; nested: { wf: string }; plain: string }
|
||||
}>
|
||||
expect(inserted[0].data.kb).toBe('/workspace/child-ws/knowledge/kb-2')
|
||||
expect(inserted[0].data.nested.wf).toBe('/workspace/child-ws/w/wf-2')
|
||||
expect(inserted[0].data.plain).toBe('no url here')
|
||||
})
|
||||
|
||||
it('#1 binds a copied KB document blob to the CHILD workspace + initiating user', async () => {
|
||||
// One live document page, then the embeddings page resolves empty (default).
|
||||
dbChainMockFns.limit.mockResolvedValueOnce([sourceDoc])
|
||||
|
||||
const result = await copyForkResourceContent({
|
||||
contentPlan: basePlan({
|
||||
knowledgeBases: [{ sourceId: 'src-kb', childId: 'child-kb', documentIdMap: {} }],
|
||||
}),
|
||||
requestId: 'test',
|
||||
})
|
||||
|
||||
expect(result.failed).toBe(0)
|
||||
expect(result.copied).toBe(1)
|
||||
expect(storageServiceMockFns.mockUploadFile).toHaveBeenCalledTimes(1)
|
||||
const uploadArg = storageServiceMockFns.mockUploadFile.mock.calls[0][0]
|
||||
expect(uploadArg.context).toBe('knowledge-base')
|
||||
expect(uploadArg.preserveKey).toBe(true)
|
||||
// The ownership binding is what verifyKBFileAccess resolves the owning workspace from;
|
||||
// it must name the CHILD workspace and the initiating user, or the copy is download-denied.
|
||||
expect(uploadArg.metadata).toEqual({
|
||||
userId: 'user-1',
|
||||
workspaceId: 'child-ws',
|
||||
originalName: 'report.pdf',
|
||||
})
|
||||
})
|
||||
|
||||
it('#1 never touches storage accounting for copied KB document blobs (mirrors the KB upload path)', async () => {
|
||||
// The normal KB upload path never increments `storage_used_bytes` (and embeddings are
|
||||
// uncounted DB rows), so a fork-copied KB blob must not be charged either - copied KB
|
||||
// bytes are only headroom-checked pre-fork, exactly like the multipart-initiate check.
|
||||
dbChainMockFns.limit.mockResolvedValueOnce([sourceDoc])
|
||||
|
||||
const result = await copyForkResourceContent({
|
||||
contentPlan: basePlan({
|
||||
knowledgeBases: [{ sourceId: 'src-kb', childId: 'child-kb', documentIdMap: {} }],
|
||||
}),
|
||||
requestId: 'test',
|
||||
})
|
||||
|
||||
expect(result.copied).toBe(1)
|
||||
expect(storageServiceMockFns.mockUploadFile).toHaveBeenCalledTimes(1)
|
||||
expect(mockIncrementStorageUsage).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('#4 re-reads a copied skill body post-commit and rewrites it via db.update (never from payload)', async () => {
|
||||
// The body is no longer carried in the plan - the content phase keyset-re-reads the child row.
|
||||
dbChainMockFns.limit.mockResolvedValueOnce([
|
||||
{ id: 'child-skill-1', content: 'see [K](sim:knowledge/src-kb)' },
|
||||
])
|
||||
|
||||
const result = await copyForkResourceContent({
|
||||
contentPlan: basePlan({ skills: [{ childId: 'child-skill-1' }] }),
|
||||
contentRefMaps: { knowledgeBases: new Map([['src-kb', 'child-kb']]) },
|
||||
requestId: 'test',
|
||||
})
|
||||
|
||||
expect(result.failed).toBe(0)
|
||||
expect(dbChainMockFns.update).toHaveBeenCalledTimes(1)
|
||||
expect(dbChainMockFns.set).toHaveBeenCalledWith({
|
||||
content: 'see [K](sim:knowledge/child-kb)',
|
||||
})
|
||||
})
|
||||
|
||||
it('#4 leaves a skill untouched when nothing in its re-read body remaps', async () => {
|
||||
dbChainMockFns.limit.mockResolvedValueOnce([
|
||||
{ id: 'child-skill-1', content: 'no references here' },
|
||||
])
|
||||
|
||||
const result = await copyForkResourceContent({
|
||||
contentPlan: basePlan({ skills: [{ childId: 'child-skill-1' }] }),
|
||||
contentRefMaps: { knowledgeBases: new Map([['src-kb', 'child-kb']]) },
|
||||
requestId: 'test',
|
||||
})
|
||||
|
||||
expect(result.failed).toBe(0)
|
||||
expect(dbChainMockFns.update).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('#4 skips the skill re-read + rewrite entirely when no content maps are supplied', async () => {
|
||||
await copyForkResourceContent({
|
||||
contentPlan: basePlan({ skills: [{ childId: 'child-skill-1' }] }),
|
||||
requestId: 'test',
|
||||
})
|
||||
|
||||
// No maps -> the body is neither re-read from the DB nor updated.
|
||||
expect(dbChainMockFns.select).not.toHaveBeenCalled()
|
||||
expect(dbChainMockFns.update).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('#3 fails the whole KB (all-or-nothing) when one document copy throws', async () => {
|
||||
dbChainMockFns.limit.mockResolvedValueOnce([sourceDoc])
|
||||
// The document row insert throws; the blob copy is best-effort (never throws) so the
|
||||
// failure must come from the persisted copy, marking the entire KB failed for cleanup.
|
||||
dbChainMockFns.values.mockImplementationOnce(() => {
|
||||
throw new Error('insert failed')
|
||||
})
|
||||
|
||||
const result = await copyForkResourceContent({
|
||||
contentPlan: basePlan({
|
||||
knowledgeBases: [{ sourceId: 'src-kb', childId: 'child-kb', documentIdMap: {} }],
|
||||
}),
|
||||
requestId: 'test',
|
||||
})
|
||||
|
||||
expect(result.copied).toBe(0)
|
||||
expect(result.failed).toBe(1)
|
||||
expect(result.failures).toEqual([
|
||||
{ kind: 'knowledge-base', childId: 'child-kb', documentChildIds: [] },
|
||||
])
|
||||
})
|
||||
|
||||
it('U-docs: fills a document copied into an existing target KB (blob re-key + placeholder update)', async () => {
|
||||
const result = await copyForkResourceContent({
|
||||
contentPlan: basePlan({
|
||||
documents: [
|
||||
{
|
||||
sourceDocId: 'doc-1',
|
||||
childDocId: 'child-doc-1',
|
||||
childKnowledgeBaseId: 'existing-target-kb',
|
||||
storageKey: 'kb/source-key',
|
||||
filename: 'report.pdf',
|
||||
mimeType: 'application/pdf',
|
||||
},
|
||||
],
|
||||
}),
|
||||
requestId: 'test',
|
||||
})
|
||||
|
||||
expect(result.failed).toBe(0)
|
||||
expect(result.copied).toBe(1)
|
||||
// The blob is re-keyed and the pre-created placeholder row's blob fields are updated.
|
||||
expect(storageServiceMockFns.mockUploadFile).toHaveBeenCalledTimes(1)
|
||||
expect(dbChainMockFns.update).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('U-docs: a failed document fill is reported as a knowledge-document failure (for cleanup)', async () => {
|
||||
// The placeholder blob update throws; the doc fails on its own without touching its KB.
|
||||
dbChainMockFns.set.mockImplementationOnce(() => {
|
||||
throw new Error('update failed')
|
||||
})
|
||||
|
||||
const result = await copyForkResourceContent({
|
||||
contentPlan: basePlan({
|
||||
documents: [
|
||||
{
|
||||
sourceDocId: 'doc-1',
|
||||
childDocId: 'child-doc-1',
|
||||
childKnowledgeBaseId: 'existing-target-kb',
|
||||
storageKey: 'kb/source-key',
|
||||
filename: 'report.pdf',
|
||||
mimeType: 'application/pdf',
|
||||
},
|
||||
],
|
||||
}),
|
||||
requestId: 'test',
|
||||
})
|
||||
|
||||
expect(result.copied).toBe(0)
|
||||
expect(result.failed).toBe(1)
|
||||
expect(result.failures).toEqual([{ kind: 'knowledge-document', childId: 'child-doc-1' }])
|
||||
})
|
||||
})
|
||||
|
||||
describe('copyForkResourceContainers custom-tool code env rewrite', () => {
|
||||
function makeContainerTx(rows: Array<Record<string, unknown>>) {
|
||||
const inserted: Array<Record<string, unknown>> = []
|
||||
const tx = {
|
||||
select: () => ({ from: () => ({ where: () => Promise.resolve(rows) }) }),
|
||||
insert: () => ({
|
||||
values: (values: Array<Record<string, unknown>>) => {
|
||||
inserted.push(...values)
|
||||
return Promise.resolve()
|
||||
},
|
||||
}),
|
||||
}
|
||||
return { tx: tx as unknown as DbOrTx, inserted }
|
||||
}
|
||||
|
||||
const customToolSelection = {
|
||||
customTools: ['ct-1'],
|
||||
skills: [],
|
||||
mcpServers: [],
|
||||
workflowMcpServers: [],
|
||||
tables: [],
|
||||
knowledgeBases: [],
|
||||
}
|
||||
|
||||
it('rewrites {{ENV}} refs in copied custom-tool code when a sync renames the env var', async () => {
|
||||
const { tx, inserted } = makeContainerTx([
|
||||
{ id: 'ct-1', title: 'Tool', code: 'fetch("{{SLACK_API_KEY}}", "{{KEEP}}")' },
|
||||
])
|
||||
await copyForkResourceContainers({
|
||||
tx,
|
||||
sourceWorkspaceId: 'src-ws',
|
||||
childWorkspaceId: 'child-ws',
|
||||
userId: 'user-1',
|
||||
now: new Date(),
|
||||
selection: customToolSelection,
|
||||
workflowIdMap: new Map(),
|
||||
resolveEnvName: (key) => (key === 'SLACK_API_KEY' ? 'SLACK_API_KEY_TEST' : key),
|
||||
})
|
||||
expect(inserted).toHaveLength(1)
|
||||
// The renamed key is rewritten; the same-name key is left verbatim.
|
||||
expect(inserted[0].code).toBe('fetch("{{SLACK_API_KEY_TEST}}", "{{KEEP}}")')
|
||||
expect(inserted[0].workspaceId).toBe('child-ws')
|
||||
})
|
||||
|
||||
it('preserves custom-tool code verbatim when no env resolver is provided (fork-create)', async () => {
|
||||
const { tx, inserted } = makeContainerTx([
|
||||
{ id: 'ct-1', title: 'Tool', code: 'fetch("{{SLACK_API_KEY}}")' },
|
||||
])
|
||||
await copyForkResourceContainers({
|
||||
tx,
|
||||
sourceWorkspaceId: 'src-ws',
|
||||
childWorkspaceId: 'child-ws',
|
||||
userId: 'user-1',
|
||||
now: new Date(),
|
||||
selection: customToolSelection,
|
||||
workflowIdMap: new Map(),
|
||||
})
|
||||
expect(inserted[0].code).toBe('fetch("{{SLACK_API_KEY}}")')
|
||||
})
|
||||
})
|
||||
|
||||
describe('copyForkResourceContainers external MCP server copy', () => {
|
||||
function makeServerTx(rows: Array<Record<string, unknown>>) {
|
||||
const inserted: Array<Record<string, unknown>> = []
|
||||
const tx = {
|
||||
select: () => ({ from: () => ({ where: () => Promise.resolve(rows) }) }),
|
||||
insert: () => ({
|
||||
values: (values: Array<Record<string, unknown>>) => {
|
||||
inserted.push(...values)
|
||||
return Promise.resolve()
|
||||
},
|
||||
}),
|
||||
}
|
||||
return { tx: tx as unknown as DbOrTx, inserted }
|
||||
}
|
||||
|
||||
it('copies the config row with runtime status reset, records the mapping, and never copies tokens', async () => {
|
||||
const { tx, inserted } = makeServerTx([
|
||||
{
|
||||
id: 'mcp-1',
|
||||
workspaceId: 'src-ws',
|
||||
createdBy: 'src-user',
|
||||
name: 'Linear MCP',
|
||||
transport: 'streamable-http',
|
||||
url: 'https://mcp.linear.app/mcp',
|
||||
authType: 'headers',
|
||||
headers: { Authorization: 'Bearer {{LINEAR_KEY}}' },
|
||||
connectionStatus: 'connected',
|
||||
lastConnected: new Date(),
|
||||
lastError: 'old error',
|
||||
statusConfig: { consecutiveFailures: 2, lastSuccessfulDiscovery: 'x' },
|
||||
toolCount: 12,
|
||||
lastToolsRefresh: new Date(),
|
||||
totalRequests: 99,
|
||||
lastUsed: new Date(),
|
||||
deletedAt: null,
|
||||
},
|
||||
])
|
||||
|
||||
const result = await copyForkResourceContainers({
|
||||
tx,
|
||||
sourceWorkspaceId: 'src-ws',
|
||||
childWorkspaceId: 'child-ws',
|
||||
userId: 'user-1',
|
||||
now: new Date(),
|
||||
selection: {
|
||||
customTools: [],
|
||||
skills: [],
|
||||
mcpServers: ['mcp-1'],
|
||||
workflowMcpServers: [],
|
||||
tables: [],
|
||||
knowledgeBases: [],
|
||||
},
|
||||
workflowIdMap: new Map(),
|
||||
})
|
||||
|
||||
expect(inserted).toHaveLength(1)
|
||||
const child = inserted[0]
|
||||
expect(child.id).not.toBe('mcp-1')
|
||||
expect(child.workspaceId).toBe('child-ws')
|
||||
expect(child.createdBy).toBe('user-1')
|
||||
// Config copies verbatim - url/headers ({{ENV}} refs resolve against the child's env).
|
||||
expect(child.url).toBe('https://mcp.linear.app/mcp')
|
||||
expect(child.headers).toEqual({ Authorization: 'Bearer {{LINEAR_KEY}}' })
|
||||
// Runtime status resets: tools re-discover on first use in the child (cache is
|
||||
// workspace-keyed), and no `mcp_server_oauth` row is ever inserted (re-auth required).
|
||||
expect(child.connectionStatus).toBe('disconnected')
|
||||
expect(child.lastConnected).toBeNull()
|
||||
expect(child.lastError).toBeNull()
|
||||
expect(child.toolCount).toBe(0)
|
||||
expect(child.lastToolsRefresh).toBeNull()
|
||||
// The id map + mapping rows record the copy so subblock references remap onto it.
|
||||
expect(result.idMap.get('mcp_server')?.get('mcp-1')).toBe(child.id)
|
||||
expect(result.mappingEntries).toContainEqual({
|
||||
resourceType: 'mcp_server',
|
||||
parentResourceId: 'mcp-1',
|
||||
childResourceId: child.id,
|
||||
})
|
||||
expect(result.names.mcpServers).toEqual(['Linear MCP'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('copyForkResourceContainers skill copy', () => {
|
||||
function makeSkillTx(rows: Array<Record<string, unknown>>) {
|
||||
const inserted: Array<Record<string, unknown>> = []
|
||||
const tx = {
|
||||
select: () => ({ from: () => ({ where: () => Promise.resolve(rows) }) }),
|
||||
insert: () => ({
|
||||
values: (values: Array<Record<string, unknown>>) => {
|
||||
inserted.push(...values)
|
||||
return Promise.resolve()
|
||||
},
|
||||
}),
|
||||
}
|
||||
return { tx: tx as unknown as DbOrTx, inserted }
|
||||
}
|
||||
|
||||
const skillSelection = {
|
||||
customTools: [],
|
||||
skills: ['sk-1'],
|
||||
mcpServers: [],
|
||||
workflowMcpServers: [],
|
||||
tables: [],
|
||||
knowledgeBases: [],
|
||||
}
|
||||
|
||||
it('copies the skill body IN-DB and carries only the child id in the content plan', async () => {
|
||||
// The source projection deliberately omits `content` (it is copied server-side), so the row
|
||||
// fed to the tx mock has none - the body must never be materialized in app memory here.
|
||||
const { tx, inserted } = makeSkillTx([
|
||||
{
|
||||
id: 'sk-1',
|
||||
name: 'My Skill',
|
||||
description: 'desc',
|
||||
workspaceId: 'src-ws',
|
||||
userId: 'src-user',
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
])
|
||||
|
||||
const result = await copyForkResourceContainers({
|
||||
tx,
|
||||
sourceWorkspaceId: 'src-ws',
|
||||
childWorkspaceId: 'child-ws',
|
||||
userId: 'user-1',
|
||||
now: new Date(),
|
||||
selection: skillSelection,
|
||||
workflowIdMap: new Map(),
|
||||
})
|
||||
|
||||
expect(inserted).toHaveLength(1)
|
||||
const childId = inserted[0].id as string
|
||||
expect(childId).not.toBe('sk-1')
|
||||
expect(inserted[0].workspaceId).toBe('child-ws')
|
||||
expect(inserted[0].userId).toBe('user-1')
|
||||
// The body is deferred to a correlated subquery (in-DB copy), never a materialized string.
|
||||
expect(typeof inserted[0].content).not.toBe('string')
|
||||
// The content plan carries ONLY the child id - no skill body text crosses the job payload.
|
||||
expect(result.contentPlan.skills).toEqual([{ childId }])
|
||||
expect(result.names.skills).toEqual(['My Skill'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('copyForkResourceContainers knowledge-base tag definitions', () => {
|
||||
/** Sequential tx mock: each select resolves the next queued row set; inserts are captured per call. */
|
||||
function makeKbTx(selects: Array<Array<Record<string, unknown>>>) {
|
||||
let call = 0
|
||||
const inserts: Array<Array<Record<string, unknown>>> = []
|
||||
const tx = {
|
||||
select: () => ({
|
||||
from: () => ({ where: () => Promise.resolve(selects[call++] ?? []) }),
|
||||
}),
|
||||
insert: () => ({
|
||||
values: (rows: Array<Record<string, unknown>>) => {
|
||||
inserts.push(rows)
|
||||
return Promise.resolve()
|
||||
},
|
||||
}),
|
||||
}
|
||||
return { tx: tx as unknown as DbOrTx, inserts }
|
||||
}
|
||||
|
||||
const kbSelection = {
|
||||
customTools: [],
|
||||
skills: [],
|
||||
mcpServers: [],
|
||||
workflowMcpServers: [],
|
||||
tables: [],
|
||||
knowledgeBases: ['kb-1'],
|
||||
}
|
||||
|
||||
const sourceBase = { id: 'kb-1', name: 'Docs KB', workspaceId: 'src-ws', deletedAt: null }
|
||||
|
||||
it('copies the source KB tag definitions to the child KB with fresh ids (other columns verbatim)', async () => {
|
||||
const { tx, inserts } = makeKbTx([
|
||||
[sourceBase],
|
||||
[
|
||||
{
|
||||
id: 'tag-1',
|
||||
knowledgeBaseId: 'kb-1',
|
||||
tagSlot: 'tag1',
|
||||
displayName: 'Category',
|
||||
fieldType: 'text',
|
||||
},
|
||||
{
|
||||
id: 'tag-2',
|
||||
knowledgeBaseId: 'kb-1',
|
||||
tagSlot: 'boolean1',
|
||||
displayName: 'Reviewed',
|
||||
fieldType: 'boolean',
|
||||
},
|
||||
],
|
||||
])
|
||||
|
||||
const result = await copyForkResourceContainers({
|
||||
tx,
|
||||
sourceWorkspaceId: 'src-ws',
|
||||
childWorkspaceId: 'child-ws',
|
||||
userId: 'user-1',
|
||||
now: new Date(),
|
||||
selection: kbSelection,
|
||||
workflowIdMap: new Map(),
|
||||
})
|
||||
|
||||
const childKbId = result.idMap.get('knowledge_base')?.get('kb-1')
|
||||
expect(childKbId).toBeTruthy()
|
||||
// insert #0 is the KB row; insert #1 is the tag-definition batch.
|
||||
expect(inserts).toHaveLength(2)
|
||||
const tagRows = inserts[1]
|
||||
expect(tagRows).toHaveLength(2)
|
||||
for (const row of tagRows) {
|
||||
expect(row.knowledgeBaseId).toBe(childKbId)
|
||||
expect(row.id).not.toBe('tag-1')
|
||||
expect(row.id).not.toBe('tag-2')
|
||||
}
|
||||
expect(tagRows.map((row) => [row.tagSlot, row.displayName, row.fieldType])).toEqual([
|
||||
['tag1', 'Category', 'text'],
|
||||
['boolean1', 'Reviewed', 'boolean'],
|
||||
])
|
||||
})
|
||||
|
||||
it('no-ops the tag-definition copy for a KB with zero definitions', async () => {
|
||||
const { tx, inserts } = makeKbTx([[sourceBase], []])
|
||||
|
||||
await copyForkResourceContainers({
|
||||
tx,
|
||||
sourceWorkspaceId: 'src-ws',
|
||||
childWorkspaceId: 'child-ws',
|
||||
userId: 'user-1',
|
||||
now: new Date(),
|
||||
selection: kbSelection,
|
||||
workflowIdMap: new Map(),
|
||||
})
|
||||
|
||||
// Only the KB row itself is inserted - no empty tag-definition insert.
|
||||
expect(inserts).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('planForkMappedKbDocumentCopies', () => {
|
||||
const sourceRow = (id: string, knowledgeBaseId: string) => ({
|
||||
id,
|
||||
knowledgeBaseId,
|
||||
storageKey: `kb/${id}`,
|
||||
filename: `${id}.pdf`,
|
||||
mimeType: 'application/pdf',
|
||||
connectorId: 'connector-1',
|
||||
deletedAt: null,
|
||||
archivedAt: null,
|
||||
})
|
||||
|
||||
function makeTx(docs: ReturnType<typeof sourceRow>[]) {
|
||||
const inserted: Array<Record<string, unknown>> = []
|
||||
let selectCalled = false
|
||||
const tx = {
|
||||
select: () => {
|
||||
selectCalled = true
|
||||
return { from: () => ({ where: () => Promise.resolve(docs) }) }
|
||||
},
|
||||
insert: () => ({
|
||||
values: (rows: Array<Record<string, unknown>>) => {
|
||||
inserted.push(...rows)
|
||||
return Promise.resolve()
|
||||
},
|
||||
}),
|
||||
}
|
||||
return { tx: tx as unknown as DbOrTx, inserted, wasSelectCalled: () => selectCalled }
|
||||
}
|
||||
|
||||
const mappedKbResolver: ForkReferenceResolver = (kind, id) =>
|
||||
kind === 'knowledge-base' && id === 'src-kb' ? 'target-kb' : null
|
||||
|
||||
it('places a referenced doc into its already-mapped existing KB and returns the maps', async () => {
|
||||
const { tx, inserted } = makeTx([sourceRow('doc-1', 'src-kb')])
|
||||
const result = await planForkMappedKbDocumentCopies({
|
||||
tx,
|
||||
resolver: mappedKbResolver,
|
||||
referencedDocumentIds: ['doc-1'],
|
||||
alreadyCopiedSourceDocIds: new Set(),
|
||||
})
|
||||
|
||||
const childId = result.docIdMap.get('doc-1')
|
||||
expect(childId).toBeTruthy()
|
||||
expect(inserted).toHaveLength(1)
|
||||
expect(inserted[0]).toMatchObject({
|
||||
id: childId,
|
||||
knowledgeBaseId: 'target-kb',
|
||||
connectorId: null,
|
||||
deletedAt: null,
|
||||
archivedAt: null,
|
||||
})
|
||||
expect(result.mappingEntries).toEqual([
|
||||
{ resourceType: 'knowledge_document', parentResourceId: 'doc-1', childResourceId: childId },
|
||||
])
|
||||
expect(result.documents).toEqual([
|
||||
{
|
||||
sourceDocId: 'doc-1',
|
||||
childDocId: childId,
|
||||
childKnowledgeBaseId: 'target-kb',
|
||||
storageKey: 'kb/doc-1',
|
||||
filename: 'doc-1.pdf',
|
||||
mimeType: 'application/pdf',
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it('skips a referenced doc whose parent KB is not mapped (reference is left to be cleared)', async () => {
|
||||
const { tx, inserted } = makeTx([sourceRow('doc-1', 'unmapped-kb')])
|
||||
const result = await planForkMappedKbDocumentCopies({
|
||||
tx,
|
||||
resolver: mappedKbResolver,
|
||||
referencedDocumentIds: ['doc-1'],
|
||||
alreadyCopiedSourceDocIds: new Set(),
|
||||
})
|
||||
expect(inserted).toHaveLength(0)
|
||||
expect(result.docIdMap.size).toBe(0)
|
||||
expect(result.documents).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('skips a doc already placed under a copied KB this sync (no duplicate query)', async () => {
|
||||
const { tx, wasSelectCalled } = makeTx([sourceRow('doc-1', 'src-kb')])
|
||||
const result = await planForkMappedKbDocumentCopies({
|
||||
tx,
|
||||
resolver: mappedKbResolver,
|
||||
referencedDocumentIds: ['doc-1'],
|
||||
alreadyCopiedSourceDocIds: new Set(['doc-1']),
|
||||
})
|
||||
expect(result.documents).toHaveLength(0)
|
||||
expect(wasSelectCalled()).toBe(false)
|
||||
})
|
||||
|
||||
it('skips a doc that already resolves (mapped by a prior sync)', async () => {
|
||||
const { tx, wasSelectCalled } = makeTx([sourceRow('doc-1', 'src-kb')])
|
||||
const result = await planForkMappedKbDocumentCopies({
|
||||
tx,
|
||||
resolver: (kind, id) =>
|
||||
kind === 'knowledge-document' && id === 'doc-1' ? 'existing-child-doc' : null,
|
||||
referencedDocumentIds: ['doc-1'],
|
||||
alreadyCopiedSourceDocIds: new Set(),
|
||||
})
|
||||
expect(result.documents).toHaveLength(0)
|
||||
expect(wasSelectCalled()).toBe(false)
|
||||
})
|
||||
})
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,356 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { DbOrTx } from '@/lib/db/types'
|
||||
|
||||
const { mockSaveWorkflowToNormalizedTables } = vi.hoisted(() => ({
|
||||
mockSaveWorkflowToNormalizedTables: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/workflows/persistence/utils', () => ({
|
||||
saveWorkflowToNormalizedTables: mockSaveWorkflowToNormalizedTables,
|
||||
}))
|
||||
|
||||
import {
|
||||
buildWorkflowNameRegistry,
|
||||
copyWorkflowStateIntoTarget,
|
||||
resolveForkFolderMapping,
|
||||
} from '@/ee/workspace-forking/lib/copy/copy-workflows'
|
||||
|
||||
describe('buildWorkflowNameRegistry', () => {
|
||||
it('reports a name as taken by another workflow in the same folder', () => {
|
||||
const reg = buildWorkflowNameRegistry([{ id: 'w1', folderId: 'f1', name: 'Onboarding' }])
|
||||
expect(reg.isTaken('f1', 'Onboarding', null)).toBe(true)
|
||||
expect(reg.isTaken('f1', 'Onboarding', 'w2')).toBe(true)
|
||||
})
|
||||
|
||||
it('excludes the workflow itself so a replace can keep its own name', () => {
|
||||
const reg = buildWorkflowNameRegistry([{ id: 'w1', folderId: 'f1', name: 'Onboarding' }])
|
||||
expect(reg.isTaken('f1', 'Onboarding', 'w1')).toBe(false)
|
||||
})
|
||||
|
||||
it('is folder-scoped: the same name in another folder is free', () => {
|
||||
const reg = buildWorkflowNameRegistry([{ id: 'w1', folderId: 'f1', name: 'Onboarding' }])
|
||||
expect(reg.isTaken('f2', 'Onboarding', null)).toBe(false)
|
||||
expect(reg.isTaken(null, 'Onboarding', null)).toBe(false)
|
||||
})
|
||||
|
||||
it('treats the root (null) folder distinctly, matching coalesce(folderId, "")', () => {
|
||||
const reg = buildWorkflowNameRegistry([{ id: 'w1', folderId: null, name: 'Root WF' }])
|
||||
expect(reg.isTaken(null, 'Root WF', null)).toBe(true)
|
||||
expect(reg.isTaken('f1', 'Root WF', null)).toBe(false)
|
||||
})
|
||||
|
||||
it('claims a new name so a later workflow in the same copy loop sees it taken', () => {
|
||||
const reg = buildWorkflowNameRegistry([])
|
||||
expect(reg.isTaken('f1', 'Report', null)).toBe(false)
|
||||
reg.claim('f1', 'Report', 'wA')
|
||||
expect(reg.isTaken('f1', 'Report', null)).toBe(true)
|
||||
expect(reg.isTaken('f1', 'Report', 'wA')).toBe(false)
|
||||
})
|
||||
|
||||
it('releases the prior name when a workflow is renamed (claim moves keys)', () => {
|
||||
const reg = buildWorkflowNameRegistry([{ id: 'w1', folderId: 'f1', name: 'Old' }])
|
||||
reg.claim('f1', 'New', 'w1')
|
||||
expect(reg.isTaken('f1', 'Old', null)).toBe(false)
|
||||
expect(reg.isTaken('f1', 'New', null)).toBe(true)
|
||||
})
|
||||
|
||||
it('re-claiming the same (folder, name) is a no-op', () => {
|
||||
const reg = buildWorkflowNameRegistry([{ id: 'w1', folderId: 'f1', name: 'Same' }])
|
||||
reg.claim('f1', 'Same', 'w1')
|
||||
expect(reg.isTaken('f1', 'Same', 'w1')).toBe(false)
|
||||
expect(reg.isTaken('f1', 'Same', null)).toBe(true)
|
||||
})
|
||||
|
||||
it('handles multiple holders (legacy duplicates) and partial release', () => {
|
||||
const reg = buildWorkflowNameRegistry([
|
||||
{ id: 'w1', folderId: 'f1', name: 'Dup' },
|
||||
{ id: 'w2', folderId: 'f1', name: 'Dup' },
|
||||
])
|
||||
expect(reg.isTaken('f1', 'Dup', 'w1')).toBe(true)
|
||||
reg.claim('f1', 'Other', 'w2')
|
||||
expect(reg.isTaken('f1', 'Dup', 'w1')).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
interface FolderRow {
|
||||
id: string
|
||||
name: string
|
||||
userId: string
|
||||
workspaceId: string
|
||||
parentId: string | null
|
||||
color: string | null
|
||||
isExpanded: boolean
|
||||
locked: boolean
|
||||
sortOrder: number
|
||||
createdAt: Date
|
||||
updatedAt: Date
|
||||
archivedAt: Date | null
|
||||
}
|
||||
|
||||
function folderRow(id: string, name: string, parentId: string | null = null): FolderRow {
|
||||
return {
|
||||
id,
|
||||
name,
|
||||
userId: 'source-user',
|
||||
workspaceId: 'ws-source',
|
||||
parentId,
|
||||
color: '#6B7280',
|
||||
isExpanded: true,
|
||||
locked: false,
|
||||
sortOrder: 0,
|
||||
createdAt: new Date('2026-01-01'),
|
||||
updatedAt: new Date('2026-01-01'),
|
||||
archivedAt: null,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Transaction stub for {@link resolveForkFolderMapping}: the first awaited select resolves
|
||||
* the source folders, the second the target folders, and inserted rows are captured.
|
||||
*/
|
||||
function buildFolderTx(sourceFolders: FolderRow[], targetFolders: FolderRow[] = []) {
|
||||
const insertedRows: FolderRow[] = []
|
||||
const selects = [sourceFolders, targetFolders]
|
||||
let selectIndex = 0
|
||||
const tx = {
|
||||
select: () => ({
|
||||
from: () => ({
|
||||
where: () => Promise.resolve(selects[selectIndex++] ?? []),
|
||||
}),
|
||||
}),
|
||||
insert: () => ({
|
||||
values: (rows: FolderRow[]) => {
|
||||
insertedRows.push(...rows)
|
||||
return Promise.resolve()
|
||||
},
|
||||
}),
|
||||
} as unknown as DbOrTx
|
||||
return { tx, insertedRows }
|
||||
}
|
||||
|
||||
function resolveMapping(params: {
|
||||
tx: DbOrTx
|
||||
contentFolderIds: ReadonlyArray<string | null>
|
||||
}): Promise<Map<string, string>> {
|
||||
return resolveForkFolderMapping({
|
||||
tx: params.tx,
|
||||
sourceWorkspaceId: 'ws-source',
|
||||
targetWorkspaceId: 'ws-target',
|
||||
userId: 'target-user',
|
||||
now: new Date('2026-07-01'),
|
||||
contentFolderIds: params.contentFolderIds,
|
||||
})
|
||||
}
|
||||
|
||||
describe('resolveForkFolderMapping', () => {
|
||||
it('keeps the full ancestor chain of a nested folder holding a copied workflow', async () => {
|
||||
const { tx, insertedRows } = buildFolderTx([
|
||||
folderRow('A', 'Alpha'),
|
||||
folderRow('B', 'Beta', 'A'),
|
||||
folderRow('C', 'Gamma', 'B'),
|
||||
])
|
||||
|
||||
const map = await resolveMapping({ tx, contentFolderIds: ['C'] })
|
||||
|
||||
expect(map.size).toBe(3)
|
||||
expect(insertedRows).toHaveLength(3)
|
||||
const byName = new Map(insertedRows.map((row) => [row.name, row]))
|
||||
expect(byName.get('Alpha')?.parentId).toBeNull()
|
||||
expect(byName.get('Beta')?.parentId).toBe(map.get('A'))
|
||||
expect(byName.get('Gamma')?.parentId).toBe(map.get('B'))
|
||||
for (const row of insertedRows) {
|
||||
expect(row.workspaceId).toBe('ws-target')
|
||||
expect(row.userId).toBe('target-user')
|
||||
expect(row.locked).toBe(false)
|
||||
expect(['A', 'B', 'C']).not.toContain(row.id)
|
||||
}
|
||||
})
|
||||
|
||||
it('prunes an empty sibling subtree while keeping the occupied folder', async () => {
|
||||
const { tx, insertedRows } = buildFolderTx([
|
||||
folderRow('A', 'Occupied'),
|
||||
folderRow('D', 'Empty parent'),
|
||||
folderRow('E', 'Empty child', 'D'),
|
||||
])
|
||||
|
||||
const map = await resolveMapping({ tx, contentFolderIds: ['A'] })
|
||||
|
||||
expect(insertedRows).toHaveLength(1)
|
||||
expect(insertedRows[0].name).toBe('Occupied')
|
||||
expect(map.has('A')).toBe(true)
|
||||
expect(map.has('D')).toBe(false)
|
||||
expect(map.has('E')).toBe(false)
|
||||
})
|
||||
|
||||
it('prunes a root-level empty folder when the copied workflows live at root', async () => {
|
||||
const { tx, insertedRows } = buildFolderTx([folderRow('F', 'Never used')])
|
||||
|
||||
const map = await resolveMapping({ tx, contentFolderIds: [null, null] })
|
||||
|
||||
expect(insertedRows).toHaveLength(0)
|
||||
expect(map.size).toBe(0)
|
||||
})
|
||||
|
||||
it('creates no folders when nothing is copied into any folder', async () => {
|
||||
const { tx, insertedRows } = buildFolderTx([
|
||||
folderRow('A', 'Alpha'),
|
||||
folderRow('B', 'Beta', 'A'),
|
||||
])
|
||||
|
||||
const map = await resolveMapping({ tx, contentFolderIds: [] })
|
||||
|
||||
expect(insertedRows).toHaveLength(0)
|
||||
expect(map.size).toBe(0)
|
||||
})
|
||||
|
||||
it('reuses an existing target folder for a kept folder instead of duplicating it', async () => {
|
||||
const existing = { ...folderRow('T1', 'Shared'), workspaceId: 'ws-target' }
|
||||
const { tx, insertedRows } = buildFolderTx([folderRow('G', 'Shared')], [existing])
|
||||
|
||||
const map = await resolveMapping({ tx, contentFolderIds: ['G'] })
|
||||
|
||||
expect(insertedRows).toHaveLength(0)
|
||||
expect(map.get('G')).toBe('T1')
|
||||
})
|
||||
|
||||
it('maps a pruned folder onto a matching existing target folder without creating it', async () => {
|
||||
const existing = { ...folderRow('T1', 'Prior sync'), workspaceId: 'ws-target' }
|
||||
const { tx, insertedRows } = buildFolderTx([folderRow('P', 'Prior sync')], [existing])
|
||||
|
||||
const map = await resolveMapping({ tx, contentFolderIds: [] })
|
||||
|
||||
expect(insertedRows).toHaveLength(0)
|
||||
expect(map.get('P')).toBe('T1')
|
||||
})
|
||||
|
||||
it('never root-aliases a pruned nested folder onto a same-named root target folder', async () => {
|
||||
// Source X is nested under unmatched P; the target's root-level "X" is unrelated.
|
||||
const existing = { ...folderRow('T-root-x', 'X'), workspaceId: 'ws-target' }
|
||||
const { tx, insertedRows } = buildFolderTx(
|
||||
[folderRow('P', 'Parent'), folderRow('X', 'X', 'P')],
|
||||
[existing]
|
||||
)
|
||||
|
||||
const map = await resolveMapping({ tx, contentFolderIds: [] })
|
||||
|
||||
expect(insertedRows).toHaveLength(0)
|
||||
expect(map.size).toBe(0)
|
||||
})
|
||||
|
||||
it('creates a kept child under a reused existing parent folder', async () => {
|
||||
const existingParent = { ...folderRow('T-parent', 'Parent'), workspaceId: 'ws-target' }
|
||||
const { tx, insertedRows } = buildFolderTx(
|
||||
[folderRow('P', 'Parent'), folderRow('C', 'Child', 'P')],
|
||||
[existingParent]
|
||||
)
|
||||
|
||||
const map = await resolveMapping({ tx, contentFolderIds: ['C'] })
|
||||
|
||||
expect(map.get('P')).toBe('T-parent')
|
||||
expect(insertedRows).toHaveLength(1)
|
||||
expect(insertedRows[0].name).toBe('Child')
|
||||
expect(insertedRows[0].parentId).toBe('T-parent')
|
||||
})
|
||||
})
|
||||
|
||||
describe('copyWorkflowStateIntoTarget folder fallback', () => {
|
||||
it('places a copied workflow at the target root when its source folder has no mapping', async () => {
|
||||
mockSaveWorkflowToNormalizedTables.mockResolvedValue({ success: true })
|
||||
const insertedWorkflows: Array<Record<string, unknown>> = []
|
||||
const tx = {
|
||||
insert: () => ({
|
||||
values: (row: Record<string, unknown>) => {
|
||||
insertedWorkflows.push(row)
|
||||
return Promise.resolve()
|
||||
},
|
||||
}),
|
||||
} as unknown as DbOrTx
|
||||
|
||||
const result = await copyWorkflowStateIntoTarget({
|
||||
tx,
|
||||
targetWorkflowId: 'wf-child',
|
||||
targetWorkspaceId: 'ws-target',
|
||||
userId: 'target-user',
|
||||
mode: 'create',
|
||||
now: new Date('2026-07-01'),
|
||||
sourceState: { blocks: {}, edges: [], loops: {}, parallels: {}, variables: {} },
|
||||
sourceMeta: {
|
||||
name: 'Orphaned placement',
|
||||
description: null,
|
||||
folderId: 'folder-with-no-mapping',
|
||||
sortOrder: 0,
|
||||
},
|
||||
workflowIdMap: new Map(),
|
||||
folderIdMap: new Map(),
|
||||
nameRegistry: buildWorkflowNameRegistry([]),
|
||||
})
|
||||
|
||||
expect(insertedWorkflows).toHaveLength(1)
|
||||
expect(insertedWorkflows[0].folderId).toBeNull()
|
||||
expect(result.name).toBe('Orphaned placement')
|
||||
})
|
||||
})
|
||||
|
||||
describe('copyWorkflowStateIntoTarget canonicalModes reindex propagation', () => {
|
||||
it(
|
||||
"persists a transform's reindexed canonicalModes on the copied block, and uses that " +
|
||||
"SAME reindexed value (not the source's stale one) for every subsequent remap step",
|
||||
async () => {
|
||||
mockSaveWorkflowToNormalizedTables.mockResolvedValue({ success: true })
|
||||
const seenCanonicalModes: Array<Record<string, 'basic' | 'advanced'> | undefined> = []
|
||||
const tx = {
|
||||
insert: () => ({ values: () => Promise.resolve() }),
|
||||
} as unknown as DbOrTx
|
||||
|
||||
await copyWorkflowStateIntoTarget({
|
||||
tx,
|
||||
targetWorkflowId: 'wf-child',
|
||||
targetWorkspaceId: 'ws-target',
|
||||
userId: 'target-user',
|
||||
mode: 'create',
|
||||
now: new Date('2026-07-01'),
|
||||
sourceState: {
|
||||
blocks: {
|
||||
block1: {
|
||||
id: 'block1',
|
||||
type: 'agent',
|
||||
name: 'Agent',
|
||||
subBlocks: {},
|
||||
// The source's ORIGINAL (pre-drop) canonicalModes - every step after the
|
||||
// transform must see the REINDEXED value below instead, not this one.
|
||||
data: { canonicalModes: { '1:credential': 'advanced' } },
|
||||
},
|
||||
},
|
||||
edges: [],
|
||||
loops: {},
|
||||
parallels: {},
|
||||
variables: {},
|
||||
},
|
||||
sourceMeta: { name: 'Reindex test', description: null, folderId: null, sortOrder: 0 },
|
||||
workflowIdMap: new Map(),
|
||||
folderIdMap: new Map(),
|
||||
nameRegistry: buildWorkflowNameRegistry([]),
|
||||
// Simulates a `tool-input` drop shifting tool 1 -> 0: returns subBlocks unchanged but
|
||||
// reports the reindexed canonicalModes via the callback, exactly like
|
||||
// `createForkBootstrapTransform`/`createForkSubBlockTransform` do.
|
||||
transformSubBlocks: (subBlocks, _blockType, canonicalModes, onCanonicalModesChanged) => {
|
||||
seenCanonicalModes.push(canonicalModes)
|
||||
onCanonicalModesChanged?.({ '0:credential': 'advanced' })
|
||||
return subBlocks
|
||||
},
|
||||
})
|
||||
|
||||
const [, remappedState] = mockSaveWorkflowToNormalizedTables.mock.calls.at(-1)!
|
||||
const persistedBlock = Object.values(remappedState.blocks)[0] as {
|
||||
data?: { canonicalModes?: Record<string, 'basic' | 'advanced'> }
|
||||
}
|
||||
// The transform received the source's original value...
|
||||
expect(seenCanonicalModes).toEqual([{ '1:credential': 'advanced' }])
|
||||
// ...and the PERSISTED block carries the reindexed one, not the stale source value.
|
||||
expect(persistedBlock.data?.canonicalModes).toEqual({ '0:credential': 'advanced' })
|
||||
}
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,610 @@
|
||||
import { workflow, workflowBlocks, workflowFolder } from '@sim/db/schema'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { generateId } from '@sim/utils/id'
|
||||
import { and, eq, inArray, isNull } from 'drizzle-orm'
|
||||
import type { DbOrTx } from '@/lib/db/types'
|
||||
import { remapConditionEdgeHandle } from '@/lib/workflows/condition-ids'
|
||||
import {
|
||||
remapConditionIdsInSubBlocks,
|
||||
remapVariableIdsInSubBlocks,
|
||||
remapWorkflowReferencesInSubBlocks,
|
||||
type SubBlockRecord,
|
||||
sanitizeSubBlocksForDuplicate,
|
||||
} from '@/lib/workflows/persistence/remap-internal-ids'
|
||||
import { saveWorkflowToNormalizedTables } from '@/lib/workflows/persistence/utils'
|
||||
import type { CanonicalModeOverrides } from '@/lib/workflows/subblocks/visibility'
|
||||
import {
|
||||
deriveForkBlockId,
|
||||
type ForkBlockIdResolver,
|
||||
} from '@/ee/workspace-forking/lib/remap/block-identity'
|
||||
import {
|
||||
applyDependentOverrides,
|
||||
collectClearedDependents,
|
||||
type NeedsConfigurationField,
|
||||
type SubBlockTransform,
|
||||
} from '@/ee/workspace-forking/lib/remap/remap-references'
|
||||
import type {
|
||||
BlockData,
|
||||
BlockState,
|
||||
Loop,
|
||||
Parallel,
|
||||
SubBlockState,
|
||||
WorkflowState,
|
||||
Variable as WorkflowStateVariable,
|
||||
} from '@/stores/workflows/workflow/types'
|
||||
|
||||
const logger = createLogger('WorkspaceForkCopyWorkflows')
|
||||
|
||||
interface ResolveForkFolderMappingParams {
|
||||
tx: DbOrTx
|
||||
sourceWorkspaceId: string
|
||||
targetWorkspaceId: string
|
||||
userId: string
|
||||
now: Date
|
||||
/**
|
||||
* Source folder ids that will directly hold copied content (workflows); null entries
|
||||
* (root-placed content) are ignored. A source folder is copied into the target only when
|
||||
* its subtree contains at least one of these, so a fork/sync never creates folders that
|
||||
* would end up empty. Copied workspace FILES never influence this set: they live in the
|
||||
* separate `workspace_file_folders` entity and are flattened to root by the copy.
|
||||
*/
|
||||
contentFolderIds: ReadonlyArray<string | null>
|
||||
}
|
||||
|
||||
/**
|
||||
* Mirror into the target workspace the part of the source folder tree that will actually
|
||||
* receive copied content: the folders in `contentFolderIds` plus their ancestor chains (so
|
||||
* nesting stays intact). Target folders that already match by name within the same (mapped)
|
||||
* parent are reused instead of duplicated. Folders whose subtree holds no copied content are
|
||||
* pruned - never created - though a pruned folder still maps onto an existing target folder
|
||||
* when one matches, so previously-synced content refs keep resolving. Returns a map from
|
||||
* source folder id to target folder id; a copied workflow whose folder is absent from the
|
||||
* map is placed at the target's root (see {@link copyWorkflowStateIntoTarget}).
|
||||
*/
|
||||
export async function resolveForkFolderMapping({
|
||||
tx,
|
||||
sourceWorkspaceId,
|
||||
targetWorkspaceId,
|
||||
userId,
|
||||
now,
|
||||
contentFolderIds,
|
||||
}: ResolveForkFolderMappingParams): Promise<Map<string, string>> {
|
||||
const map = new Map<string, string>()
|
||||
|
||||
const sourceFolders = await tx
|
||||
.select()
|
||||
.from(workflowFolder)
|
||||
.where(
|
||||
and(eq(workflowFolder.workspaceId, sourceWorkspaceId), isNull(workflowFolder.archivedAt))
|
||||
)
|
||||
|
||||
if (sourceFolders.length === 0) return map
|
||||
|
||||
const byId = new Map(sourceFolders.map((folder) => [folder.id, folder]))
|
||||
|
||||
// Kept = folders that directly hold copied content plus every ancestor; everything else
|
||||
// would be empty in the target and is pruned. A dangling (archived) parent ends the walk,
|
||||
// matching the re-root fallback below.
|
||||
const kept = new Set<string>()
|
||||
for (const folderId of contentFolderIds) {
|
||||
let current = folderId ? byId.get(folderId) : undefined
|
||||
while (current && !kept.has(current.id)) {
|
||||
kept.add(current.id)
|
||||
current = current.parentId ? byId.get(current.parentId) : undefined
|
||||
}
|
||||
}
|
||||
|
||||
const targetFolders = await tx
|
||||
.select()
|
||||
.from(workflowFolder)
|
||||
.where(
|
||||
and(eq(workflowFolder.workspaceId, targetWorkspaceId), isNull(workflowFolder.archivedAt))
|
||||
)
|
||||
|
||||
const targetByKey = new Map<string, string>()
|
||||
for (const folder of targetFolders) {
|
||||
targetByKey.set(`${folder.parentId ?? ''}::${folder.name}`, folder.id)
|
||||
}
|
||||
|
||||
const ordered: typeof sourceFolders = []
|
||||
const seen = new Set<string>()
|
||||
const visit = (folder: (typeof sourceFolders)[number]) => {
|
||||
if (seen.has(folder.id)) return
|
||||
const parent = folder.parentId ? byId.get(folder.parentId) : undefined
|
||||
if (parent) visit(parent)
|
||||
seen.add(folder.id)
|
||||
ordered.push(folder)
|
||||
}
|
||||
for (const folder of sourceFolders) visit(folder)
|
||||
|
||||
const newFolders: (typeof sourceFolders)[number][] = []
|
||||
for (const folder of ordered) {
|
||||
const isKept = kept.has(folder.id)
|
||||
const mappedParentId = folder.parentId ? (map.get(folder.parentId) ?? null) : null
|
||||
const key = `${mappedParentId ?? ''}::${folder.name}`
|
||||
const existing = targetByKey.get(key)
|
||||
if (existing) {
|
||||
// A pruned folder may still MAP onto an existing target folder, but only when its
|
||||
// parent chain actually resolved: an unmapped pruned parent aliases the key to root
|
||||
// level, which could match an unrelated same-named root folder.
|
||||
if (isKept || !folder.parentId || map.has(folder.parentId)) {
|
||||
map.set(folder.id, existing)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (!isKept) continue
|
||||
const newFolderId = generateId()
|
||||
map.set(folder.id, newFolderId)
|
||||
targetByKey.set(key, newFolderId)
|
||||
newFolders.push({
|
||||
...folder,
|
||||
id: newFolderId,
|
||||
userId,
|
||||
workspaceId: targetWorkspaceId,
|
||||
parentId: mappedParentId,
|
||||
locked: false,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
}
|
||||
|
||||
if (newFolders.length > 0) {
|
||||
await tx.insert(workflowFolder).values(newFolders)
|
||||
}
|
||||
|
||||
return map
|
||||
}
|
||||
|
||||
// `\u0000` (a NUL byte) can never appear in a Postgres text column, so it is a
|
||||
// collision-free separator between the folder id and the name.
|
||||
const workflowNameKey = (folderId: string | null, name: string) => `${folderId ?? ''}\u0000${name}`
|
||||
|
||||
/**
|
||||
* In-memory registry of a workspace's active workflow names, keyed by
|
||||
* (folderId, name) - the exact columns of the `workflow_workspace_folder_name_active_unique`
|
||||
* partial index. Lets fork/promote resolve name collisions across many copied workflows
|
||||
* from a single load instead of one `nameTaken` query per workflow inside the (locked)
|
||||
* transaction.
|
||||
*
|
||||
* Correctness is still guaranteed by that DB unique index; this is only a proactive
|
||||
* collision avoider. A stale snapshot (e.g. a concurrent non-fork rename mid-promote) can
|
||||
* therefore only cause a rare, retry-able unique violation - never a duplicate name -
|
||||
* exactly as the prior per-workflow check-then-write already could.
|
||||
*/
|
||||
export interface WorkflowNameRegistry {
|
||||
/** True when (folderId, name) is held by any active workflow other than `excludeWorkflowId`. */
|
||||
isTaken(folderId: string | null, name: string, excludeWorkflowId: string | null): boolean
|
||||
/** Record that `workflowId` now holds (folderId, name), releasing any name it held before. */
|
||||
claim(folderId: string | null, name: string, workflowId: string): void
|
||||
}
|
||||
|
||||
/** Build a {@link WorkflowNameRegistry} from already-loaded rows (pure - unit-testable). */
|
||||
export function buildWorkflowNameRegistry(
|
||||
rows: Array<{ id: string; folderId: string | null; name: string }>
|
||||
): WorkflowNameRegistry {
|
||||
const holdersByKey = new Map<string, Set<string>>()
|
||||
const keyByWorkflow = new Map<string, string>()
|
||||
for (const row of rows) {
|
||||
const key = workflowNameKey(row.folderId, row.name)
|
||||
const holders = holdersByKey.get(key)
|
||||
if (holders) holders.add(row.id)
|
||||
else holdersByKey.set(key, new Set([row.id]))
|
||||
keyByWorkflow.set(row.id, key)
|
||||
}
|
||||
|
||||
return {
|
||||
isTaken(folderId, name, excludeWorkflowId) {
|
||||
const holders = holdersByKey.get(workflowNameKey(folderId, name))
|
||||
if (!holders) return false
|
||||
for (const id of holders) if (id !== excludeWorkflowId) return true
|
||||
return false
|
||||
},
|
||||
claim(folderId, name, workflowId) {
|
||||
const newKey = workflowNameKey(folderId, name)
|
||||
const prevKey = keyByWorkflow.get(workflowId)
|
||||
if (prevKey === newKey) return
|
||||
if (prevKey) holdersByKey.get(prevKey)?.delete(workflowId)
|
||||
const holders = holdersByKey.get(newKey)
|
||||
if (holders) holders.add(workflowId)
|
||||
else holdersByKey.set(newKey, new Set([workflowId]))
|
||||
keyByWorkflow.set(workflowId, newKey)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/** Load every active workflow name in a workspace into a {@link WorkflowNameRegistry}. */
|
||||
export async function loadWorkflowNameRegistry(
|
||||
executor: DbOrTx,
|
||||
workspaceId: string
|
||||
): Promise<WorkflowNameRegistry> {
|
||||
const rows = await executor
|
||||
.select({ id: workflow.id, folderId: workflow.folderId, name: workflow.name })
|
||||
.from(workflow)
|
||||
.where(and(eq(workflow.workspaceId, workspaceId), isNull(workflow.archivedAt)))
|
||||
return buildWorkflowNameRegistry(rows)
|
||||
}
|
||||
|
||||
/**
|
||||
* Batched read of the current DRAFT subBlocks for a set of (replace) target
|
||||
* workflows, keyed `workflowId -> blockId -> subBlocks`. One query for the whole
|
||||
* promote so the locked apply phase doesn't do N per-workflow loads; called
|
||||
* pre-write so it reflects the target state the user configured before this sync
|
||||
* overwrites it. Promote uses it to detect required dependents the sync left empty
|
||||
* (see {@link collectClearedDependents}); fork-create has no prior target and skips it.
|
||||
*/
|
||||
export async function loadTargetDraftSubBlocks(
|
||||
executor: DbOrTx,
|
||||
workflowIds: string[]
|
||||
): Promise<Map<string, Map<string, SubBlockRecord>>> {
|
||||
const byWorkflow = new Map<string, Map<string, SubBlockRecord>>()
|
||||
if (workflowIds.length === 0) return byWorkflow
|
||||
const rows = await executor
|
||||
.select({
|
||||
workflowId: workflowBlocks.workflowId,
|
||||
blockId: workflowBlocks.id,
|
||||
subBlocks: workflowBlocks.subBlocks,
|
||||
})
|
||||
.from(workflowBlocks)
|
||||
.where(inArray(workflowBlocks.workflowId, workflowIds))
|
||||
for (const row of rows) {
|
||||
let blocks = byWorkflow.get(row.workflowId)
|
||||
if (!blocks) {
|
||||
blocks = new Map<string, SubBlockRecord>()
|
||||
byWorkflow.set(row.workflowId, blocks)
|
||||
}
|
||||
blocks.set(row.blockId, (row.subBlocks ?? {}) as SubBlockRecord)
|
||||
}
|
||||
return byWorkflow
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick a non-colliding name for a copied workflow against the preloaded registry, which
|
||||
* mirrors the workspace's (folder, name, not-archived, exclude-self) predicate from one
|
||||
* query instead of one per candidate. Mirrors {@link deduplicateWorkflowName}'s ` (n)`
|
||||
* numbering, but reads from memory so the copy loop issues no per-workflow name queries.
|
||||
*/
|
||||
function resolveTargetWorkflowName(
|
||||
registry: WorkflowNameRegistry,
|
||||
folderId: string | null,
|
||||
name: string,
|
||||
excludeWorkflowId: string | null
|
||||
): string {
|
||||
const taken = (candidate: string) => registry.isTaken(folderId, candidate, excludeWorkflowId)
|
||||
if (!taken(name)) return name
|
||||
for (let i = 2; i < 100; i++) {
|
||||
const candidate = `${name} (${i})`
|
||||
if (!taken(candidate)) return candidate
|
||||
}
|
||||
return `${name} (${generateId().slice(0, 6)})`
|
||||
}
|
||||
|
||||
export interface CopyWorkflowResult {
|
||||
targetWorkflowId: string
|
||||
mode: 'create' | 'replace'
|
||||
name: string
|
||||
blocksCount: number
|
||||
edgesCount: number
|
||||
subflowsCount: number
|
||||
/**
|
||||
* `dependsOn` fields (top-level and nested tool-input) a remapped parent left empty
|
||||
* that weren't restored from the target draft - the parent legitimately changed.
|
||||
* Carries `required` per field: promote skips redeploy + gates on required ones and
|
||||
* surfaces optional ones so a cleared filter never broadens behavior silently.
|
||||
*/
|
||||
clearedDependents: NeedsConfigurationField[]
|
||||
/**
|
||||
* Source block id -> assigned target block id, so the caller can persist the
|
||||
* block-identity pairs (see `recordForkBlockPairs`) that keep promotes reversible.
|
||||
*/
|
||||
blockIdMapping: Map<string, string>
|
||||
}
|
||||
|
||||
export interface CopyWorkflowStateParams {
|
||||
tx: DbOrTx
|
||||
targetWorkflowId: string
|
||||
targetWorkspaceId: string
|
||||
userId: string
|
||||
mode: 'create' | 'replace'
|
||||
now: Date
|
||||
/** Source workflow's deployed state (the only thing fork/promote copies). */
|
||||
sourceState: WorkflowState
|
||||
/** Source workflow metadata for naming, folder placement, and sort order. */
|
||||
sourceMeta: {
|
||||
name: string
|
||||
description: string | null
|
||||
folderId: string | null
|
||||
sortOrder: number
|
||||
/**
|
||||
* Whether the source's deployed API is public (unauthenticated). Carried onto sync targets
|
||||
* so a public source stays public after push/pull - the target org's own access-control
|
||||
* gate (`validatePublicApiAllowed`) still applies at execution. Omitted at fork-create:
|
||||
* the child starts undeployed and private (going public is an explicit act there).
|
||||
*/
|
||||
isPublicApi?: boolean
|
||||
}
|
||||
/** source workflow id -> target workflow id, for `workflow-selector` references */
|
||||
workflowIdMap: Map<string, string>
|
||||
/** source folder id -> target folder id */
|
||||
folderIdMap: Map<string, string>
|
||||
/** Optional resource-reference remap applied to every block's subBlocks. */
|
||||
transformSubBlocks?: SubBlockTransform
|
||||
/**
|
||||
* The target workflow's current draft subBlocks (block id -> subBlocks), for
|
||||
* `replace` mode only. When present, required dependents that the sync left empty
|
||||
* (the parent change cleared and the stored mapping didn't fill) are reported in
|
||||
* {@link CopyWorkflowResult.needsConfiguration}.
|
||||
*/
|
||||
targetCurrentBlocks?: Map<string, SubBlockRecord>
|
||||
/**
|
||||
* Per-block (block id -> subBlock key -> value) stored dependent values applied last,
|
||||
* after the reference transform cleared the source's, so the stored mapping is the sole
|
||||
* source of truth for what each dependent selector resolves to.
|
||||
*/
|
||||
dependentOverrides?: Map<string, Map<string, string>>
|
||||
/**
|
||||
* Preloaded name registry so name-collision resolution reads from memory instead of one
|
||||
* query per workflow inside the tx. Build once per copy loop via {@link loadWorkflowNameRegistry}.
|
||||
*/
|
||||
nameRegistry: WorkflowNameRegistry
|
||||
/**
|
||||
* Resolve each source block to its target block id, reusing the persisted counterpart
|
||||
* when one exists so a push keeps the parent's original block ids (and webhook URLs)
|
||||
* instead of re-deriving them (see {@link buildForkBlockIdResolver}). Omitted on fork
|
||||
* creation, where every id is derived fresh.
|
||||
*/
|
||||
resolveBlockId?: ForkBlockIdResolver
|
||||
requestId?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy a source workflow's deployed `WorkflowState` into a target workflow,
|
||||
* assigning deterministic block ids (so trigger webhook URLs and external block
|
||||
* references stay stable across promotes) and applying the resource-reference
|
||||
* transform. Writes the remapped state to the target's draft via
|
||||
* `saveWorkflowToNormalizedTables`. In `create` mode a new workflow row is
|
||||
* inserted (undeployed); in `replace` mode the existing target row is kept and
|
||||
* its draft is overwritten. Deploying the target (and capturing the rollback
|
||||
* point) is the caller's responsibility.
|
||||
*/
|
||||
export async function copyWorkflowStateIntoTarget(
|
||||
params: CopyWorkflowStateParams
|
||||
): Promise<CopyWorkflowResult> {
|
||||
const {
|
||||
tx,
|
||||
targetWorkflowId,
|
||||
targetWorkspaceId,
|
||||
userId,
|
||||
mode,
|
||||
now,
|
||||
sourceState,
|
||||
sourceMeta,
|
||||
workflowIdMap,
|
||||
folderIdMap,
|
||||
transformSubBlocks,
|
||||
targetCurrentBlocks,
|
||||
dependentOverrides,
|
||||
nameRegistry,
|
||||
resolveBlockId,
|
||||
requestId = 'unknown',
|
||||
} = params
|
||||
|
||||
const targetFolderId = sourceMeta.folderId ? (folderIdMap.get(sourceMeta.folderId) ?? null) : null
|
||||
|
||||
const varIdMapping = new Map<string, string>()
|
||||
const remappedVariables: Record<string, WorkflowStateVariable> = {}
|
||||
for (const [oldVarId, variable] of Object.entries(sourceState.variables ?? {})) {
|
||||
const newVarId = generateId()
|
||||
varIdMapping.set(oldVarId, newVarId)
|
||||
remappedVariables[newVarId] = { ...variable, id: newVarId }
|
||||
}
|
||||
|
||||
const blockIdMapping = new Map<string, string>()
|
||||
for (const oldBlockId of Object.keys(sourceState.blocks)) {
|
||||
blockIdMapping.set(
|
||||
oldBlockId,
|
||||
resolveBlockId
|
||||
? resolveBlockId(targetWorkflowId, oldBlockId)
|
||||
: deriveForkBlockId(targetWorkflowId, oldBlockId)
|
||||
)
|
||||
}
|
||||
|
||||
const newBlocks: Record<string, BlockState> = {}
|
||||
const clearedDependents: NeedsConfigurationField[] = []
|
||||
for (const [oldBlockId, block] of Object.entries(sourceState.blocks)) {
|
||||
const newBlockId = blockIdMapping.get(oldBlockId)!
|
||||
|
||||
let updatedData = block.data
|
||||
if (block.data && typeof block.data === 'object' && !Array.isArray(block.data)) {
|
||||
const dataObj = block.data as Record<string, unknown>
|
||||
if (typeof dataObj.parentId === 'string' && blockIdMapping.has(dataObj.parentId)) {
|
||||
updatedData = {
|
||||
...dataObj,
|
||||
parentId: blockIdMapping.get(dataObj.parentId)!,
|
||||
extent: 'parent',
|
||||
} as BlockData
|
||||
}
|
||||
}
|
||||
|
||||
// double-cast-allowed: SubBlockState is structurally a SubBlockRecord entry but lacks the open index signature SubBlockRecord declares
|
||||
const sourceSubBlocks = (block.subBlocks ?? {}) as unknown as SubBlockRecord
|
||||
const sanitizedSource = sanitizeSubBlocksForDuplicate(sourceSubBlocks)
|
||||
let subBlocks: SubBlockRecord = sanitizedSource
|
||||
// Tracks the block's live `canonicalModes` through this pass, so a `tool-input` reindex
|
||||
// (a dropped custom-tool/MCP entry shifts later tools' array positions) is visible to every
|
||||
// later step below that resolves a nested tool's basic/advanced mode - not just the final
|
||||
// persisted `updatedData`. Starts as the source value; `transformSubBlocks` may replace it.
|
||||
let activeCanonicalModes: CanonicalModeOverrides | undefined = (
|
||||
block.data as { canonicalModes?: Record<string, 'basic' | 'advanced'> } | undefined
|
||||
)?.canonicalModes
|
||||
if (transformSubBlocks) {
|
||||
subBlocks = transformSubBlocks(subBlocks, block.type, activeCanonicalModes, (next) => {
|
||||
activeCanonicalModes = next
|
||||
updatedData = { ...updatedData, canonicalModes: next } as BlockData
|
||||
})
|
||||
}
|
||||
if (varIdMapping.size > 0) {
|
||||
subBlocks = remapVariableIdsInSubBlocks(subBlocks, varIdMapping)
|
||||
}
|
||||
// Cross-workspace copy: clear references to workflows that weren't copied
|
||||
// rather than leave them pointing at the source workspace.
|
||||
subBlocks = remapWorkflowReferencesInSubBlocks(subBlocks, workflowIdMap, {
|
||||
clearUnmapped: true,
|
||||
canonicalModes: activeCanonicalModes,
|
||||
})
|
||||
subBlocks = remapConditionIdsInSubBlocks(subBlocks, oldBlockId, newBlockId) as SubBlockRecord
|
||||
|
||||
// Apply the stored dependent values for this block (the modal's mapping). The reference
|
||||
// transform already cleared the source's dependent values when their parent was remapped,
|
||||
// so the stored mapping is the SOLE source of truth - no implicit "preserve the target's
|
||||
// value" path. Allowlisted (top-level + nested tool params) inside applyDependentOverrides
|
||||
// so a crafted value can't touch a parent/credential field or inject a bogus subblock.
|
||||
const targetCurrent = targetCurrentBlocks?.get(newBlockId)
|
||||
const blockOverrides = dependentOverrides?.get(newBlockId)
|
||||
if (blockOverrides && blockOverrides.size > 0) {
|
||||
subBlocks = applyDependentOverrides(subBlocks, block.type, blockOverrides)
|
||||
}
|
||||
|
||||
// Dependents the TARGET had configured that the parent change cleared and nothing
|
||||
// restored: the target must re-pick required ones (promote skips this workflow's
|
||||
// redeploy) and is told about optional ones. Keyed on the target draft so a field the
|
||||
// source carried but the target never set isn't flagged.
|
||||
if (mode === 'replace' && targetCurrent) {
|
||||
clearedDependents.push(
|
||||
...collectClearedDependents(
|
||||
block.type,
|
||||
newBlockId,
|
||||
block.name,
|
||||
targetCurrent,
|
||||
subBlocks,
|
||||
activeCanonicalModes
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
newBlocks[newBlockId] = {
|
||||
...block,
|
||||
id: newBlockId,
|
||||
// double-cast-allowed: remap helpers return SubBlockRecord; the entries retain the SubBlockState shape this block requires
|
||||
subBlocks: subBlocks as unknown as Record<string, SubBlockState>,
|
||||
data: updatedData,
|
||||
}
|
||||
}
|
||||
|
||||
const newEdges = sourceState.edges.flatMap((edge) => {
|
||||
const newSource = blockIdMapping.get(edge.source)
|
||||
const newTarget = blockIdMapping.get(edge.target)
|
||||
if (!newSource || !newTarget) {
|
||||
logger.warn(`[${requestId}] Skipping edge with unmapped block reference during fork copy`, {
|
||||
edgeId: edge.id,
|
||||
})
|
||||
return []
|
||||
}
|
||||
const newSourceHandle = edge.sourceHandle
|
||||
? remapConditionEdgeHandle(edge.sourceHandle, edge.source, newSource)
|
||||
: edge.sourceHandle
|
||||
return [
|
||||
{
|
||||
...edge,
|
||||
id: generateId(),
|
||||
source: newSource,
|
||||
target: newTarget,
|
||||
sourceHandle: newSourceHandle,
|
||||
targetHandle: edge.targetHandle,
|
||||
},
|
||||
]
|
||||
})
|
||||
|
||||
const newLoops: Record<string, Loop> = {}
|
||||
for (const [oldId, loop] of Object.entries(sourceState.loops ?? {})) {
|
||||
const newId = blockIdMapping.get(oldId) ?? oldId
|
||||
newLoops[newId] = {
|
||||
...loop,
|
||||
id: newId,
|
||||
nodes: loop.nodes.flatMap((nodeId) => {
|
||||
const mapped = blockIdMapping.get(nodeId)
|
||||
return mapped ? [mapped] : []
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
const newParallels: Record<string, Parallel> = {}
|
||||
for (const [oldId, parallel] of Object.entries(sourceState.parallels ?? {})) {
|
||||
const newId = blockIdMapping.get(oldId) ?? oldId
|
||||
newParallels[newId] = {
|
||||
...parallel,
|
||||
id: newId,
|
||||
nodes: parallel.nodes.flatMap((nodeId) => {
|
||||
const mapped = blockIdMapping.get(nodeId)
|
||||
return mapped ? [mapped] : []
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
const resolvedName = resolveTargetWorkflowName(
|
||||
nameRegistry,
|
||||
targetFolderId,
|
||||
sourceMeta.name,
|
||||
mode === 'replace' ? targetWorkflowId : null
|
||||
)
|
||||
// Claim the resolved name so the next workflow in this copy loop sees it taken. The DB
|
||||
// write below uses the same (folderId, name), so the registry stays consistent with it.
|
||||
nameRegistry.claim(targetFolderId, resolvedName, targetWorkflowId)
|
||||
|
||||
if (mode === 'create') {
|
||||
await tx.insert(workflow).values({
|
||||
id: targetWorkflowId,
|
||||
userId,
|
||||
workspaceId: targetWorkspaceId,
|
||||
folderId: targetFolderId,
|
||||
sortOrder: sourceMeta.sortOrder,
|
||||
name: resolvedName,
|
||||
description: sourceMeta.description,
|
||||
lastSynced: now,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
isDeployed: false,
|
||||
runCount: 0,
|
||||
locked: false,
|
||||
variables: remappedVariables,
|
||||
// Deployment visibility follows the source on sync (a public source stays public in
|
||||
// the target); fork-create omits the field, so the child starts private.
|
||||
...(sourceMeta.isPublicApi !== undefined ? { isPublicApi: sourceMeta.isPublicApi } : {}),
|
||||
})
|
||||
} else {
|
||||
await tx
|
||||
.update(workflow)
|
||||
.set({
|
||||
name: resolvedName,
|
||||
description: sourceMeta.description,
|
||||
folderId: targetFolderId,
|
||||
variables: remappedVariables,
|
||||
lastSynced: now,
|
||||
updatedAt: now,
|
||||
...(sourceMeta.isPublicApi !== undefined ? { isPublicApi: sourceMeta.isPublicApi } : {}),
|
||||
})
|
||||
.where(eq(workflow.id, targetWorkflowId))
|
||||
}
|
||||
|
||||
const remappedState: WorkflowState = {
|
||||
blocks: newBlocks,
|
||||
edges: newEdges,
|
||||
loops: newLoops,
|
||||
parallels: newParallels,
|
||||
variables: remappedVariables,
|
||||
}
|
||||
const saved = await saveWorkflowToNormalizedTables(targetWorkflowId, remappedState, tx)
|
||||
if (!saved.success) {
|
||||
throw new Error(`Failed to write forked workflow ${targetWorkflowId}: ${saved.error}`)
|
||||
}
|
||||
|
||||
return {
|
||||
targetWorkflowId,
|
||||
mode,
|
||||
name: resolvedName,
|
||||
blocksCount: Object.keys(newBlocks).length,
|
||||
edgesCount: newEdges.length,
|
||||
subflowsCount: Object.keys(newLoops).length + Object.keys(newParallels).length,
|
||||
clearedDependents,
|
||||
blockIdMapping,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
import { db, runOutsideTransactionContext } from '@sim/db'
|
||||
import { workflow, workflowDeploymentVersion } from '@sim/db/schema'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { and, eq, exists, inArray, isNull, sql } from 'drizzle-orm'
|
||||
import type { DbOrTx } from '@/lib/db/types'
|
||||
import { loadDeployedWorkflowState } from '@/lib/workflows/persistence/utils'
|
||||
import { ForkError } from '@/ee/workspace-forking/lib/lineage/authz'
|
||||
import type { Variable, WorkflowState } from '@/stores/workflows/workflow/types'
|
||||
|
||||
const logger = createLogger('WorkspaceForkDeployBridge')
|
||||
|
||||
/**
|
||||
* Hard ceiling on how many deployed workflows one fork/promote loads into memory at
|
||||
* once (each as a full `WorkflowState`). There is no per-workspace workflow cap in
|
||||
* the product, so this is the safety valve: real workspaces hold tens to low
|
||||
* hundreds, making this ~5-10x headroom that never blocks legitimate use, it sits
|
||||
* below the fork feature's other item caps (resource selection 2000, mapping
|
||||
* entries 5000 - both lighter-weight than full states), and it bounds a pathological
|
||||
* workspace to a few hundred MB of transient state instead of an unbounded load.
|
||||
*/
|
||||
export const MAX_FORK_DEPLOYED_WORKFLOWS = 1000
|
||||
|
||||
export interface DeployedWorkflowSummary {
|
||||
id: string
|
||||
name: string
|
||||
description: string | null
|
||||
folderId: string | null
|
||||
sortOrder: number
|
||||
/** Whether the deployed API accepts unauthenticated calls; carried onto sync targets. */
|
||||
isPublicApi: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Workflows in a workspace that are deployed and not archived - the only ones that
|
||||
* fork/promote. Requires an actually-active deployment version, not just the
|
||||
* `isDeployed` flag: a workflow flagged deployed with no active version (a "ghost"
|
||||
* left by an inconsistent state) has nothing to copy, so excluding it here keeps the
|
||||
* diff/plan counts aligned with what apply actually writes instead of over-reporting
|
||||
* then silently skipping it. Correlated `exists` (not a join) so a workflow is never
|
||||
* double-listed if more than one active version row ever exists.
|
||||
*/
|
||||
export async function listDeployedWorkflows(
|
||||
executor: DbOrTx,
|
||||
workspaceId: string
|
||||
): Promise<DeployedWorkflowSummary[]> {
|
||||
return executor
|
||||
.select({
|
||||
id: workflow.id,
|
||||
name: workflow.name,
|
||||
description: workflow.description,
|
||||
folderId: workflow.folderId,
|
||||
sortOrder: workflow.sortOrder,
|
||||
isPublicApi: workflow.isPublicApi,
|
||||
})
|
||||
.from(workflow)
|
||||
.where(
|
||||
and(
|
||||
eq(workflow.workspaceId, workspaceId),
|
||||
eq(workflow.isDeployed, true),
|
||||
isNull(workflow.archivedAt),
|
||||
exists(
|
||||
db
|
||||
.select({ one: sql`1` })
|
||||
.from(workflowDeploymentVersion)
|
||||
.where(
|
||||
and(
|
||||
eq(workflowDeploymentVersion.workflowId, workflow.id),
|
||||
eq(workflowDeploymentVersion.isActive, true)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
/** The active deployment version number for a workflow, or null when it has none. */
|
||||
export async function getActiveDeploymentVersionNumber(
|
||||
executor: DbOrTx,
|
||||
workflowId: string
|
||||
): Promise<number | null> {
|
||||
const [row] = await executor
|
||||
.select({ version: workflowDeploymentVersion.version })
|
||||
.from(workflowDeploymentVersion)
|
||||
.where(
|
||||
and(
|
||||
eq(workflowDeploymentVersion.workflowId, workflowId),
|
||||
eq(workflowDeploymentVersion.isActive, true)
|
||||
)
|
||||
)
|
||||
.limit(1)
|
||||
return row?.version ?? null
|
||||
}
|
||||
|
||||
/**
|
||||
* Batched {@link getActiveDeploymentVersionNumber}: the active deployed version per
|
||||
* workflow id, so promote's apply phase resolves every prior version in one query
|
||||
* instead of N round-trips inside the (locked) transaction. Workflows with no active
|
||||
* version are absent from the map.
|
||||
*/
|
||||
export async function getActiveDeploymentVersionNumbers(
|
||||
executor: DbOrTx,
|
||||
workflowIds: string[]
|
||||
): Promise<Map<string, number>> {
|
||||
if (workflowIds.length === 0) return new Map()
|
||||
const rows = await executor
|
||||
.select({
|
||||
workflowId: workflowDeploymentVersion.workflowId,
|
||||
version: workflowDeploymentVersion.version,
|
||||
})
|
||||
.from(workflowDeploymentVersion)
|
||||
.where(
|
||||
and(
|
||||
inArray(workflowDeploymentVersion.workflowId, workflowIds),
|
||||
eq(workflowDeploymentVersion.isActive, true)
|
||||
)
|
||||
)
|
||||
return new Map(rows.map((row) => [row.workflowId, row.version]))
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a source workspace's deployed workflows and each one's active deployed state
|
||||
* on the global pool. Fork/promote callers MUST run this BEFORE opening their
|
||||
* transaction: doing these heavy per-workflow reads inside the tx checks out a
|
||||
* SECOND pooled connection while the tx holds the first, which can deadlock the
|
||||
* pool at saturation (primary pool max is 15). The source is read-only for the
|
||||
* operation, so a pre-transaction snapshot is the value that gets force-pushed.
|
||||
*
|
||||
* Holds every source state in memory at once (bounded by the workspace's deployed
|
||||
* workflow count) - the apply step needs each state to write its target inside the
|
||||
* single atomic transaction, so it cannot stream them one at a time.
|
||||
*/
|
||||
export async function loadSourceDeployedStates(sourceWorkspaceId: string): Promise<{
|
||||
deployedWorkflows: DeployedWorkflowSummary[]
|
||||
sourceStates: Map<string, WorkflowState>
|
||||
}> {
|
||||
const deployedWorkflows = await listDeployedWorkflows(db, sourceWorkspaceId)
|
||||
// Fail fast on the cheap count before loading any heavy state into memory.
|
||||
if (deployedWorkflows.length > MAX_FORK_DEPLOYED_WORKFLOWS) {
|
||||
throw new ForkError(
|
||||
`This workspace has ${deployedWorkflows.length} deployed workflows, which exceeds the fork/sync limit of ${MAX_FORK_DEPLOYED_WORKFLOWS}.`,
|
||||
400
|
||||
)
|
||||
}
|
||||
// Read states in bounded-concurrency batches instead of one serial await per workflow:
|
||||
// serial cost is O(workflows) round trips (this also runs on the diff preview, refetched
|
||||
// while the sync modal is open). The cap keeps concurrent global-pool checkouts well
|
||||
// under the pool max even at the workflow ceiling, and this runs BEFORE any transaction.
|
||||
const sourceStates = new Map<string, WorkflowState>()
|
||||
const READ_CONCURRENCY = 5
|
||||
for (let i = 0; i < deployedWorkflows.length; i += READ_CONCURRENCY) {
|
||||
const batch = deployedWorkflows.slice(i, i + READ_CONCURRENCY)
|
||||
const states = await Promise.all(batch.map((wf) => readDeployedState(wf.id, sourceWorkspaceId)))
|
||||
batch.forEach((wf, index) => {
|
||||
const state = states[index]
|
||||
if (state) sourceStates.set(wf.id, state)
|
||||
})
|
||||
}
|
||||
return { deployedWorkflows, sourceStates }
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a workflow's active deployed state as a `WorkflowState`. Returns null ONLY
|
||||
* when the workflow genuinely has no active deployment (a legitimate skip); real
|
||||
* DB/migration errors propagate so the caller fails loudly instead of silently
|
||||
* dropping the workflow from the fork/promote. Block migrations (credential remap
|
||||
* to current ids) are applied so copied references reflect current resources.
|
||||
*/
|
||||
export async function readDeployedState(
|
||||
workflowId: string,
|
||||
workspaceId: string
|
||||
): Promise<WorkflowState | null> {
|
||||
// This reads the (unchanged) SOURCE workspace on the global pool. Callers like
|
||||
// promote run it inside their transaction, so escape the tx context: the read
|
||||
// must not join the promote's transaction (and the tripwire forbids global-pool
|
||||
// queries inside a tx). Outside a transaction this is a no-op.
|
||||
return runOutsideTransactionContext(async () => {
|
||||
const version = await getActiveDeploymentVersionNumber(db, workflowId)
|
||||
if (version == null) {
|
||||
logger.warn('No active deployment for workflow during fork/promote', { workflowId })
|
||||
return null
|
||||
}
|
||||
const data = await loadDeployedWorkflowState(workflowId, workspaceId)
|
||||
return {
|
||||
blocks: data.blocks,
|
||||
edges: data.edges,
|
||||
loops: data.loops,
|
||||
parallels: data.parallels,
|
||||
variables: (data.variables ?? {}) as Record<string, Variable>,
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const { mockCheckStorageQuota } = vi.hoisted(() => ({
|
||||
mockCheckStorageQuota: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/billing/storage', () => ({
|
||||
checkStorageQuota: mockCheckStorageQuota,
|
||||
}))
|
||||
|
||||
/**
|
||||
* Minimal stand-in for the domain error so this unit test never loads the authz module's
|
||||
* billing/feature-flag import chain. Shape-compatible with the real `ForkError`.
|
||||
*/
|
||||
vi.mock('@/ee/workspace-forking/lib/lineage/authz', () => ({
|
||||
ForkError: class ForkError extends Error {
|
||||
statusCode: number
|
||||
constructor(message: string, statusCode = 400) {
|
||||
super(message)
|
||||
this.name = 'ForkError'
|
||||
this.statusCode = statusCode
|
||||
}
|
||||
},
|
||||
}))
|
||||
|
||||
import type { DbOrTx } from '@/lib/db/types'
|
||||
import {
|
||||
assertForkStorageHeadroom,
|
||||
sumForkCopyBytes,
|
||||
} from '@/ee/workspace-forking/lib/copy/storage-quota'
|
||||
import { ForkError } from '@/ee/workspace-forking/lib/lineage/authz'
|
||||
|
||||
/**
|
||||
* Fake executor resolving one aggregate row per query, in call order. Supports both sum
|
||||
* shapes: `select().from().where()` (files) and `select().from().innerJoin().where()` (KB
|
||||
* documents joined to their live KB row).
|
||||
*/
|
||||
function makeExecutor(totals: Array<number | string>) {
|
||||
let call = 0
|
||||
const next = () => Promise.resolve([{ total: totals[call++] ?? 0 }])
|
||||
const select = vi.fn(() => ({
|
||||
from: () => ({
|
||||
where: next,
|
||||
innerJoin: () => ({ where: next }),
|
||||
}),
|
||||
}))
|
||||
return { executor: { select } as unknown as DbOrTx, select }
|
||||
}
|
||||
|
||||
describe('sumForkCopyBytes', () => {
|
||||
it('adds the workspace-file and KB-document byte sums', async () => {
|
||||
const { executor, select } = makeExecutor([300, 700])
|
||||
|
||||
const bytes = await sumForkCopyBytes(executor, 'src-ws', {
|
||||
fileIds: ['wf-1'],
|
||||
knowledgeBaseIds: ['kb-1'],
|
||||
})
|
||||
|
||||
expect(bytes).toBe(1000)
|
||||
expect(select).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('coerces driver string aggregates (bigint sums) to numbers', async () => {
|
||||
const { executor } = makeExecutor(['1024'])
|
||||
|
||||
const bytes = await sumForkCopyBytes(executor, 'src-ws', { fileKeys: ['workspace/src/k1'] })
|
||||
|
||||
expect(bytes).toBe(1024)
|
||||
})
|
||||
|
||||
it('runs no query for an empty selection', async () => {
|
||||
const { executor, select } = makeExecutor([])
|
||||
|
||||
const bytes = await sumForkCopyBytes(executor, 'src-ws', {
|
||||
fileIds: [],
|
||||
fileKeys: [],
|
||||
knowledgeBaseIds: [],
|
||||
})
|
||||
|
||||
expect(bytes).toBe(0)
|
||||
expect(select).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('skips the file query when only KBs are selected (and vice versa)', async () => {
|
||||
const { executor, select } = makeExecutor([555])
|
||||
|
||||
const bytes = await sumForkCopyBytes(executor, 'src-ws', { knowledgeBaseIds: ['kb-1'] })
|
||||
|
||||
expect(bytes).toBe(555)
|
||||
expect(select).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('assertForkStorageHeadroom', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('never consults the quota helper for zero bytes', async () => {
|
||||
await assertForkStorageHeadroom({ userId: 'user-1', bytes: 0 })
|
||||
expect(mockCheckStorageQuota).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('resolves when the scope has headroom', async () => {
|
||||
mockCheckStorageQuota.mockResolvedValue({ allowed: true, currentUsage: 10, limit: 100 })
|
||||
|
||||
await expect(
|
||||
assertForkStorageHeadroom({ userId: 'user-1', bytes: 50 })
|
||||
).resolves.toBeUndefined()
|
||||
expect(mockCheckStorageQuota).toHaveBeenCalledWith('user-1', 50)
|
||||
})
|
||||
|
||||
it("throws a 413 ForkError carrying the upload path's quota message when over quota", async () => {
|
||||
mockCheckStorageQuota.mockResolvedValue({
|
||||
allowed: false,
|
||||
currentUsage: 99,
|
||||
limit: 100,
|
||||
error: 'Storage limit exceeded. Used: 10.50GB, Limit: 10GB',
|
||||
})
|
||||
|
||||
const rejection = expect(assertForkStorageHeadroom({ userId: 'user-1', bytes: 50 })).rejects
|
||||
await rejection.toBeInstanceOf(ForkError)
|
||||
await rejection.toMatchObject({
|
||||
statusCode: 413,
|
||||
message:
|
||||
'Not enough storage to copy the selected resources. Storage limit exceeded. Used: 10.50GB, Limit: 10GB',
|
||||
})
|
||||
})
|
||||
|
||||
it('falls back to a generic storage message when the quota helper omits one', async () => {
|
||||
mockCheckStorageQuota.mockResolvedValue({ allowed: false, currentUsage: 0, limit: 0 })
|
||||
|
||||
await expect(assertForkStorageHeadroom({ userId: 'user-1', bytes: 1 })).rejects.toThrow(
|
||||
'Not enough storage to copy the selected resources. Storage limit exceeded'
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,126 @@
|
||||
import { document, knowledgeBase, workspaceFiles } from '@sim/db/schema'
|
||||
import { and, eq, inArray, isNotNull, isNull, or, sql } from 'drizzle-orm'
|
||||
import { checkStorageQuota } from '@/lib/billing/storage'
|
||||
import type { DbOrTx } from '@/lib/db/types'
|
||||
import { ForkError } from '@/ee/workspace-forking/lib/lineage/authz'
|
||||
|
||||
/** Resource ids whose blob bytes a fork/sync copy would duplicate into the target. */
|
||||
export interface ForkCopyBytesSelection {
|
||||
/** Workspace files selected by `workspace_files.id` (the fork modal's picker shape). */
|
||||
fileIds?: string[]
|
||||
/** Workspace files selected by storage key (the sync copy selection shape). */
|
||||
fileKeys?: string[]
|
||||
/** Knowledge bases whose live documents' stored blobs would be re-keyed into the target. */
|
||||
knowledgeBaseIds?: string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Byte total of the workspace-file blobs a copy selection would duplicate. Applies the
|
||||
* same row filters as `planForkFileCopies` (source workspace, durable `workspace`
|
||||
* context, non-deleted, id/key selectors OR'd), so the sum covers exactly the rows the
|
||||
* copy would plan.
|
||||
*/
|
||||
async function sumWorkspaceFileBytes(
|
||||
executor: DbOrTx,
|
||||
sourceWorkspaceId: string,
|
||||
fileIds: string[],
|
||||
fileKeys: string[]
|
||||
): Promise<number> {
|
||||
if (fileIds.length === 0 && fileKeys.length === 0) return 0
|
||||
const selectors = [
|
||||
fileIds.length > 0 ? inArray(workspaceFiles.id, fileIds) : undefined,
|
||||
fileKeys.length > 0 ? inArray(workspaceFiles.key, fileKeys) : undefined,
|
||||
].filter((clause): clause is NonNullable<typeof clause> => clause !== undefined)
|
||||
const rows = await executor
|
||||
.select({ total: sql<string>`coalesce(sum(${workspaceFiles.size}), 0)` })
|
||||
.from(workspaceFiles)
|
||||
.where(
|
||||
and(
|
||||
selectors.length === 1 ? selectors[0] : or(...selectors),
|
||||
eq(workspaceFiles.workspaceId, sourceWorkspaceId),
|
||||
eq(workspaceFiles.context, 'workspace'),
|
||||
isNull(workspaceFiles.deletedAt)
|
||||
)
|
||||
)
|
||||
// `sum()` comes back as a string (bigint) from the driver; coerce explicitly.
|
||||
return Number(rows[0]?.total ?? 0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Byte total of the KB document blobs the selected knowledge bases would re-key into the
|
||||
* target. Scoped to live KBs in the source workspace (mirroring the container copy) and
|
||||
* to LIVE documents with an internal blob: external/`data:` documents have a null
|
||||
* `storageKey` (no blob is duplicated), and embeddings are DB rows the upload path never
|
||||
* counts, so neither contributes bytes here.
|
||||
*/
|
||||
async function sumKbDocumentBytes(
|
||||
executor: DbOrTx,
|
||||
sourceWorkspaceId: string,
|
||||
knowledgeBaseIds: string[]
|
||||
): Promise<number> {
|
||||
if (knowledgeBaseIds.length === 0) return 0
|
||||
const rows = await executor
|
||||
.select({ total: sql<string>`coalesce(sum(${document.fileSize}), 0)` })
|
||||
.from(document)
|
||||
.innerJoin(knowledgeBase, eq(document.knowledgeBaseId, knowledgeBase.id))
|
||||
.where(
|
||||
and(
|
||||
inArray(knowledgeBase.id, knowledgeBaseIds),
|
||||
eq(knowledgeBase.workspaceId, sourceWorkspaceId),
|
||||
isNull(knowledgeBase.deletedAt),
|
||||
isNull(document.deletedAt),
|
||||
isNull(document.archivedAt),
|
||||
isNotNull(document.storageKey)
|
||||
)
|
||||
)
|
||||
return Number(rows[0]?.total ?? 0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Byte total a fork/sync copy selection would duplicate into the target: selected
|
||||
* workspace-file blobs plus the selected knowledge bases' stored document blobs. Sizes
|
||||
* come from the metadata rows (`workspace_files.size`, `document.file_size`) - no blob
|
||||
* reads. Both sums scope to the source workspace with the same filters the copy itself
|
||||
* applies, so an id that is not actually copyable can only over-count (block), never
|
||||
* under-count.
|
||||
*/
|
||||
export async function sumForkCopyBytes(
|
||||
executor: DbOrTx,
|
||||
sourceWorkspaceId: string,
|
||||
selection: ForkCopyBytesSelection
|
||||
): Promise<number> {
|
||||
const fileBytes = await sumWorkspaceFileBytes(
|
||||
executor,
|
||||
sourceWorkspaceId,
|
||||
selection.fileIds ?? [],
|
||||
selection.fileKeys ?? []
|
||||
)
|
||||
const kbBytes = await sumKbDocumentBytes(
|
||||
executor,
|
||||
sourceWorkspaceId,
|
||||
selection.knowledgeBaseIds ?? []
|
||||
)
|
||||
return fileBytes + kbBytes
|
||||
}
|
||||
|
||||
/**
|
||||
* Assert the initiating user's storage scope has headroom for `bytes` of copied blobs,
|
||||
* using the exact quota helper the upload path uses (`checkStorageQuota`, which resolves
|
||||
* the org-pooled vs personal scope from the user's subscription and always allows when
|
||||
* billing is disabled). Over quota throws a {@link ForkError} (413, matching the upload
|
||||
* routes' storage-limit status) carrying the upload path's quota error message, so the
|
||||
* fork/sync modals surface the same user-facing text an over-quota upload would.
|
||||
*/
|
||||
export async function assertForkStorageHeadroom(params: {
|
||||
userId: string
|
||||
bytes: number
|
||||
}): Promise<void> {
|
||||
const { userId, bytes } = params
|
||||
if (bytes <= 0) return
|
||||
const quota = await checkStorageQuota(userId, bytes)
|
||||
if (quota.allowed) return
|
||||
throw new ForkError(
|
||||
`Not enough storage to copy the selected resources. ${quota.error ?? 'Storage limit exceeded'}`,
|
||||
413
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { buildForkWorkflowIdMap } from '@/ee/workspace-forking/lib/copy/workflow-id-map'
|
||||
|
||||
describe('buildForkWorkflowIdMap', () => {
|
||||
const sequentialIds = () => {
|
||||
let n = 0
|
||||
return () => `child-${++n}`
|
||||
}
|
||||
|
||||
it('excludes a deployed source whose state failed to load from the map (and so the identity seed)', () => {
|
||||
const deployed = [{ id: 'wf-a' }, { id: 'wf-b' }, { id: 'wf-c' }]
|
||||
// wf-b's deployed state failed to load - the copy loop skips it.
|
||||
const map = buildForkWorkflowIdMap(deployed, new Set(['wf-a', 'wf-c']), sequentialIds())
|
||||
// wf-b is absent, so a copied workflow's ref to it clears (not dangle) and the identity seed
|
||||
// (derived from this map's entries) never gets an orphan row pointing at a never-created child.
|
||||
expect([...map.keys()]).toEqual(['wf-a', 'wf-c'])
|
||||
expect(map.has('wf-b')).toBe(false)
|
||||
expect(map.get('wf-a')).toBe('child-1')
|
||||
expect(map.get('wf-c')).toBe('child-2')
|
||||
})
|
||||
|
||||
it('maps a both-deployed pair when both states loaded (refs remap, not clear)', () => {
|
||||
const deployed = [{ id: 'parent-wf' }, { id: 'child-wf' }]
|
||||
const map = buildForkWorkflowIdMap(
|
||||
deployed,
|
||||
new Set(['parent-wf', 'child-wf']),
|
||||
sequentialIds()
|
||||
)
|
||||
expect([...map.keys()]).toEqual(['parent-wf', 'child-wf'])
|
||||
expect(map.get('parent-wf')).toBe('child-1')
|
||||
expect(map.get('child-wf')).toBe('child-2')
|
||||
})
|
||||
|
||||
it('returns an empty map when no states loaded', () => {
|
||||
const map = buildForkWorkflowIdMap([{ id: 'wf-a' }], new Set(), () => 'x')
|
||||
expect(map.size).toBe(0)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,21 @@
|
||||
import { generateId } from '@sim/utils/id'
|
||||
|
||||
/**
|
||||
* Build the source->child workflow id map for a fork create, scoped to the deployed workflows whose
|
||||
* state actually LOADED (the set the copy loop writes). A deployed source whose state failed to load
|
||||
* is EXCLUDED, so a copied workflow's reference to it clears (clearUnmapped) instead of pointing at a
|
||||
* never-created child, and no orphan `workspace_fork_resource_map` identity row is seeded for it (the
|
||||
* identity seed is derived from this map). Mirrors promote's writtenItems-only scoping
|
||||
* (`buildPromoteWorkflowIdMap`). `generateChildId` is injectable for deterministic tests.
|
||||
*/
|
||||
export function buildForkWorkflowIdMap(
|
||||
deployedWorkflows: ReadonlyArray<{ id: string }>,
|
||||
loadedStateWorkflowIds: ReadonlySet<string>,
|
||||
generateChildId: () => string = generateId
|
||||
): Map<string, string> {
|
||||
const map = new Map<string, string>()
|
||||
for (const wf of deployedWorkflows) {
|
||||
if (loadedStateWorkflowIds.has(wf.id)) map.set(wf.id, generateChildId())
|
||||
}
|
||||
return map
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const { mockGetEdgeMappingRows, mockAcquireLock } = vi.hoisted(() => ({
|
||||
mockGetEdgeMappingRows: vi.fn(),
|
||||
mockAcquireLock: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/ee/workspace-forking/lib/mapping/mapping-store', () => ({
|
||||
getEdgeMappingRows: mockGetEdgeMappingRows,
|
||||
}))
|
||||
vi.mock('@/lib/mcp/server-locks', () => ({
|
||||
acquireWorkflowMcpServerLock: mockAcquireLock,
|
||||
}))
|
||||
|
||||
import type { DbOrTx } from '@/lib/db/types'
|
||||
import {
|
||||
copyForkWorkflowMcpAttachments,
|
||||
reconcileForkWorkflowMcpAttachments,
|
||||
} from '@/ee/workspace-forking/lib/copy/workflow-mcp-attachments'
|
||||
|
||||
/** Sequenced select mock + captured inserts/updates. */
|
||||
function makeTx(selectResults: unknown[][]) {
|
||||
const inserted: Array<Record<string, unknown>> = []
|
||||
const updates: Array<Record<string, unknown>> = []
|
||||
let call = 0
|
||||
const select = vi.fn(() => ({
|
||||
from: () => ({ where: () => Promise.resolve(selectResults[call++] ?? []) }),
|
||||
}))
|
||||
const tx = {
|
||||
select,
|
||||
insert: () => ({
|
||||
values: (values: Array<Record<string, unknown>>) => {
|
||||
inserted.push(...values)
|
||||
return Promise.resolve()
|
||||
},
|
||||
}),
|
||||
update: () => ({
|
||||
set: (set: Record<string, unknown>) => ({
|
||||
where: () => {
|
||||
updates.push(set)
|
||||
return Promise.resolve()
|
||||
},
|
||||
}),
|
||||
}),
|
||||
}
|
||||
return { tx: tx as unknown as DbOrTx, inserted, updates, select }
|
||||
}
|
||||
|
||||
const attachment = (overrides: Record<string, unknown> = {}) => ({
|
||||
serverId: 'srv-src',
|
||||
workflowId: 'wf-src',
|
||||
toolName: 'run_support_flow',
|
||||
toolDescription: 'Runs the support flow',
|
||||
parameterSchema: { type: 'object', properties: {} },
|
||||
parameterDescriptionOverrides: {},
|
||||
...overrides,
|
||||
})
|
||||
|
||||
const serverMappingRow = {
|
||||
id: 'map-1',
|
||||
childWorkspaceId: 'child-ws',
|
||||
resourceType: 'workflow_mcp_server' as const,
|
||||
parentResourceId: 'srv-parent',
|
||||
childResourceId: 'srv-child',
|
||||
}
|
||||
|
||||
describe('reconcileForkWorkflowMcpAttachments', () => {
|
||||
it('creates the target attachment for a mapped server + written pair (push: child -> parent)', async () => {
|
||||
mockGetEdgeMappingRows.mockResolvedValue([serverMappingRow])
|
||||
mockAcquireLock.mockClear()
|
||||
const { tx, inserted, select } = makeTx([
|
||||
[{ id: 'srv-child' }, { id: 'srv-parent' }], // both mapped servers still live
|
||||
[attachment({ serverId: 'srv-child', workflowId: 'wf-child' })],
|
||||
[], // no existing target attachments
|
||||
])
|
||||
const result = await reconcileForkWorkflowMcpAttachments({
|
||||
tx,
|
||||
childWorkspaceId: 'child-ws',
|
||||
sourceIsParent: false, // push: source is the child
|
||||
now: new Date(),
|
||||
writtenPairs: [{ sourceWorkflowId: 'wf-child', targetWorkflowId: 'wf-parent' }],
|
||||
})
|
||||
expect(inserted).toHaveLength(1)
|
||||
expect(inserted[0]).toMatchObject({
|
||||
serverId: 'srv-parent',
|
||||
workflowId: 'wf-parent',
|
||||
toolName: 'run_support_flow',
|
||||
})
|
||||
expect(result.affectedServerIds).toEqual(['srv-parent'])
|
||||
expect(mockAcquireLock).toHaveBeenCalledWith(tx, 'srv-parent')
|
||||
// The lock must precede every read: locking after the diff is computed would let a
|
||||
// concurrent attach commit in between and abort the promote on the unique constraint.
|
||||
expect(mockAcquireLock.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
select.mock.invocationCallOrder[0]
|
||||
)
|
||||
})
|
||||
|
||||
it('archives a target attachment whose source counterpart was detached', async () => {
|
||||
mockGetEdgeMappingRows.mockResolvedValue([serverMappingRow])
|
||||
const { tx, inserted, updates } = makeTx([
|
||||
[{ id: 'srv-child' }, { id: 'srv-parent' }], // both mapped servers still live
|
||||
[], // source has no attachments left
|
||||
[
|
||||
{
|
||||
id: 'tool-tgt',
|
||||
serverId: 'srv-parent',
|
||||
workflowId: 'wf-parent',
|
||||
toolName: 'run_support_flow',
|
||||
toolDescription: null,
|
||||
parameterDescriptionOverrides: {},
|
||||
},
|
||||
],
|
||||
])
|
||||
const result = await reconcileForkWorkflowMcpAttachments({
|
||||
tx,
|
||||
childWorkspaceId: 'child-ws',
|
||||
sourceIsParent: false,
|
||||
now: new Date(),
|
||||
writtenPairs: [{ sourceWorkflowId: 'wf-child', targetWorkflowId: 'wf-parent' }],
|
||||
})
|
||||
expect(inserted).toHaveLength(0)
|
||||
expect(updates).toHaveLength(1)
|
||||
expect(updates[0].archivedAt).toBeInstanceOf(Date)
|
||||
expect(result.affectedServerIds).toEqual(['srv-parent'])
|
||||
})
|
||||
|
||||
it('refreshes drifted metadata on an existing target attachment', async () => {
|
||||
mockGetEdgeMappingRows.mockResolvedValue([serverMappingRow])
|
||||
const { tx, updates } = makeTx([
|
||||
[{ id: 'srv-child' }, { id: 'srv-parent' }], // both mapped servers still live
|
||||
[attachment({ serverId: 'srv-child', workflowId: 'wf-child', toolName: 'renamed_tool' })],
|
||||
[
|
||||
{
|
||||
id: 'tool-tgt',
|
||||
serverId: 'srv-parent',
|
||||
workflowId: 'wf-parent',
|
||||
toolName: 'run_support_flow',
|
||||
toolDescription: 'Runs the support flow',
|
||||
parameterDescriptionOverrides: {},
|
||||
},
|
||||
],
|
||||
])
|
||||
await reconcileForkWorkflowMcpAttachments({
|
||||
tx,
|
||||
childWorkspaceId: 'child-ws',
|
||||
sourceIsParent: false,
|
||||
now: new Date(),
|
||||
writtenPairs: [{ sourceWorkflowId: 'wf-child', targetWorkflowId: 'wf-parent' }],
|
||||
})
|
||||
expect(updates).toHaveLength(1)
|
||||
expect(updates[0].toolName).toBe('renamed_tool')
|
||||
})
|
||||
|
||||
it('skips a mapped pair whose target server was deleted (stale identity row must never FK-crash the sync)', async () => {
|
||||
mockGetEdgeMappingRows.mockResolvedValue([serverMappingRow])
|
||||
const { tx, inserted } = makeTx([
|
||||
[{ id: 'srv-child' }], // target server srv-parent hard-deleted: only the source is live
|
||||
])
|
||||
const result = await reconcileForkWorkflowMcpAttachments({
|
||||
tx,
|
||||
childWorkspaceId: 'child-ws',
|
||||
sourceIsParent: false,
|
||||
now: new Date(),
|
||||
writtenPairs: [{ sourceWorkflowId: 'wf-child', targetWorkflowId: 'wf-parent' }],
|
||||
})
|
||||
expect(inserted).toHaveLength(0)
|
||||
expect(result.affectedServerIds).toEqual([])
|
||||
})
|
||||
|
||||
it('no-ops with no mapped servers (attachments follow the server identity)', async () => {
|
||||
mockGetEdgeMappingRows.mockResolvedValue([
|
||||
{ ...serverMappingRow, resourceType: 'table' as const },
|
||||
])
|
||||
const { tx, inserted } = makeTx([])
|
||||
const result = await reconcileForkWorkflowMcpAttachments({
|
||||
tx,
|
||||
childWorkspaceId: 'child-ws',
|
||||
sourceIsParent: false,
|
||||
now: new Date(),
|
||||
writtenPairs: [{ sourceWorkflowId: 'wf-child', targetWorkflowId: 'wf-parent' }],
|
||||
})
|
||||
expect(inserted).toHaveLength(0)
|
||||
expect(result.affectedServerIds).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('copyForkWorkflowMcpAttachments', () => {
|
||||
it('copies an attachment only when BOTH its server and workflow were copied', async () => {
|
||||
const { tx, inserted } = makeTx([
|
||||
[
|
||||
attachment(), // both mapped
|
||||
attachment({ serverId: 'srv-uncopied', workflowId: 'wf-src' }),
|
||||
attachment({ serverId: 'srv-src', workflowId: 'wf-uncopied' }),
|
||||
],
|
||||
])
|
||||
const result = await copyForkWorkflowMcpAttachments({
|
||||
tx,
|
||||
serverIdMap: new Map([['srv-src', 'srv-copy']]),
|
||||
workflowIdMap: new Map([['wf-src', 'wf-copy']]),
|
||||
now: new Date(),
|
||||
})
|
||||
expect(result.copied).toBe(1)
|
||||
expect(inserted[0]).toMatchObject({
|
||||
serverId: 'srv-copy',
|
||||
workflowId: 'wf-copy',
|
||||
toolName: 'run_support_flow',
|
||||
})
|
||||
})
|
||||
|
||||
it('no-ops when either id map is empty', async () => {
|
||||
const { tx, inserted } = makeTx([])
|
||||
const result = await copyForkWorkflowMcpAttachments({
|
||||
tx,
|
||||
serverIdMap: new Map(),
|
||||
workflowIdMap: new Map([['wf-src', 'wf-copy']]),
|
||||
now: new Date(),
|
||||
})
|
||||
expect(result.copied).toBe(0)
|
||||
expect(inserted).toHaveLength(0)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,296 @@
|
||||
import { workflowMcpServer, workflowMcpTool } from '@sim/db/schema'
|
||||
import { generateId } from '@sim/utils/id'
|
||||
import { and, eq, inArray, isNull } from 'drizzle-orm'
|
||||
import type { DbOrTx } from '@/lib/db/types'
|
||||
import { acquireWorkflowMcpServerLock } from '@/lib/mcp/server-locks'
|
||||
import { validateMcpToolMetadataForStorage } from '@/lib/mcp/tool-limits'
|
||||
import { getEdgeMappingRows } from '@/ee/workspace-forking/lib/mapping/mapping-store'
|
||||
|
||||
/**
|
||||
* The seed `parameterSchema` for a copied attachment. The source's schema is copied so the tool
|
||||
* serves correctly before the target's first deploy, UNLESS it exceeds the per-tool storage
|
||||
* limit - then the empty schema is seeded instead (the same degradation the deploy-time sync
|
||||
* applies) and the deployment outbox re-derives the real one when the target deploys.
|
||||
*/
|
||||
function seedParameterSchema(parameterSchema: unknown): unknown {
|
||||
const invalid = validateMcpToolMetadataForStorage({
|
||||
parameterSchema: parameterSchema as Record<string, unknown>,
|
||||
})
|
||||
return invalid ? { type: 'object', properties: {} } : parameterSchema
|
||||
}
|
||||
|
||||
export interface ForkMcpAttachmentPair {
|
||||
sourceWorkflowId: string
|
||||
targetWorkflowId: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy `workflow_mcp_tool` attachments into a fresh fork: every source attachment whose server
|
||||
* AND workflow were both copied gets a child row (fresh id; metadata + schema copied - the child
|
||||
* re-derives the schema when it first deploys). Insert-only: the child is brand new, so there is
|
||||
* nothing to update or archive, and no server locks are needed (the child's servers are
|
||||
* invisible until the fork transaction commits). Must run AFTER the child workflow rows exist
|
||||
* (FK). A no-op when either map is empty.
|
||||
*/
|
||||
export async function copyForkWorkflowMcpAttachments(params: {
|
||||
tx: DbOrTx
|
||||
/** Source workflow-publishing server id -> child copy id. */
|
||||
serverIdMap: ReadonlyMap<string, string>
|
||||
/** Source workflow id -> child workflow id. */
|
||||
workflowIdMap: ReadonlyMap<string, string>
|
||||
now: Date
|
||||
}): Promise<{ copied: number }> {
|
||||
const { tx, serverIdMap, workflowIdMap, now } = params
|
||||
if (serverIdMap.size === 0 || workflowIdMap.size === 0) return { copied: 0 }
|
||||
|
||||
const sourceAttachments = await tx
|
||||
.select({
|
||||
serverId: workflowMcpTool.serverId,
|
||||
workflowId: workflowMcpTool.workflowId,
|
||||
toolName: workflowMcpTool.toolName,
|
||||
toolDescription: workflowMcpTool.toolDescription,
|
||||
parameterSchema: workflowMcpTool.parameterSchema,
|
||||
parameterDescriptionOverrides: workflowMcpTool.parameterDescriptionOverrides,
|
||||
})
|
||||
.from(workflowMcpTool)
|
||||
.where(
|
||||
and(
|
||||
inArray(workflowMcpTool.serverId, [...serverIdMap.keys()]),
|
||||
inArray(workflowMcpTool.workflowId, [...workflowIdMap.keys()]),
|
||||
isNull(workflowMcpTool.archivedAt)
|
||||
)
|
||||
)
|
||||
|
||||
const inserts: (typeof workflowMcpTool.$inferInsert)[] = []
|
||||
for (const attachment of sourceAttachments) {
|
||||
const childServerId = serverIdMap.get(attachment.serverId)
|
||||
const childWorkflowId = workflowIdMap.get(attachment.workflowId)
|
||||
if (!childServerId || !childWorkflowId) continue
|
||||
inserts.push({
|
||||
id: generateId(),
|
||||
serverId: childServerId,
|
||||
workflowId: childWorkflowId,
|
||||
toolName: attachment.toolName,
|
||||
toolDescription: attachment.toolDescription,
|
||||
parameterSchema: seedParameterSchema(attachment.parameterSchema),
|
||||
parameterDescriptionOverrides: attachment.parameterDescriptionOverrides,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
}
|
||||
if (inserts.length > 0) await tx.insert(workflowMcpTool).values(inserts)
|
||||
return { copied: inserts.length }
|
||||
}
|
||||
|
||||
/**
|
||||
* Mirror `workflow_mcp_tool` attachments (a workflow exposed as a tool on a
|
||||
* workflow-publishing MCP server) onto the target side of a sync, through the edge's
|
||||
* `workflow_mcp_server` identity map (seeded when a fork copies the server shells).
|
||||
*
|
||||
* For each written workflow pair whose source is attached to a MAPPED server:
|
||||
* - a missing target attachment is created (metadata copied; `parameterSchema` is copied as a
|
||||
* seed and re-derived by the deployment outbox when the target deploys),
|
||||
* - an existing one has its user-set metadata (tool name / description / description
|
||||
* overrides) refreshed to the source's,
|
||||
* - a target attachment on a mapped server + synced workflow with NO source counterpart is
|
||||
* archived (the source detached it) - target attachments on unmapped servers or unsynced
|
||||
* workflows are never touched.
|
||||
*
|
||||
* Unmapped servers are skipped entirely: attachment sync follows the server identity, exactly
|
||||
* like subblock references follow resource mappings. Bounded by (written workflows x mapped
|
||||
* servers); acquires the same per-server advisory locks the deploy-time tool sync takes.
|
||||
* Returns the affected target server ids so the caller can notify them post-commit.
|
||||
*/
|
||||
export async function reconcileForkWorkflowMcpAttachments(params: {
|
||||
tx: DbOrTx
|
||||
childWorkspaceId: string
|
||||
/** True when the sync SOURCE is the parent workspace (a pull). */
|
||||
sourceIsParent: boolean
|
||||
now: Date
|
||||
/** The workflow pairs THIS sync wrote (replace + create). */
|
||||
writtenPairs: ForkMcpAttachmentPair[]
|
||||
}): Promise<{ affectedServerIds: string[] }> {
|
||||
const { tx, childWorkspaceId, sourceIsParent, now, writtenPairs } = params
|
||||
if (writtenPairs.length === 0) return { affectedServerIds: [] }
|
||||
|
||||
const mappingRows = await getEdgeMappingRows(tx, childWorkspaceId)
|
||||
const serverMap = new Map<string, string>()
|
||||
for (const row of mappingRows) {
|
||||
if (row.resourceType !== 'workflow_mcp_server' || row.childResourceId == null) continue
|
||||
if (sourceIsParent) serverMap.set(row.parentResourceId, row.childResourceId)
|
||||
else serverMap.set(row.childResourceId, row.parentResourceId)
|
||||
}
|
||||
if (serverMap.size === 0) return { affectedServerIds: [] }
|
||||
|
||||
// Same per-server serialization as the deploy-time tool sync and the attach/delete routes,
|
||||
// in sorted order so two concurrent syncs can't deadlock on each other's server locks.
|
||||
// Acquired BEFORE the reads below: every other attachment writer locks first, so locking
|
||||
// after computing the diff would let a concurrent attach commit in between and turn our
|
||||
// insert into a unique-constraint abort of the whole promote transaction (and a concurrent
|
||||
// server delete into an FK abort).
|
||||
for (const serverId of [...new Set(serverMap.values())].sort()) {
|
||||
await acquireWorkflowMcpServerLock(tx, serverId)
|
||||
}
|
||||
|
||||
// Liveness guard: a mapped server may have been deleted since the fork (server deletion is a
|
||||
// hard delete that cascades its tools but leaves the identity row). A dead SOURCE server has
|
||||
// nothing to mirror; a dead TARGET server must be skipped or the insert below would violate
|
||||
// the `server_id` FK and abort the whole promote transaction. Target liveness is stable for
|
||||
// the rest of the transaction: the delete route takes the per-server lock we now hold.
|
||||
const mappedServerIds = [...new Set([...serverMap.keys(), ...serverMap.values()])]
|
||||
const liveServerIds = new Set(
|
||||
(
|
||||
await tx
|
||||
.select({ id: workflowMcpServer.id })
|
||||
.from(workflowMcpServer)
|
||||
.where(
|
||||
and(inArray(workflowMcpServer.id, mappedServerIds), isNull(workflowMcpServer.deletedAt))
|
||||
)
|
||||
).map((row) => row.id)
|
||||
)
|
||||
for (const [sourceServerId, targetServerId] of serverMap) {
|
||||
if (!liveServerIds.has(sourceServerId) || !liveServerIds.has(targetServerId)) {
|
||||
serverMap.delete(sourceServerId)
|
||||
}
|
||||
}
|
||||
if (serverMap.size === 0) return { affectedServerIds: [] }
|
||||
|
||||
const targetBySource = new Map(
|
||||
writtenPairs.map((pair) => [pair.sourceWorkflowId, pair.targetWorkflowId])
|
||||
)
|
||||
const sourceAttachments = await tx
|
||||
.select({
|
||||
serverId: workflowMcpTool.serverId,
|
||||
workflowId: workflowMcpTool.workflowId,
|
||||
toolName: workflowMcpTool.toolName,
|
||||
toolDescription: workflowMcpTool.toolDescription,
|
||||
parameterSchema: workflowMcpTool.parameterSchema,
|
||||
parameterDescriptionOverrides: workflowMcpTool.parameterDescriptionOverrides,
|
||||
})
|
||||
.from(workflowMcpTool)
|
||||
.where(
|
||||
and(
|
||||
inArray(workflowMcpTool.workflowId, [...targetBySource.keys()]),
|
||||
inArray(workflowMcpTool.serverId, [...serverMap.keys()]),
|
||||
isNull(workflowMcpTool.archivedAt)
|
||||
)
|
||||
)
|
||||
|
||||
/** Desired live target pairs, keyed `${targetServerId}\u0000${targetWorkflowId}`. */
|
||||
const desired = new Map<
|
||||
string,
|
||||
{
|
||||
serverId: string
|
||||
workflowId: string
|
||||
toolName: string
|
||||
toolDescription: string | null
|
||||
parameterSchema: unknown
|
||||
parameterDescriptionOverrides: Record<string, string>
|
||||
}
|
||||
>()
|
||||
for (const attachment of sourceAttachments) {
|
||||
const targetServerId = serverMap.get(attachment.serverId)
|
||||
const targetWorkflowId = targetBySource.get(attachment.workflowId)
|
||||
if (!targetServerId || !targetWorkflowId) continue
|
||||
desired.set(`${targetServerId}\u0000${targetWorkflowId}`, {
|
||||
serverId: targetServerId,
|
||||
workflowId: targetWorkflowId,
|
||||
toolName: attachment.toolName,
|
||||
toolDescription: attachment.toolDescription,
|
||||
parameterSchema: attachment.parameterSchema,
|
||||
parameterDescriptionOverrides: attachment.parameterDescriptionOverrides,
|
||||
})
|
||||
}
|
||||
|
||||
// The reconcile scope: every mapped TARGET server x every synced TARGET workflow. Rows
|
||||
// outside this product are never touched.
|
||||
const mappedTargetServerIds = [...new Set(serverMap.values())].sort()
|
||||
const syncedTargetWorkflowIds = [...new Set(targetBySource.values())]
|
||||
const existing = await tx
|
||||
.select({
|
||||
id: workflowMcpTool.id,
|
||||
serverId: workflowMcpTool.serverId,
|
||||
workflowId: workflowMcpTool.workflowId,
|
||||
toolName: workflowMcpTool.toolName,
|
||||
toolDescription: workflowMcpTool.toolDescription,
|
||||
parameterDescriptionOverrides: workflowMcpTool.parameterDescriptionOverrides,
|
||||
})
|
||||
.from(workflowMcpTool)
|
||||
.where(
|
||||
and(
|
||||
inArray(workflowMcpTool.serverId, mappedTargetServerIds),
|
||||
inArray(workflowMcpTool.workflowId, syncedTargetWorkflowIds),
|
||||
isNull(workflowMcpTool.archivedAt)
|
||||
)
|
||||
)
|
||||
const existingByKey = new Map(
|
||||
existing.map((row) => [`${row.serverId}\u0000${row.workflowId}`, row])
|
||||
)
|
||||
|
||||
const inserts: (typeof workflowMcpTool.$inferInsert)[] = []
|
||||
const updates: Array<{ id: string; set: Partial<typeof workflowMcpTool.$inferInsert> }> = []
|
||||
const archiveIds: string[] = []
|
||||
const affectedServerIds = new Set<string>()
|
||||
|
||||
for (const [key, want] of desired) {
|
||||
const current = existingByKey.get(key)
|
||||
if (!current) {
|
||||
inserts.push({
|
||||
id: generateId(),
|
||||
serverId: want.serverId,
|
||||
workflowId: want.workflowId,
|
||||
toolName: want.toolName,
|
||||
toolDescription: want.toolDescription,
|
||||
// Seed with the source's schema; the deployment outbox re-derives it from the
|
||||
// target's deployed state right after this sync deploys the workflow.
|
||||
parameterSchema: seedParameterSchema(want.parameterSchema),
|
||||
parameterDescriptionOverrides: want.parameterDescriptionOverrides,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
affectedServerIds.add(want.serverId)
|
||||
continue
|
||||
}
|
||||
const overridesChanged =
|
||||
JSON.stringify(current.parameterDescriptionOverrides ?? {}) !==
|
||||
JSON.stringify(want.parameterDescriptionOverrides ?? {})
|
||||
if (
|
||||
current.toolName !== want.toolName ||
|
||||
current.toolDescription !== want.toolDescription ||
|
||||
overridesChanged
|
||||
) {
|
||||
updates.push({
|
||||
id: current.id,
|
||||
set: {
|
||||
toolName: want.toolName,
|
||||
toolDescription: want.toolDescription,
|
||||
parameterDescriptionOverrides: want.parameterDescriptionOverrides,
|
||||
updatedAt: now,
|
||||
},
|
||||
})
|
||||
affectedServerIds.add(current.serverId)
|
||||
}
|
||||
}
|
||||
for (const [key, row] of existingByKey) {
|
||||
if (desired.has(key)) continue
|
||||
archiveIds.push(row.id)
|
||||
affectedServerIds.add(row.serverId)
|
||||
}
|
||||
|
||||
if (inserts.length === 0 && updates.length === 0 && archiveIds.length === 0) {
|
||||
return { affectedServerIds: [] }
|
||||
}
|
||||
|
||||
if (inserts.length > 0) await tx.insert(workflowMcpTool).values(inserts)
|
||||
for (const update of updates) {
|
||||
await tx.update(workflowMcpTool).set(update.set).where(eq(workflowMcpTool.id, update.id))
|
||||
}
|
||||
if (archiveIds.length > 0) {
|
||||
await tx
|
||||
.update(workflowMcpTool)
|
||||
.set({ archivedAt: now, updatedAt: now })
|
||||
.where(inArray(workflowMcpTool.id, archiveIds))
|
||||
}
|
||||
|
||||
return { affectedServerIds: [...affectedServerIds] }
|
||||
}
|
||||
Reference in New Issue
Block a user