chore: import upstream snapshot with attribution
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
CI / Migrate Dev DB (push) Has been skipped
CI / Detect Version (push) Has been cancelled
CI / Migrate DB (push) Has been cancelled
CI / Build Dev ECR (./docker/app.Dockerfile, ECR_APP) (push) Has been cancelled
CI / Build Dev ECR (./docker/db.Dockerfile, ECR_MIGRATIONS) (push) Has been cancelled
CI / Build Dev ECR (./docker/pii.Dockerfile, ECR_PII) (push) Has been cancelled
CI / Build Dev ECR (./docker/realtime.Dockerfile, ECR_REALTIME) (push) Has been cancelled
CI / Deploy Trigger.dev (Dev) (push) Has been cancelled
CI / Build AMD64 (./docker/app.Dockerfile, ECR_APP, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build AMD64 (./docker/db.Dockerfile, ECR_MIGRATIONS, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build AMD64 (./docker/pii.Dockerfile, ECR_PII, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build AMD64 (./docker/realtime.Dockerfile, ECR_REALTIME, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/app.Dockerfile, ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/db.Dockerfile, ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/pii.Dockerfile, ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Build ARM64 (GHCR Only) (./docker/realtime.Dockerfile, ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/migrations) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/pii) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/realtime) (push) Has been cancelled
CI / Create GHCR Manifests (ghcr.io/simstudioai/simstudio) (push) Has been cancelled
CI / Check Docs Changes (push) Has been cancelled
CI / Process Docs (push) Has been cancelled
CI / Create GitHub Release (push) Has been cancelled
CI / Test and Build (push) Has been cancelled
Publish CLI Package / publish-npm (push) Has been cancelled
Publish Python SDK / publish-pypi (push) Has been cancelled
Publish TypeScript SDK / publish-npm (push) Has been cancelled
This commit is contained in:
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,228 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { MEMORY } from '@/executor/constants'
|
||||
import { Memory } from '@/executor/handlers/agent/memory'
|
||||
import type { Message } from '@/executor/handlers/agent/types'
|
||||
|
||||
vi.mock('@/lib/tokenization/estimators', () => ({
|
||||
getAccurateTokenCount: vi.fn((text: string) => {
|
||||
return Math.ceil(text.length / 4)
|
||||
}),
|
||||
}))
|
||||
|
||||
describe('Memory', () => {
|
||||
let memoryService: Memory
|
||||
|
||||
beforeEach(() => {
|
||||
memoryService = new Memory()
|
||||
})
|
||||
|
||||
describe('applyWindow (message-based)', () => {
|
||||
it('should keep last N messages', () => {
|
||||
const messages: Message[] = [
|
||||
{ role: 'user', content: 'Message 1' },
|
||||
{ role: 'assistant', content: 'Response 1' },
|
||||
{ role: 'user', content: 'Message 2' },
|
||||
{ role: 'assistant', content: 'Response 2' },
|
||||
{ role: 'user', content: 'Message 3' },
|
||||
{ role: 'assistant', content: 'Response 3' },
|
||||
]
|
||||
|
||||
const result = (memoryService as any).applyWindow(messages, 4)
|
||||
|
||||
expect(result.length).toBe(4)
|
||||
expect(result[0].content).toBe('Message 2')
|
||||
expect(result[3].content).toBe('Response 3')
|
||||
})
|
||||
|
||||
it('should return all messages if limit exceeds array length', () => {
|
||||
const messages: Message[] = [
|
||||
{ role: 'user', content: 'Test' },
|
||||
{ role: 'assistant', content: 'Response' },
|
||||
]
|
||||
|
||||
const result = (memoryService as any).applyWindow(messages, 10)
|
||||
expect(result.length).toBe(2)
|
||||
})
|
||||
|
||||
it('should handle invalid window size', () => {
|
||||
const messages: Message[] = [{ role: 'user', content: 'Test' }]
|
||||
|
||||
const result = (memoryService as any).applyWindow(messages, Number.NaN)
|
||||
expect(result).toEqual(messages)
|
||||
})
|
||||
|
||||
it('should handle zero limit', () => {
|
||||
const messages: Message[] = [{ role: 'user', content: 'Test' }]
|
||||
|
||||
const result = (memoryService as any).applyWindow(messages, 0)
|
||||
expect(result).toEqual(messages)
|
||||
})
|
||||
})
|
||||
|
||||
describe('applyTokenWindow (token-based)', () => {
|
||||
it('should keep messages within token limit', () => {
|
||||
const messages: Message[] = [
|
||||
{ role: 'user', content: 'Short' },
|
||||
{ role: 'assistant', content: 'This is a longer response message' },
|
||||
{ role: 'user', content: 'Another user message here' },
|
||||
{ role: 'assistant', content: 'Final response' },
|
||||
]
|
||||
|
||||
const result = (memoryService as any).applyTokenWindow(messages, 15, 'gpt-4o')
|
||||
|
||||
expect(result.length).toBeGreaterThan(0)
|
||||
expect(result.length).toBeLessThan(messages.length)
|
||||
expect(result[result.length - 1].content).toBe('Final response')
|
||||
})
|
||||
|
||||
it('should include at least 1 message even if it exceeds limit', () => {
|
||||
const messages: Message[] = [
|
||||
{
|
||||
role: 'user',
|
||||
content:
|
||||
'This is a very long message that definitely exceeds our small token limit of just 5 tokens',
|
||||
},
|
||||
]
|
||||
|
||||
const result = (memoryService as any).applyTokenWindow(messages, 5, 'gpt-4o')
|
||||
|
||||
expect(result.length).toBe(1)
|
||||
expect(result[0].content).toBe(messages[0].content)
|
||||
})
|
||||
|
||||
it('should process messages from newest to oldest', () => {
|
||||
const messages: Message[] = [
|
||||
{ role: 'user', content: 'Old message' },
|
||||
{ role: 'assistant', content: 'Old response' },
|
||||
{ role: 'user', content: 'New message' },
|
||||
{ role: 'assistant', content: 'New response' },
|
||||
]
|
||||
|
||||
const result = (memoryService as any).applyTokenWindow(messages, 10, 'gpt-4o')
|
||||
|
||||
expect(result[result.length - 1].content).toBe('New response')
|
||||
})
|
||||
|
||||
it('should handle invalid token limit', () => {
|
||||
const messages: Message[] = [{ role: 'user', content: 'Test' }]
|
||||
|
||||
const result = (memoryService as any).applyTokenWindow(messages, Number.NaN, 'gpt-4o')
|
||||
expect(result).toEqual(messages)
|
||||
})
|
||||
|
||||
it('should handle zero or negative token limit', () => {
|
||||
const messages: Message[] = [{ role: 'user', content: 'Test' }]
|
||||
|
||||
const result1 = (memoryService as any).applyTokenWindow(messages, 0, 'gpt-4o')
|
||||
expect(result1).toEqual(messages)
|
||||
|
||||
const result2 = (memoryService as any).applyTokenWindow(messages, -5, 'gpt-4o')
|
||||
expect(result2).toEqual(messages)
|
||||
})
|
||||
|
||||
it('should work without model specified', () => {
|
||||
const messages: Message[] = [{ role: 'user', content: 'Test message' }]
|
||||
|
||||
const result = (memoryService as any).applyTokenWindow(messages, 100, undefined)
|
||||
expect(result.length).toBe(1)
|
||||
})
|
||||
|
||||
it('should handle empty messages array', () => {
|
||||
const messages: Message[] = []
|
||||
|
||||
const result = (memoryService as any).applyTokenWindow(messages, 100, 'gpt-4o')
|
||||
expect(result).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('validateConversationId', () => {
|
||||
it('should throw error for missing conversationId', () => {
|
||||
expect(() => {
|
||||
;(memoryService as any).validateConversationId(undefined)
|
||||
}).toThrow('Conversation ID is required')
|
||||
})
|
||||
|
||||
it('should throw error for empty conversationId', () => {
|
||||
expect(() => {
|
||||
;(memoryService as any).validateConversationId(' ')
|
||||
}).toThrow('Conversation ID is required')
|
||||
})
|
||||
|
||||
it('should throw error for too long conversationId', () => {
|
||||
const longId = 'a'.repeat(MEMORY.MAX_CONVERSATION_ID_LENGTH + 1)
|
||||
expect(() => {
|
||||
;(memoryService as any).validateConversationId(longId)
|
||||
}).toThrow('Conversation ID too long')
|
||||
})
|
||||
|
||||
it('should accept valid conversationId', () => {
|
||||
expect(() => {
|
||||
;(memoryService as any).validateConversationId('user-123')
|
||||
}).not.toThrow()
|
||||
})
|
||||
})
|
||||
|
||||
describe('validateContent', () => {
|
||||
it('should throw error for content exceeding max size', () => {
|
||||
const largeContent = 'x'.repeat(MEMORY.MAX_MESSAGE_CONTENT_BYTES + 1)
|
||||
expect(() => {
|
||||
;(memoryService as any).validateContent(largeContent)
|
||||
}).toThrow('Message content too large')
|
||||
})
|
||||
|
||||
it('should accept content within limit', () => {
|
||||
const content = 'Normal sized content'
|
||||
expect(() => {
|
||||
;(memoryService as any).validateContent(content)
|
||||
}).not.toThrow()
|
||||
})
|
||||
})
|
||||
|
||||
describe('sanitizeMessageForStorage', () => {
|
||||
it('should strip file payloads but preserve tool-call fields before memory persistence', () => {
|
||||
const message: Message = {
|
||||
role: 'user',
|
||||
content: 'Analyze this file',
|
||||
executionId: 'exec-1',
|
||||
files: [
|
||||
{
|
||||
id: 'file-1',
|
||||
key: 'workspace/ws-1/example.png',
|
||||
name: 'example.png',
|
||||
url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace',
|
||||
size: 128,
|
||||
type: 'image/png',
|
||||
base64: 'iVBORw0KGgo=',
|
||||
},
|
||||
],
|
||||
tool_calls: [{ id: 'call-1' }],
|
||||
}
|
||||
|
||||
expect((memoryService as any).sanitizeMessageForStorage(message)).toEqual({
|
||||
role: 'user',
|
||||
content: 'Analyze this file',
|
||||
executionId: 'exec-1',
|
||||
tool_calls: [{ id: 'call-1' }],
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('Token-based vs Message-based comparison', () => {
|
||||
it('should produce different results for same limit concept', () => {
|
||||
const messages: Message[] = [
|
||||
{ role: 'user', content: 'A' },
|
||||
{
|
||||
role: 'assistant',
|
||||
content: 'This is a much longer response that takes many more tokens',
|
||||
},
|
||||
{ role: 'user', content: 'B' },
|
||||
]
|
||||
|
||||
const messageResult = (memoryService as any).applyWindow(messages, 2)
|
||||
expect(messageResult.length).toBe(2)
|
||||
|
||||
const tokenResult = (memoryService as any).applyTokenWindow(messages, 10, 'gpt-4o')
|
||||
expect(tokenResult.length).toBeGreaterThanOrEqual(1)
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,300 @@
|
||||
import { db } from '@sim/db'
|
||||
import { memory } from '@sim/db/schema'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { generateId } from '@sim/utils/id'
|
||||
import { and, eq, sql } from 'drizzle-orm'
|
||||
import { redactObjectStrings } from '@/lib/logs/execution/pii-redaction'
|
||||
import { getAccurateTokenCount } from '@/lib/tokenization/estimators'
|
||||
import { MEMORY } from '@/executor/constants'
|
||||
import type { AgentInputs, Message } from '@/executor/handlers/agent/types'
|
||||
import type { ExecutionContext } from '@/executor/types'
|
||||
import { PROVIDER_DEFINITIONS } from '@/providers/models'
|
||||
|
||||
const logger = createLogger('Memory')
|
||||
|
||||
export class Memory {
|
||||
async fetchMemoryMessages(ctx: ExecutionContext, inputs: AgentInputs): Promise<Message[]> {
|
||||
if (!inputs.memoryType || inputs.memoryType === 'none') {
|
||||
return []
|
||||
}
|
||||
|
||||
const workspaceId = this.requireWorkspaceId(ctx)
|
||||
this.validateConversationId(inputs.conversationId)
|
||||
|
||||
const messages = await this.fetchMemory(workspaceId, inputs.conversationId!)
|
||||
|
||||
switch (inputs.memoryType) {
|
||||
case 'conversation':
|
||||
return this.applyContextWindowLimit(messages, inputs.model)
|
||||
|
||||
case 'sliding_window': {
|
||||
const limit = this.parsePositiveInt(
|
||||
inputs.slidingWindowSize,
|
||||
MEMORY.DEFAULT_SLIDING_WINDOW_SIZE
|
||||
)
|
||||
return this.applyWindow(messages, limit)
|
||||
}
|
||||
|
||||
case 'sliding_window_tokens': {
|
||||
const maxTokens = this.parsePositiveInt(
|
||||
inputs.slidingWindowTokens,
|
||||
MEMORY.DEFAULT_SLIDING_WINDOW_TOKENS
|
||||
)
|
||||
return this.applyTokenWindow(messages, maxTokens, inputs.model)
|
||||
}
|
||||
|
||||
default:
|
||||
return messages
|
||||
}
|
||||
}
|
||||
|
||||
async appendToMemory(
|
||||
ctx: ExecutionContext,
|
||||
inputs: AgentInputs,
|
||||
message: Message
|
||||
): Promise<void> {
|
||||
if (!inputs.memoryType || inputs.memoryType === 'none') {
|
||||
return
|
||||
}
|
||||
|
||||
const workspaceId = this.requireWorkspaceId(ctx)
|
||||
this.validateConversationId(inputs.conversationId)
|
||||
|
||||
message = await this.maskContentForStorage(ctx, message)
|
||||
|
||||
this.validateContent(message.content)
|
||||
|
||||
const key = inputs.conversationId!
|
||||
|
||||
await this.appendMessage(workspaceId, key, message)
|
||||
|
||||
logger.debug('Appended message to memory', {
|
||||
workspaceId,
|
||||
key,
|
||||
role: message.role,
|
||||
})
|
||||
}
|
||||
|
||||
async seedMemory(ctx: ExecutionContext, inputs: AgentInputs, messages: Message[]): Promise<void> {
|
||||
if (!inputs.memoryType || inputs.memoryType === 'none') {
|
||||
return
|
||||
}
|
||||
|
||||
const workspaceId = this.requireWorkspaceId(ctx)
|
||||
|
||||
const conversationMessages = messages.filter((m) => m.role !== 'system')
|
||||
if (conversationMessages.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
this.validateConversationId(inputs.conversationId)
|
||||
|
||||
const key = inputs.conversationId!
|
||||
|
||||
let messagesToStore = conversationMessages
|
||||
if (inputs.memoryType === 'sliding_window') {
|
||||
const limit = this.parsePositiveInt(
|
||||
inputs.slidingWindowSize,
|
||||
MEMORY.DEFAULT_SLIDING_WINDOW_SIZE
|
||||
)
|
||||
messagesToStore = this.applyWindow(conversationMessages, limit)
|
||||
} else if (inputs.memoryType === 'sliding_window_tokens') {
|
||||
const maxTokens = this.parsePositiveInt(
|
||||
inputs.slidingWindowTokens,
|
||||
MEMORY.DEFAULT_SLIDING_WINDOW_TOKENS
|
||||
)
|
||||
messagesToStore = this.applyTokenWindow(conversationMessages, maxTokens, inputs.model)
|
||||
}
|
||||
|
||||
messagesToStore = await Promise.all(
|
||||
messagesToStore.map((message) => this.maskContentForStorage(ctx, message))
|
||||
)
|
||||
|
||||
await this.seedMemoryRecord(workspaceId, key, messagesToStore)
|
||||
|
||||
logger.debug('Seeded memory', {
|
||||
workspaceId,
|
||||
key,
|
||||
count: messagesToStore.length,
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Handlers persist messages to memory before the executor redacts block
|
||||
* output, so mask content here too when the block-output stage is enabled —
|
||||
* otherwise raw PII is stored in the memory table and read back on later runs.
|
||||
* `onFailure: 'throw'` aborts rather than persisting unredacted content.
|
||||
*/
|
||||
private async maskContentForStorage(ctx: ExecutionContext, message: Message): Promise<Message> {
|
||||
if (!ctx.piiBlockOutputRedaction?.enabled || !message.content) {
|
||||
return message
|
||||
}
|
||||
return {
|
||||
...message,
|
||||
content: await redactObjectStrings(message.content, {
|
||||
entityTypes: ctx.piiBlockOutputRedaction.entityTypes,
|
||||
language: ctx.piiBlockOutputRedaction.language,
|
||||
onFailure: 'throw',
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
private requireWorkspaceId(ctx: ExecutionContext): string {
|
||||
if (!ctx.workspaceId) {
|
||||
throw new Error('workspaceId is required for memory operations')
|
||||
}
|
||||
return ctx.workspaceId
|
||||
}
|
||||
|
||||
private applyWindow(messages: Message[], limit: number): Message[] {
|
||||
return messages.slice(-limit)
|
||||
}
|
||||
|
||||
private sanitizeMessageForStorage(message: Message): Message {
|
||||
const { files: _files, ...messageWithoutFiles } = message
|
||||
return messageWithoutFiles
|
||||
}
|
||||
|
||||
private applyTokenWindow(messages: Message[], maxTokens: number, model?: string): Message[] {
|
||||
const result: Message[] = []
|
||||
let tokenCount = 0
|
||||
|
||||
for (let i = messages.length - 1; i >= 0; i--) {
|
||||
const msg = messages[i]
|
||||
const msgTokens = getAccurateTokenCount(msg.content, model)
|
||||
|
||||
if (tokenCount + msgTokens <= maxTokens) {
|
||||
result.unshift(msg)
|
||||
tokenCount += msgTokens
|
||||
} else if (result.length === 0) {
|
||||
result.unshift(msg)
|
||||
break
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
private applyContextWindowLimit(messages: Message[], model?: string): Message[] {
|
||||
if (!model) return messages
|
||||
|
||||
for (const provider of Object.values(PROVIDER_DEFINITIONS)) {
|
||||
if (provider.contextInformationAvailable === false) continue
|
||||
|
||||
const matchesPattern = provider.modelPatterns?.some((p) => p.test(model))
|
||||
const matchesModel = provider.models.some((m) => m.id === model)
|
||||
|
||||
if (matchesPattern || matchesModel) {
|
||||
const modelDef = provider.models.find((m) => m.id === model)
|
||||
if (modelDef?.contextWindow) {
|
||||
const maxTokens = Math.floor(modelDef.contextWindow * MEMORY.CONTEXT_WINDOW_UTILIZATION)
|
||||
return this.applyTokenWindow(messages, maxTokens, model)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return messages
|
||||
}
|
||||
|
||||
private async fetchMemory(workspaceId: string, key: string): Promise<Message[]> {
|
||||
const result = await db
|
||||
.select({ data: memory.data })
|
||||
.from(memory)
|
||||
.where(and(eq(memory.workspaceId, workspaceId), eq(memory.key, key)))
|
||||
.limit(1)
|
||||
|
||||
if (result.length === 0) return []
|
||||
|
||||
const data = result[0].data
|
||||
if (!Array.isArray(data)) return []
|
||||
|
||||
return data
|
||||
.filter(
|
||||
(msg): msg is Message =>
|
||||
msg &&
|
||||
typeof msg === 'object' &&
|
||||
'role' in msg &&
|
||||
'content' in msg &&
|
||||
['system', 'user', 'assistant'].includes(msg.role) &&
|
||||
typeof msg.content === 'string'
|
||||
)
|
||||
.map((msg) => this.sanitizeMessageForStorage(msg))
|
||||
}
|
||||
|
||||
private async seedMemoryRecord(
|
||||
workspaceId: string,
|
||||
key: string,
|
||||
messages: Message[]
|
||||
): Promise<void> {
|
||||
const now = new Date()
|
||||
|
||||
const sanitizedMessages = messages.map((message) => this.sanitizeMessageForStorage(message))
|
||||
|
||||
await db
|
||||
.insert(memory)
|
||||
.values({
|
||||
id: generateId(),
|
||||
workspaceId,
|
||||
key,
|
||||
data: sanitizedMessages,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
.onConflictDoNothing()
|
||||
}
|
||||
|
||||
private async appendMessage(workspaceId: string, key: string, message: Message): Promise<void> {
|
||||
const now = new Date()
|
||||
|
||||
const sanitizedMessage = this.sanitizeMessageForStorage(message)
|
||||
|
||||
await db
|
||||
.insert(memory)
|
||||
.values({
|
||||
id: generateId(),
|
||||
workspaceId,
|
||||
key,
|
||||
data: [sanitizedMessage],
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: [memory.workspaceId, memory.key],
|
||||
set: {
|
||||
data: sql`${memory.data} || ${JSON.stringify([sanitizedMessage])}::jsonb`,
|
||||
updatedAt: now,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
private parsePositiveInt(value: string | undefined, defaultValue: number): number {
|
||||
if (!value) return defaultValue
|
||||
const parsed = Number.parseInt(value, 10)
|
||||
if (Number.isNaN(parsed) || parsed <= 0) return defaultValue
|
||||
return parsed
|
||||
}
|
||||
|
||||
private validateConversationId(conversationId?: string): void {
|
||||
if (!conversationId || conversationId.trim() === '') {
|
||||
throw new Error('Conversation ID is required')
|
||||
}
|
||||
if (conversationId.length > MEMORY.MAX_CONVERSATION_ID_LENGTH) {
|
||||
throw new Error(
|
||||
`Conversation ID too long (max ${MEMORY.MAX_CONVERSATION_ID_LENGTH} characters)`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private validateContent(content: string): void {
|
||||
const size = Buffer.byteLength(content, 'utf8')
|
||||
if (size > MEMORY.MAX_MESSAGE_CONTENT_BYTES) {
|
||||
throw new Error(
|
||||
`Message content too large (${size} bytes, max ${MEMORY.MAX_MESSAGE_CONTENT_BYTES})`
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const memoryService = new Memory()
|
||||
@@ -0,0 +1,50 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const { limitMock } = vi.hoisted(() => ({ limitMock: vi.fn() }))
|
||||
|
||||
vi.mock('@sim/db', () => ({
|
||||
db: { select: () => ({ from: () => ({ where: () => ({ limit: limitMock }) }) }) },
|
||||
skill: { workspaceId: 'workspaceId', name: 'name', content: 'content' },
|
||||
}))
|
||||
vi.mock('@sim/logger', () => ({
|
||||
createLogger: () => ({ error: vi.fn(), warn: vi.fn(), info: vi.fn(), debug: vi.fn() }),
|
||||
}))
|
||||
vi.mock('drizzle-orm', () => ({
|
||||
and: vi.fn(() => ({})),
|
||||
eq: vi.fn(() => ({})),
|
||||
inArray: vi.fn(() => ({})),
|
||||
}))
|
||||
|
||||
import { resolveSkillContent } from './skills-resolver'
|
||||
|
||||
// resolveSkillContent is the shared resolver invoked when the mothership calls
|
||||
// load_user_skill (and when a workflow agent block calls load_skill).
|
||||
describe('resolveSkillContent', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('returns null without a skill name or workspace', async () => {
|
||||
expect(await resolveSkillContent('', 'ws-1')).toBeNull()
|
||||
expect(await resolveSkillContent('x', '')).toBeNull()
|
||||
})
|
||||
|
||||
it('resolves builtin skills without touching the database', async () => {
|
||||
const content = await resolveSkillContent('research', 'ws-1')
|
||||
expect(content).toBeTruthy()
|
||||
expect(limitMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('resolves a workspace user skill by name', async () => {
|
||||
limitMock.mockResolvedValue([{ content: '# Playbook', name: 'posthog-playbook' }])
|
||||
expect(await resolveSkillContent('posthog-playbook', 'ws-1')).toBe('# Playbook')
|
||||
})
|
||||
|
||||
it('returns null when the user skill is not found', async () => {
|
||||
limitMock.mockResolvedValue([])
|
||||
expect(await resolveSkillContent('missing', 'ws-1')).toBeNull()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,165 @@
|
||||
import { db } from '@sim/db'
|
||||
import { skill } from '@sim/db/schema'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { and, eq, inArray } from 'drizzle-orm'
|
||||
import { getBuiltinSkillById, getBuiltinSkillByName } from '@/lib/workflows/skills/builtin-skills'
|
||||
import type { SkillInput } from '@/executor/handlers/agent/types'
|
||||
|
||||
const logger = createLogger('SkillsResolver')
|
||||
|
||||
function escapeXml(str: string): string {
|
||||
return str
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
}
|
||||
|
||||
interface SkillMetadata {
|
||||
name: string
|
||||
description: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch skill metadata (name + description) for system prompt injection.
|
||||
* Only returns lightweight data so the LLM knows what skills are available.
|
||||
*/
|
||||
export async function resolveSkillMetadata(
|
||||
skillInputs: SkillInput[],
|
||||
workspaceId: string
|
||||
): Promise<SkillMetadata[]> {
|
||||
if (!skillInputs.length || !workspaceId) return []
|
||||
|
||||
const metadata: SkillMetadata[] = []
|
||||
const dbSkillIds: string[] = []
|
||||
for (const input of skillInputs) {
|
||||
const builtin = getBuiltinSkillById(input.skillId)
|
||||
if (builtin) {
|
||||
metadata.push({ name: builtin.name, description: builtin.description })
|
||||
} else {
|
||||
dbSkillIds.push(input.skillId)
|
||||
}
|
||||
}
|
||||
|
||||
if (dbSkillIds.length === 0) return metadata
|
||||
|
||||
try {
|
||||
const rows = await db
|
||||
.select({ name: skill.name, description: skill.description })
|
||||
.from(skill)
|
||||
.where(and(eq(skill.workspaceId, workspaceId), inArray(skill.id, dbSkillIds)))
|
||||
|
||||
return [...metadata, ...rows]
|
||||
} catch (error) {
|
||||
logger.error('Failed to resolve skill metadata', { error, dbSkillIds, workspaceId })
|
||||
return metadata
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch full skill content for a load_skill tool response.
|
||||
* Called when the LLM decides a skill is relevant and invokes load_skill.
|
||||
*/
|
||||
export async function resolveSkillContent(
|
||||
skillName: string,
|
||||
workspaceId: string
|
||||
): Promise<string | null> {
|
||||
if (!skillName || !workspaceId) return null
|
||||
|
||||
const builtin = getBuiltinSkillByName(skillName)
|
||||
if (builtin) return builtin.content
|
||||
|
||||
try {
|
||||
const rows = await db
|
||||
.select({ content: skill.content, name: skill.name })
|
||||
.from(skill)
|
||||
.where(and(eq(skill.workspaceId, workspaceId), eq(skill.name, skillName)))
|
||||
.limit(1)
|
||||
|
||||
if (rows.length === 0) {
|
||||
logger.warn('Skill not found', { skillName, workspaceId })
|
||||
return null
|
||||
}
|
||||
|
||||
return rows[0].content
|
||||
} catch (error) {
|
||||
logger.error('Failed to resolve skill content', { error, skillName, workspaceId })
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
export async function resolveSkillContentById(
|
||||
skillId: string,
|
||||
workspaceId: string
|
||||
): Promise<{ name: string; content: string } | null> {
|
||||
if (!skillId || !workspaceId) return null
|
||||
|
||||
const builtin = getBuiltinSkillById(skillId)
|
||||
if (builtin) return { name: builtin.name, content: builtin.content }
|
||||
|
||||
try {
|
||||
const rows = await db
|
||||
.select({ content: skill.content, name: skill.name })
|
||||
.from(skill)
|
||||
.where(and(eq(skill.workspaceId, workspaceId), eq(skill.id, skillId)))
|
||||
.limit(1)
|
||||
|
||||
if (rows.length === 0) {
|
||||
logger.warn('Skill not found', { skillId, workspaceId })
|
||||
return null
|
||||
}
|
||||
|
||||
return rows[0]
|
||||
} catch (error) {
|
||||
logger.error('Failed to resolve skill content', { error, skillId, workspaceId })
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the system prompt section that lists available skills.
|
||||
* Uses XML format per the agentskills.io integration guide.
|
||||
*/
|
||||
export function buildSkillsSystemPromptSection(skills: SkillMetadata[]): string {
|
||||
if (!skills.length) return ''
|
||||
|
||||
const skillEntries = skills
|
||||
.map(
|
||||
(s) =>
|
||||
` <skill name="${escapeXml(s.name)}">\n <description>${escapeXml(s.description)}</description>\n </skill>`
|
||||
)
|
||||
.join('\n')
|
||||
|
||||
return [
|
||||
'',
|
||||
'You have access to the following skills. Use the load_skill tool to activate a skill when relevant.',
|
||||
'',
|
||||
'<available_skills>',
|
||||
skillEntries,
|
||||
'</available_skills>',
|
||||
].join('\n')
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the load_skill tool definition for injection into the tools array.
|
||||
* Returns a ProviderToolConfig-compatible object so all providers can process it.
|
||||
*/
|
||||
export function buildLoadSkillTool(skillNames: string[]) {
|
||||
return {
|
||||
id: 'load_skill',
|
||||
name: 'load_skill',
|
||||
description: `Load a skill to get specialized instructions. Available skills: ${skillNames.join(', ')}`,
|
||||
params: {},
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
skill_name: {
|
||||
type: 'string',
|
||||
description: 'Name of the skill to load',
|
||||
enum: skillNames,
|
||||
},
|
||||
},
|
||||
required: ['skill_name'],
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
import type { UserFile } from '@/executor/types'
|
||||
|
||||
export interface SkillInput {
|
||||
skillId: string
|
||||
name?: string
|
||||
description?: string
|
||||
}
|
||||
|
||||
export interface AgentInputs {
|
||||
model?: string
|
||||
responseFormat?: string | object
|
||||
tools?: ToolInput[]
|
||||
skills?: SkillInput[]
|
||||
// Legacy inputs (backward compatible)
|
||||
systemPrompt?: string
|
||||
userPrompt?: string | object
|
||||
memories?: any // Legacy memory block output
|
||||
// New message array input (from messages-input subblock)
|
||||
messages?: Message[]
|
||||
// Memory configuration
|
||||
memoryType?: 'none' | 'conversation' | 'sliding_window' | 'sliding_window_tokens'
|
||||
conversationId?: string // Required for all non-none memory types
|
||||
slidingWindowSize?: string // For message-based sliding window
|
||||
slidingWindowTokens?: string // For token-based sliding window
|
||||
// Deep research multi-turn
|
||||
previousInteractionId?: string // Interactions API previous interaction reference
|
||||
// LLM parameters
|
||||
temperature?: string
|
||||
maxTokens?: string
|
||||
apiKey?: string
|
||||
azureEndpoint?: string
|
||||
azureApiVersion?: string
|
||||
vertexProject?: string
|
||||
vertexLocation?: string
|
||||
vertexCredential?: string
|
||||
bedrockAccessKeyId?: string
|
||||
bedrockSecretKey?: string
|
||||
bedrockRegion?: string
|
||||
reasoningEffort?: string
|
||||
verbosity?: string
|
||||
thinkingLevel?: string
|
||||
files?: unknown
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a tool input for the agent block.
|
||||
*
|
||||
* @remarks
|
||||
* Valid types include:
|
||||
* - Standard block types (e.g., 'api', 'search', 'function')
|
||||
* - 'custom-tool': User-defined tools with custom code
|
||||
* - 'mcp': Individual MCP tool from a connected server
|
||||
*/
|
||||
export interface ToolInput {
|
||||
/** Tool type identifier */
|
||||
type?: string
|
||||
schema?: any
|
||||
title?: string
|
||||
code?: string
|
||||
/** Tool parameters */
|
||||
params?: Record<string, any>
|
||||
timeout?: number
|
||||
usageControl?: 'auto' | 'force' | 'none'
|
||||
operation?: string
|
||||
/** Database ID for custom tools (new reference format) */
|
||||
customToolId?: string
|
||||
}
|
||||
|
||||
export interface Message {
|
||||
role: 'system' | 'user' | 'assistant'
|
||||
content: string
|
||||
files?: UserFile[]
|
||||
executionId?: string
|
||||
function_call?: any
|
||||
tool_calls?: any[]
|
||||
}
|
||||
|
||||
export interface StreamingConfig {
|
||||
shouldUseStreaming: boolean
|
||||
isBlockSelectedForOutput: boolean
|
||||
hasOutgoingConnections: boolean
|
||||
}
|
||||
Reference in New Issue
Block a user