chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,47 @@
|
||||
/**
|
||||
* Keep the provider matrix in one place so the Vitest spec and subprocess
|
||||
* runner stay in sync.
|
||||
*/
|
||||
export const PROVIDER_MATRIX = [
|
||||
{
|
||||
provider: 'llamacpp',
|
||||
requiredEnv: 'LEON_LLAMACPP_BASE_URL',
|
||||
llmTarget: 'llamacpp'
|
||||
},
|
||||
{
|
||||
provider: 'openrouter',
|
||||
requiredEnv: 'LEON_OPENROUTER_API_KEY',
|
||||
llmTarget: 'openrouter/z-ai/glm-5-turbo'
|
||||
},
|
||||
{
|
||||
provider: 'openai',
|
||||
requiredEnv: 'LEON_OPENAI_API_KEY',
|
||||
llmTarget: 'openai/gpt-5.4'
|
||||
},
|
||||
{
|
||||
provider: 'anthropic',
|
||||
requiredEnv: 'LEON_ANTHROPIC_API_KEY',
|
||||
llmTarget: 'anthropic/claude-sonnet-4-20250514'
|
||||
},
|
||||
{
|
||||
provider: 'moonshotai',
|
||||
requiredEnv: 'LEON_MOONSHOTAI_API_KEY',
|
||||
llmTarget: 'moonshotai/kimi-k2.5'
|
||||
},
|
||||
{
|
||||
provider: 'zai',
|
||||
requiredEnv: 'LEON_ZAI_API_KEY',
|
||||
llmTarget: 'zai/glm-5-turbo'
|
||||
},
|
||||
{
|
||||
provider: 'minimax',
|
||||
requiredEnv: 'LEON_MINIMAX_API_KEY',
|
||||
llmTarget: 'minimax/MiniMax-M3'
|
||||
}
|
||||
] as const
|
||||
|
||||
export type AgentProvider = (typeof PROVIDER_MATRIX)[number]['provider']
|
||||
|
||||
export const PROVIDER_REQUIRED_ENV = Object.fromEntries(
|
||||
PROVIDER_MATRIX.map(({ provider, requiredEnv }) => [provider, requiredEnv])
|
||||
) as Record<AgentProvider, string>
|
||||
@@ -0,0 +1,357 @@
|
||||
import path from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
import execa from 'execa'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { PROVIDER_MATRIX } from './provider-matrix'
|
||||
|
||||
const CURRENT_DIR = fileURLToPath(new URL('.', import.meta.url))
|
||||
const ROOT_DIR = path.resolve(CURRENT_DIR, '..', '..', '..')
|
||||
const RESULT_PREFIX = '__AGENT_RESULT__'
|
||||
const PROGRESS_PREFIX = '__AGENT_PROGRESS__'
|
||||
|
||||
interface ProviderProgressEvent {
|
||||
provider: string
|
||||
stage:
|
||||
| 'bootstrap'
|
||||
| 'turn_start'
|
||||
| 'tool_call'
|
||||
| 'turn_result'
|
||||
| 'scenario_complete'
|
||||
turn?: number
|
||||
message: string
|
||||
data?: Record<string, unknown>
|
||||
}
|
||||
|
||||
interface ProviderScenarioResult {
|
||||
provider: string
|
||||
skipped: boolean
|
||||
reason?: string
|
||||
assetPath?: string
|
||||
turns?: Array<{
|
||||
input: string
|
||||
output: string
|
||||
finalIntent: string | null
|
||||
executionHistory: Array<{
|
||||
function: string
|
||||
status: string
|
||||
observation: string
|
||||
stepLabel?: string
|
||||
requestedToolInput?: string
|
||||
}>
|
||||
toolCalls: Array<{
|
||||
toolkitId?: string
|
||||
toolId: string
|
||||
functionName?: string
|
||||
toolInput?: string
|
||||
parsedInput?: Record<string, unknown>
|
||||
toolOutput?: string
|
||||
}>
|
||||
}>
|
||||
}
|
||||
|
||||
function extractTestNamePattern(argv: string[]): string | null {
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index]
|
||||
|
||||
if (!arg) {
|
||||
continue
|
||||
}
|
||||
|
||||
if (arg === '-t' || arg === '--testNamePattern' || arg === '--test-name-pattern') {
|
||||
return argv[index + 1] || null
|
||||
}
|
||||
|
||||
if (arg.startsWith('-t=')) {
|
||||
return arg.slice(3) || null
|
||||
}
|
||||
|
||||
if (arg.startsWith('--testNamePattern=')) {
|
||||
return arg.slice('--testNamePattern='.length) || null
|
||||
}
|
||||
|
||||
if (arg.startsWith('--test-name-pattern=')) {
|
||||
return arg.slice('--test-name-pattern='.length) || null
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
function resolveProviderMatrix(
|
||||
pattern: string | null
|
||||
): typeof PROVIDER_MATRIX {
|
||||
if (!pattern) {
|
||||
return PROVIDER_MATRIX
|
||||
}
|
||||
|
||||
const matchesPattern = (provider: string): boolean => {
|
||||
const testName = `runs the 3-turn scenario on ${provider}`
|
||||
|
||||
try {
|
||||
return new RegExp(pattern, 'i').test(testName)
|
||||
} catch {
|
||||
return testName.toLowerCase().includes(pattern.toLowerCase())
|
||||
}
|
||||
}
|
||||
|
||||
const filteredProviders = PROVIDER_MATRIX.filter(({ provider }) =>
|
||||
matchesPattern(provider)
|
||||
)
|
||||
|
||||
return filteredProviders.length > 0 ? filteredProviders : PROVIDER_MATRIX
|
||||
}
|
||||
|
||||
const ACTIVE_PROVIDER_MATRIX = resolveProviderMatrix(
|
||||
process.env['LEON_AGENT_PROVIDER_PATTERN'] ||
|
||||
extractTestNamePattern(process.argv)
|
||||
)
|
||||
|
||||
function collectTurnTrace(
|
||||
turn: NonNullable<ProviderScenarioResult['turns']>[number]
|
||||
): string {
|
||||
return [
|
||||
turn.output,
|
||||
...turn.executionHistory.map((item) => item.observation),
|
||||
...turn.executionHistory.map((item) => item.requestedToolInput || ''),
|
||||
...turn.toolCalls.map((item) => item.toolInput || ''),
|
||||
...turn.toolCalls.map((item) => item.toolOutput || ''),
|
||||
...turn.toolCalls.map((item) =>
|
||||
item.parsedInput ? JSON.stringify(item.parsedInput) : ''
|
||||
)
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join('\n')
|
||||
}
|
||||
|
||||
function summarizeText(value: string | undefined, maxLength = 500): string {
|
||||
if (!value) {
|
||||
return ''
|
||||
}
|
||||
|
||||
return value.length <= maxLength ? value : `${value.slice(0, maxLength)}...`
|
||||
}
|
||||
|
||||
function summarizeScenarioResult(result: ProviderScenarioResult): string {
|
||||
return JSON.stringify(
|
||||
{
|
||||
provider: result.provider,
|
||||
skipped: result.skipped,
|
||||
reason: result.reason,
|
||||
assetPath: result.assetPath,
|
||||
turns: result.turns?.map((turn, index) => ({
|
||||
turn: index + 1,
|
||||
input: turn.input,
|
||||
output: summarizeText(turn.output),
|
||||
finalIntent: turn.finalIntent,
|
||||
executionHistory: turn.executionHistory.map((item) => ({
|
||||
function: item.function,
|
||||
status: item.status,
|
||||
stepLabel: item.stepLabel,
|
||||
observation: summarizeText(item.observation),
|
||||
requestedToolInput: summarizeText(item.requestedToolInput)
|
||||
})),
|
||||
toolCalls: turn.toolCalls.map((item) => ({
|
||||
toolkitId: item.toolkitId,
|
||||
toolId: item.toolId,
|
||||
functionName: item.functionName,
|
||||
toolInput: summarizeText(item.toolInput),
|
||||
parsedInput: item.parsedInput,
|
||||
toolOutput: summarizeText(item.toolOutput)
|
||||
}))
|
||||
}))
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
}
|
||||
|
||||
function formatProgressEvent(event: ProviderProgressEvent): string {
|
||||
const prefix = `[agent:e2e:${event.provider}]`
|
||||
|
||||
if (event.stage === 'turn_start') {
|
||||
return `${prefix} turn ${event.turn} input=${JSON.stringify(event.data?.['input'] || '')}`
|
||||
}
|
||||
|
||||
if (event.stage === 'tool_call') {
|
||||
return `${prefix} tool=${event.data?.['toolName'] || 'unknown'} input=${JSON.stringify(event.data?.['toolInput'] || '')} output=${JSON.stringify(event.data?.['toolOutput'] || '')}`
|
||||
}
|
||||
|
||||
if (event.stage === 'turn_result') {
|
||||
return `${prefix} turn ${event.turn} intent=${String(event.data?.['finalIntent'] || 'unknown')} toolCalls=${String(event.data?.['toolCalls'] || 0)} output=${JSON.stringify(event.data?.['output'] || '')}`
|
||||
}
|
||||
|
||||
if (event.stage === 'bootstrap') {
|
||||
return `${prefix} bootstrap asset=${JSON.stringify(event.data?.['assetPath'] || '')}`
|
||||
}
|
||||
|
||||
return `${prefix} ${event.message}`
|
||||
}
|
||||
|
||||
async function runProviderScenario(
|
||||
provider: string
|
||||
): Promise<ProviderScenarioResult> {
|
||||
/**
|
||||
* Provider choice is read at module-load time, so each provider run needs a
|
||||
* fresh process with its own env.
|
||||
*/
|
||||
const childProcess = execa(
|
||||
'node',
|
||||
[
|
||||
'--import',
|
||||
'tsx',
|
||||
'test/agent/e2e/run-agent-provider-scenario.ts',
|
||||
provider
|
||||
],
|
||||
{
|
||||
cwd: ROOT_DIR,
|
||||
env: {
|
||||
...process.env,
|
||||
LEON_NODE_ENV: 'testing',
|
||||
LEON_LLM:
|
||||
PROVIDER_MATRIX.find((item) => item.provider === provider)?.llmTarget ||
|
||||
provider
|
||||
},
|
||||
all: true,
|
||||
reject: false,
|
||||
timeout: 300_000
|
||||
}
|
||||
)
|
||||
|
||||
let streamBuffer = ''
|
||||
childProcess.all?.setEncoding('utf8')
|
||||
childProcess.all?.on('data', (chunk: string) => {
|
||||
streamBuffer += chunk
|
||||
const lines = streamBuffer.split('\n')
|
||||
streamBuffer = lines.pop() || ''
|
||||
|
||||
for (const rawLine of lines) {
|
||||
const line = rawLine.trim()
|
||||
|
||||
if (!line.startsWith(PROGRESS_PREFIX)) {
|
||||
continue
|
||||
}
|
||||
|
||||
const payload = line.slice(PROGRESS_PREFIX.length)
|
||||
|
||||
try {
|
||||
const event = JSON.parse(payload) as ProviderProgressEvent
|
||||
console.info(formatProgressEvent(event))
|
||||
} catch {
|
||||
console.info(`[agent:e2e:${provider}] ${payload}`)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
const { stdout, stderr, exitCode } = await childProcess
|
||||
|
||||
const combinedOutput = `${stdout}\n${stderr}`
|
||||
const resultLine = combinedOutput
|
||||
.split('\n')
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.startsWith(RESULT_PREFIX))
|
||||
.at(-1)
|
||||
|
||||
if (!resultLine) {
|
||||
throw new Error(
|
||||
`Missing agent result marker for provider "${provider}". Output:\n${combinedOutput}`
|
||||
)
|
||||
}
|
||||
|
||||
const result = JSON.parse(
|
||||
resultLine.slice(RESULT_PREFIX.length)
|
||||
) as ProviderScenarioResult
|
||||
|
||||
if (exitCode !== 0 && !result.skipped) {
|
||||
throw new Error(
|
||||
`Provider "${provider}" scenario failed with exit code ${exitCode}. Output:\n${combinedOutput}`
|
||||
)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
describe('agent e2e', () => {
|
||||
for (const { provider, requiredEnv } of ACTIVE_PROVIDER_MATRIX) {
|
||||
/**
|
||||
* Missing credentials should skip that provider cleanly rather than fail
|
||||
* the whole matrix.
|
||||
*/
|
||||
it.skipIf(!process.env[requiredEnv])(
|
||||
`runs the 3-turn scenario on ${provider}`,
|
||||
async () => {
|
||||
const result = await runProviderScenario(provider)
|
||||
|
||||
console.info(
|
||||
`[agent:e2e:${provider}] validating turn outputs and tool usage`
|
||||
)
|
||||
|
||||
try {
|
||||
if (result.skipped) {
|
||||
console.info(
|
||||
`[agent:e2e:${provider}] skipped at runtime: ${result.reason || 'provider unavailable'}`
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
expect(result.turns).toHaveLength(3)
|
||||
|
||||
const [turn1, turn2, turn3] = result.turns!
|
||||
const turn2Trace = collectTurnTrace(turn2!)
|
||||
const turn3Trace = collectTurnTrace(turn3!)
|
||||
|
||||
expect(turn1!.output.trim().length).toBeGreaterThan(0)
|
||||
expect(turn1!.output).toMatch(/ping|pong/i)
|
||||
expect(turn1!.finalIntent).toBe('answer')
|
||||
|
||||
expect(turn2!.output.trim().length).toBeGreaterThan(0)
|
||||
expect(turn2!.finalIntent).toBe('answer')
|
||||
expect(
|
||||
turn2!.executionHistory.some(
|
||||
(item) =>
|
||||
item.function === 'weather.openmeteo.getCurrentConditions'
|
||||
) ||
|
||||
turn2!.toolCalls.some(
|
||||
(item) =>
|
||||
item.toolkitId === 'weather' &&
|
||||
item.toolId === 'openmeteo' &&
|
||||
item.functionName === 'getCurrentConditions'
|
||||
)
|
||||
).toBe(true)
|
||||
expect(turn2Trace).toMatch(/shenzhen/i)
|
||||
expect(turn2Trace).toMatch(
|
||||
/clear|rain|cloud|temperature|feels|humidity|wind|weather|°c|°f/i
|
||||
)
|
||||
|
||||
/**
|
||||
* The third turn is intentionally structural: we care that Leon read
|
||||
* the injected file and listed the project root, not about exact prose.
|
||||
*/
|
||||
expect(turn3!.output.trim().length).toBeGreaterThan(0)
|
||||
expect(turn3!.finalIntent).toBe('answer')
|
||||
expect(
|
||||
turn3!.executionHistory.some(
|
||||
(item) =>
|
||||
item.function ===
|
||||
'operating_system_control.shell.executeCommand'
|
||||
) ||
|
||||
turn3!.toolCalls.some(
|
||||
(item) =>
|
||||
item.toolkitId === 'operating_system_control' &&
|
||||
item.toolId === 'shell'
|
||||
)
|
||||
).toBe(true)
|
||||
expect(turn3Trace).toContain(result.assetPath!)
|
||||
expect(turn3Trace).toMatch(/project root/i)
|
||||
} catch (error) {
|
||||
console.info(
|
||||
`[agent:e2e:${provider}] scenario result on assertion failure:\n${summarizeScenarioResult(result)}`
|
||||
)
|
||||
throw error
|
||||
}
|
||||
},
|
||||
330_000
|
||||
)
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,478 @@
|
||||
import fs from 'node:fs/promises'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
|
||||
import type { MessageLog } from '../../../server/src/types'
|
||||
import type { AgentProvider } from './provider-matrix'
|
||||
import { PROVIDER_MATRIX, PROVIDER_REQUIRED_ENV } from './provider-matrix'
|
||||
|
||||
const RESULT_PREFIX = '__AGENT_RESULT__'
|
||||
const PROGRESS_PREFIX = '__AGENT_PROGRESS__'
|
||||
const TEST_HOME_PREFIX = 'leon-agent-e2e'
|
||||
const TEST_PROFILE_PREFIX = 'agent-e2e'
|
||||
const EMPTY_PROFILE_DISABLED_CONFIG = {
|
||||
skills: [],
|
||||
tools: []
|
||||
}
|
||||
const REACT_CONTINUATION_STATE_FILENAME =
|
||||
'.react-execution-continuation-state.json'
|
||||
const REACT_HISTORY_COMPACTION_STATE_FILENAME =
|
||||
'.react-history-compaction-state.json'
|
||||
const PROVIDER_UNAVAILABLE_PATTERNS = [
|
||||
/cannot find llama\.cpp model/i,
|
||||
/credit balance is too low/i,
|
||||
/insufficient[_\s-]?quota/i,
|
||||
/no default installed local llm was found/i,
|
||||
/no llm is configured/i,
|
||||
/rate limit/i,
|
||||
/\b429\b/i
|
||||
]
|
||||
|
||||
interface AgentProgressEvent {
|
||||
provider: AgentProvider
|
||||
stage:
|
||||
| 'bootstrap'
|
||||
| 'turn_start'
|
||||
| 'tool_call'
|
||||
| 'turn_result'
|
||||
| 'scenario_complete'
|
||||
turn?: number
|
||||
message: string
|
||||
data?: Record<string, unknown>
|
||||
}
|
||||
|
||||
interface AgentTurnResult {
|
||||
input: string
|
||||
output: string
|
||||
finalIntent: string | null
|
||||
executionHistory: Array<{
|
||||
function: string
|
||||
status: string
|
||||
observation: string
|
||||
stepLabel?: string
|
||||
requestedToolInput?: string
|
||||
}>
|
||||
toolCalls: Array<{
|
||||
toolkitId?: string
|
||||
toolId: string
|
||||
functionName?: string
|
||||
toolInput?: string
|
||||
parsedInput?: Record<string, unknown>
|
||||
toolOutput?: string
|
||||
}>
|
||||
}
|
||||
|
||||
interface AgentRunnerResult {
|
||||
provider: AgentProvider
|
||||
skipped: boolean
|
||||
reason?: string
|
||||
assetPath?: string
|
||||
turns?: AgentTurnResult[]
|
||||
}
|
||||
|
||||
type ConversationLoggerRecord = Omit<MessageLog, 'sentAt'>
|
||||
|
||||
function printResult(result: AgentRunnerResult): void {
|
||||
/**
|
||||
* A fixed marker makes it easy for the parent Vitest process to extract the
|
||||
* structured result from mixed stdout/stderr.
|
||||
*/
|
||||
console.log(`${RESULT_PREFIX}${JSON.stringify(result)}`)
|
||||
}
|
||||
|
||||
function printProgress(event: AgentProgressEvent): void {
|
||||
console.log(`${PROGRESS_PREFIX}${JSON.stringify(event)}`)
|
||||
}
|
||||
|
||||
function serializeToolOutput(value: unknown): string {
|
||||
try {
|
||||
return JSON.stringify(value)
|
||||
} catch {
|
||||
return String(value)
|
||||
}
|
||||
}
|
||||
|
||||
function summarizeValue(value: string, maxLength = 220): string {
|
||||
if (value.length <= maxLength) {
|
||||
return value
|
||||
}
|
||||
|
||||
return `${value.slice(0, maxLength)}...`
|
||||
}
|
||||
|
||||
function createConversationLoggerRecord(
|
||||
who: MessageLog['who'],
|
||||
message: string
|
||||
): ConversationLoggerRecord {
|
||||
return {
|
||||
who,
|
||||
message,
|
||||
isAddedToHistory: true
|
||||
}
|
||||
}
|
||||
|
||||
function getProviderUnavailableReason(value: unknown): string | null {
|
||||
const message =
|
||||
value instanceof Error
|
||||
? `${value.name}: ${value.message}\n${value.stack || ''}`
|
||||
: String(value || '')
|
||||
|
||||
if (
|
||||
PROVIDER_UNAVAILABLE_PATTERNS.some((pattern) => pattern.test(message))
|
||||
) {
|
||||
return summarizeValue(message.replace(/\s+/g, ' ').trim(), 500)
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
async function removeTestHomePath(
|
||||
homePath: string,
|
||||
expectedHomePath: string
|
||||
): Promise<void> {
|
||||
const resolvedHomePath = path.resolve(homePath)
|
||||
const resolvedExpectedHomePath = path.resolve(expectedHomePath)
|
||||
|
||||
if (
|
||||
resolvedHomePath !== resolvedExpectedHomePath ||
|
||||
path.dirname(resolvedHomePath) !== os.tmpdir() ||
|
||||
!path.basename(resolvedHomePath).startsWith(`${TEST_HOME_PREFIX}-`)
|
||||
) {
|
||||
throw new Error(`Refusing to remove non-test Leon home path: ${homePath}`)
|
||||
}
|
||||
|
||||
await fs.rm(resolvedHomePath, { force: true, recursive: true })
|
||||
}
|
||||
|
||||
async function prepareTestProfilePath(
|
||||
directories: string[],
|
||||
disabledConfigPath: string
|
||||
): Promise<void> {
|
||||
await Promise.all(
|
||||
directories.map((directory) => fs.mkdir(directory, { recursive: true }))
|
||||
)
|
||||
|
||||
await fs.writeFile(
|
||||
disabledConfigPath,
|
||||
`${JSON.stringify(EMPTY_PROFILE_DISABLED_CONFIG, null, 2)}\n`,
|
||||
'utf8'
|
||||
)
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const providerArg = process.argv[2] as AgentProvider | undefined
|
||||
if (!providerArg || !(providerArg in PROVIDER_REQUIRED_ENV)) {
|
||||
printResult({
|
||||
provider: (providerArg || 'openai') as AgentProvider,
|
||||
skipped: true,
|
||||
reason: 'invalid_provider'
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
const provider = providerArg
|
||||
const providerConfig = PROVIDER_MATRIX.find(
|
||||
(item) => item.provider === provider
|
||||
)
|
||||
const requiredEnv = PROVIDER_REQUIRED_ENV[provider]
|
||||
const testRunId = `${provider}-${process.pid}-${Date.now()}`
|
||||
const testProfileName = `${TEST_PROFILE_PREFIX}-${testRunId}`
|
||||
const testHomePath = path.join(
|
||||
os.tmpdir(),
|
||||
`${TEST_HOME_PREFIX}-${testRunId}`
|
||||
)
|
||||
if (!process.env[requiredEnv]) {
|
||||
printResult({
|
||||
provider,
|
||||
skipped: true,
|
||||
reason: `missing_${requiredEnv.toLowerCase()}`
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
process.env['LEON_NODE_ENV'] = 'testing'
|
||||
process.env['LEON_LLM'] = providerConfig?.llmTarget || provider
|
||||
process.env['LEON_HOME'] = testHomePath
|
||||
process.env['LEON_PROFILE'] = testProfileName
|
||||
|
||||
const tempAssetPath = path.join(
|
||||
os.tmpdir(),
|
||||
`leon-agent-${provider}-${Date.now()}.txt`
|
||||
)
|
||||
|
||||
const {
|
||||
CACHE_PATH,
|
||||
LEON_PROFILE_PATH,
|
||||
LEON_PROFILES_PATH,
|
||||
LEON_HOME_PATH,
|
||||
LEON_TOOLKITS_PATH,
|
||||
MODELS_PATH,
|
||||
PROFILE_AGENT_SKILLS_PATH,
|
||||
PROFILE_CONTEXT_PATH,
|
||||
PROFILE_DISABLED_PATH,
|
||||
PROFILE_LOGS_PATH,
|
||||
PROFILE_MEMORY_PATH,
|
||||
PROFILE_NATIVE_SKILLS_PATH,
|
||||
PROFILE_SKILLS_PATH,
|
||||
PROFILE_TOOLS_PATH,
|
||||
TMP_PATH
|
||||
} = await import('../../../server/src/constants')
|
||||
|
||||
await prepareTestProfilePath(
|
||||
[
|
||||
LEON_HOME_PATH,
|
||||
LEON_PROFILES_PATH,
|
||||
LEON_PROFILE_PATH,
|
||||
CACHE_PATH,
|
||||
LEON_TOOLKITS_PATH,
|
||||
MODELS_PATH,
|
||||
TMP_PATH,
|
||||
PROFILE_CONTEXT_PATH,
|
||||
PROFILE_MEMORY_PATH,
|
||||
PROFILE_LOGS_PATH,
|
||||
PROFILE_SKILLS_PATH,
|
||||
PROFILE_NATIVE_SKILLS_PATH,
|
||||
PROFILE_AGENT_SKILLS_PATH,
|
||||
PROFILE_TOOLS_PATH
|
||||
],
|
||||
PROFILE_DISABLED_PATH
|
||||
)
|
||||
|
||||
const continuationStatePath = path.join(
|
||||
PROFILE_CONTEXT_PATH,
|
||||
REACT_CONTINUATION_STATE_FILENAME
|
||||
)
|
||||
const historyCompactionStatePath = path.join(
|
||||
PROFILE_CONTEXT_PATH,
|
||||
REACT_HISTORY_COMPACTION_STATE_FILENAME
|
||||
)
|
||||
|
||||
await fs.writeFile(
|
||||
tempAssetPath,
|
||||
`Please list the files in this exact project root directory: ${process.cwd()}.\n`,
|
||||
'utf8'
|
||||
)
|
||||
|
||||
const {
|
||||
ReActLLMDuty
|
||||
} = await import('../../../server/src/core/llm-manager/llm-duties/react-llm-duty')
|
||||
const { CONVERSATION_LOGGER, TOOL_EXECUTOR, LLM_PROVIDER } = await import(
|
||||
'../../../server/src/core/index'
|
||||
)
|
||||
|
||||
const turns: string[] = [
|
||||
// Return final answer directly
|
||||
'Hi Leon, just doing a quick check since I switched your LLM provider. What do you reply if I tell you "ping"?',
|
||||
// Create simple plan
|
||||
'What\'s the weather like today in Shenzhen?',
|
||||
// Create plan with dynamic replanning (inject new step)
|
||||
`There is a file waiting for you in ${tempAssetPath}, do what it asks you to do.`
|
||||
]
|
||||
|
||||
const turnResults: AgentTurnResult[] = []
|
||||
const toolCalls: AgentTurnResult['toolCalls'] = []
|
||||
type ExecuteTool = typeof TOOL_EXECUTOR.executeTool
|
||||
type ToolExecutionInput = Parameters<ExecuteTool>[0]
|
||||
type ToolExecutionResult = Awaited<ReturnType<ExecuteTool>>
|
||||
|
||||
const originalExecuteTool = TOOL_EXECUTOR.executeTool.bind(
|
||||
TOOL_EXECUTOR
|
||||
) as ExecuteTool
|
||||
|
||||
/**
|
||||
* Wrap tool execution so the parent spec can assert on real tool usage
|
||||
* without changing the production ReAct path.
|
||||
*/
|
||||
TOOL_EXECUTOR.executeTool = async (
|
||||
input: ToolExecutionInput
|
||||
): Promise<ToolExecutionResult> => {
|
||||
const toolResult = await originalExecuteTool(input)
|
||||
const toolName = `${input.toolkitId}.${input.toolId}.${input.functionName || 'unknown'}`
|
||||
const serializedInput = input.toolInput || ''
|
||||
const serializedOutput = serializeToolOutput(toolResult)
|
||||
const toolCall: AgentTurnResult['toolCalls'][number] = {
|
||||
toolId: input.toolId,
|
||||
toolOutput: serializedOutput
|
||||
}
|
||||
|
||||
if (input.toolkitId !== undefined) {
|
||||
toolCall.toolkitId = input.toolkitId
|
||||
}
|
||||
|
||||
if (input.functionName !== undefined) {
|
||||
toolCall.functionName = input.functionName
|
||||
}
|
||||
|
||||
if (input.toolInput !== undefined) {
|
||||
toolCall.toolInput = input.toolInput
|
||||
}
|
||||
|
||||
if (input.parsedInput && typeof input.parsedInput === 'object') {
|
||||
toolCall.parsedInput = { ...input.parsedInput }
|
||||
}
|
||||
|
||||
printProgress({
|
||||
provider,
|
||||
stage: 'tool_call',
|
||||
message: `Executed ${toolName}`,
|
||||
data: {
|
||||
toolName,
|
||||
toolInput: summarizeValue(serializedInput),
|
||||
toolOutput: summarizeValue(serializedOutput)
|
||||
}
|
||||
})
|
||||
|
||||
toolCalls.push(toolCall)
|
||||
|
||||
return toolResult
|
||||
}
|
||||
|
||||
try {
|
||||
/**
|
||||
* The e2e subprocess bypasses the normal server bootstrap, so initialize
|
||||
* the selected provider explicitly before the first ReAct turn.
|
||||
*/
|
||||
await LLM_PROVIDER.init()
|
||||
printProgress({
|
||||
provider,
|
||||
stage: 'bootstrap',
|
||||
message: `Initialized provider ${provider}`,
|
||||
data: {
|
||||
assetPath: tempAssetPath
|
||||
}
|
||||
})
|
||||
await CONVERSATION_LOGGER.clear()
|
||||
await fs.rm(continuationStatePath, { force: true })
|
||||
await fs.rm(historyCompactionStatePath, { force: true })
|
||||
|
||||
let recordedToolCalls = 0
|
||||
for (let index = 0; index < turns.length; index += 1) {
|
||||
const input = turns[index]!
|
||||
const turnNumber = index + 1
|
||||
|
||||
printProgress({
|
||||
provider,
|
||||
stage: 'turn_start',
|
||||
turn: turnNumber,
|
||||
message: `Starting turn ${turnNumber}`,
|
||||
data: {
|
||||
input
|
||||
}
|
||||
})
|
||||
|
||||
/**
|
||||
* Push each owner/Leon turn through the shared conversation logger so the
|
||||
* next ReAct invocation sees real multi-turn history.
|
||||
*/
|
||||
await CONVERSATION_LOGGER.push(
|
||||
createConversationLoggerRecord('owner', input)
|
||||
)
|
||||
|
||||
const duty = new ReActLLMDuty({ input })
|
||||
await duty.init({ force: index === 0 })
|
||||
const result = await duty.execute()
|
||||
|
||||
const output =
|
||||
result && typeof result.output === 'string' ? result.output : ''
|
||||
const finalIntent =
|
||||
result &&
|
||||
result.data &&
|
||||
typeof result.data === 'object' &&
|
||||
'finalIntent' in result.data &&
|
||||
typeof result.data['finalIntent'] === 'string'
|
||||
? result.data['finalIntent']
|
||||
: null
|
||||
|
||||
if (finalIntent === 'error') {
|
||||
const providerUnavailableReason = getProviderUnavailableReason(output)
|
||||
|
||||
if (providerUnavailableReason) {
|
||||
printResult({
|
||||
provider,
|
||||
skipped: true,
|
||||
reason: providerUnavailableReason,
|
||||
assetPath: tempAssetPath,
|
||||
turns: turnResults
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if (output) {
|
||||
await CONVERSATION_LOGGER.push(
|
||||
createConversationLoggerRecord('leon', output)
|
||||
)
|
||||
}
|
||||
|
||||
printProgress({
|
||||
provider,
|
||||
stage: 'turn_result',
|
||||
turn: turnNumber,
|
||||
message: `Completed turn ${turnNumber}`,
|
||||
data: {
|
||||
finalIntent,
|
||||
output: summarizeValue(output),
|
||||
toolCalls: toolCalls.length - recordedToolCalls
|
||||
}
|
||||
})
|
||||
|
||||
turnResults.push({
|
||||
input,
|
||||
output,
|
||||
finalIntent,
|
||||
executionHistory:
|
||||
result &&
|
||||
result.data &&
|
||||
typeof result.data === 'object' &&
|
||||
Array.isArray(result.data['executionHistory'])
|
||||
? (result.data['executionHistory'] as AgentTurnResult['executionHistory'])
|
||||
: [],
|
||||
toolCalls: toolCalls.slice(recordedToolCalls)
|
||||
})
|
||||
|
||||
recordedToolCalls = toolCalls.length
|
||||
}
|
||||
|
||||
printResult({
|
||||
provider,
|
||||
skipped: false,
|
||||
assetPath: tempAssetPath,
|
||||
turns: turnResults
|
||||
})
|
||||
printProgress({
|
||||
provider,
|
||||
stage: 'scenario_complete',
|
||||
message: `Completed ${turnResults.length} turns`,
|
||||
data: {
|
||||
turns: turnResults.length
|
||||
}
|
||||
})
|
||||
} finally {
|
||||
TOOL_EXECUTOR.executeTool = originalExecuteTool
|
||||
await CONVERSATION_LOGGER.clear()
|
||||
await fs.rm(tempAssetPath, { force: true })
|
||||
await fs.rm(continuationStatePath, { force: true })
|
||||
await fs.rm(historyCompactionStatePath, { force: true })
|
||||
await removeTestHomePath(LEON_HOME_PATH, testHomePath)
|
||||
}
|
||||
}
|
||||
|
||||
void main()
|
||||
.then(() => {
|
||||
/**
|
||||
* Core singletons keep background handles open, so exit explicitly once the
|
||||
* structured result has been printed and cleanup has finished.
|
||||
*/
|
||||
process.exit(0)
|
||||
})
|
||||
.catch((error) => {
|
||||
const provider = (process.argv[2] || 'openai') as AgentProvider
|
||||
const providerUnavailableReason = getProviderUnavailableReason(error)
|
||||
|
||||
printResult({
|
||||
provider,
|
||||
skipped: Boolean(providerUnavailableReason),
|
||||
reason: providerUnavailableReason || String(error)
|
||||
})
|
||||
process.exit(providerUnavailableReason ? 0 : 1)
|
||||
})
|
||||
Reference in New Issue
Block a user