chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,282 @@
|
||||
/**
|
||||
* Pipeline Benchmark — Direct Pipeline Execution
|
||||
*
|
||||
* Tests Smart Auto Pipeline accuracy and cost by calling executePipeline() directly
|
||||
* with DeepSeek API as the stage executor. Bypasses combo routing overhead.
|
||||
*
|
||||
* Measures: accuracy, token usage, latency, cost — baseline (single call) vs pipeline
|
||||
*/
|
||||
|
||||
import {
|
||||
buildPipelineConfig,
|
||||
executePipeline,
|
||||
type StageExecutor,
|
||||
type StageExecutorResult,
|
||||
type FitnessTier,
|
||||
} from "../../src/domain/pipeline.ts";
|
||||
|
||||
const API_KEY = process.env.DEEPSEEK_API_KEY;
|
||||
const BASE_URL = "https://api.deepseek.com/v1";
|
||||
const MODEL = "deepseek-chat";
|
||||
|
||||
const COST_INPUT_PER_M = 0.14;
|
||||
const COST_OUTPUT_PER_M = 0.28;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test cases
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const MATH_PROBLEMS = [
|
||||
{ q: "What is 17 * 23?", expected: "391" },
|
||||
{ q: "Solve for x: 2x + 5 = 13", expected: "4" },
|
||||
{ q: "What is the derivative of x^3 + 2x?", expected: "3x^2 + 2" },
|
||||
{ q: "What is the integral of 2x dx?", expected: "x^2" },
|
||||
{ q: "If a triangle has sides 3, 4, 5, what is its area?", expected: "6" },
|
||||
];
|
||||
|
||||
const CODING_PROBLEMS = [
|
||||
{
|
||||
q: "Write a JavaScript function that returns the factorial of n.",
|
||||
check: (r: string) =>
|
||||
/function\s+\w*factorial|factorial\s*=|const\s+factorial/i.test(r) &&
|
||||
/return|n\s*\*/i.test(r),
|
||||
},
|
||||
{
|
||||
q: "Write a Python function that checks if a string is a palindrome.",
|
||||
check: (r: string) => /def\s+\w*pali|pali.*def/i.test(r) && /return|==/i.test(r),
|
||||
},
|
||||
{
|
||||
q: "Write a TypeScript function that reverses an array without mutating it.",
|
||||
check: (r: string) => /reverse|slice|spread|\.\.\./i.test(r),
|
||||
},
|
||||
{
|
||||
q: "Write a SQL query to find the second highest salary from an employees table.",
|
||||
check: (r: string) =>
|
||||
/SELECT|select/i.test(r) && /salary|LIMIT|OFFSET|DENSE_RANK|ROW_NUMBER/i.test(r),
|
||||
},
|
||||
{
|
||||
q: "Write a bash one-liner to count the number of lines in all .ts files recursively.",
|
||||
check: (r: string) => /find|grep|wc|cat/i.test(r) && /-l|lines|count/i.test(r),
|
||||
},
|
||||
];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// API call helper
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface CallResult {
|
||||
text: string;
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
latencyMs: number;
|
||||
}
|
||||
|
||||
async function callDeepSeek(
|
||||
messages: Array<{ role: string; content: string }>
|
||||
): Promise<CallResult> {
|
||||
const start = Date.now();
|
||||
const res = await fetch(`${BASE_URL}/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Authorization: `Bearer ${API_KEY}` },
|
||||
body: JSON.stringify({ model: MODEL, messages, stream: false }),
|
||||
});
|
||||
if (!res.ok) throw new Error(`DeepSeek error ${res.status}: ${await res.text()}`);
|
||||
const data = (await res.json()) as Record<string, unknown>;
|
||||
const usage = data.usage as Record<string, number> | undefined;
|
||||
const msg = (data.choices as Array<Record<string, unknown>>)?.[0]?.message as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
return {
|
||||
text: (msg?.content as string) ?? "",
|
||||
inputTokens: usage?.prompt_tokens ?? 0,
|
||||
outputTokens: usage?.completion_tokens ?? 0,
|
||||
latencyMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stage executor — calls DeepSeek directly
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function makeDeepSeekExecutor(): StageExecutor {
|
||||
return async ({ messages }): Promise<StageExecutorResult> => {
|
||||
const result = await callDeepSeek(messages);
|
||||
return {
|
||||
text: result.text,
|
||||
inputTokens: result.inputTokens,
|
||||
outputTokens: result.outputTokens,
|
||||
provider: "deepseek",
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Benchmark runner
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface BenchResult {
|
||||
question: string;
|
||||
baselineText: string;
|
||||
pipelineText: string;
|
||||
baselineCorrect: boolean;
|
||||
pipelineCorrect: boolean;
|
||||
baselineTokens: { input: number; output: number };
|
||||
pipelineTokens: { input: number; output: number };
|
||||
baselineLatencyMs: number;
|
||||
pipelineLatencyMs: number;
|
||||
stagesExecuted: number;
|
||||
}
|
||||
|
||||
async function runMathBenchmark(): Promise<BenchResult[]> {
|
||||
const results: BenchResult[] = [];
|
||||
const systemMsg = {
|
||||
role: "system",
|
||||
content: "You are a math expert. Answer concisely with just the final answer.",
|
||||
};
|
||||
|
||||
for (const problem of MATH_PROBLEMS) {
|
||||
console.log(` Math: ${problem.q}`);
|
||||
|
||||
// Baseline: single call
|
||||
const baseline = await callDeepSeek([systemMsg, { role: "user", content: problem.q }]);
|
||||
|
||||
// Pipeline: execute + reflect (math pipeline)
|
||||
const pipelineConfig = buildPipelineConfig(problem.q, "math");
|
||||
const pipelineStart = Date.now();
|
||||
const pipeline = await executePipeline(pipelineConfig, makeDeepSeekExecutor());
|
||||
|
||||
results.push({
|
||||
question: problem.q,
|
||||
baselineText: baseline.text,
|
||||
pipelineText: pipeline.text,
|
||||
baselineCorrect: baseline.text.includes(problem.expected),
|
||||
pipelineCorrect: pipeline.text.includes(problem.expected),
|
||||
baselineTokens: { input: baseline.inputTokens, output: baseline.outputTokens },
|
||||
pipelineTokens: {
|
||||
input: pipeline.stages.reduce((s, r) => s + (r.inputTokens ?? 0), 0),
|
||||
output: pipeline.stages.reduce((s, r) => s + (r.outputTokens ?? 0), 0),
|
||||
},
|
||||
baselineLatencyMs: baseline.latencyMs,
|
||||
pipelineLatencyMs: Date.now() - pipelineStart,
|
||||
stagesExecuted: pipeline.stages.length,
|
||||
});
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async function runCodingBenchmark(): Promise<BenchResult[]> {
|
||||
const results: BenchResult[] = [];
|
||||
const systemMsg = {
|
||||
role: "system",
|
||||
content: "You are an expert programmer. Write clean, working code.",
|
||||
};
|
||||
|
||||
for (const problem of CODING_PROBLEMS) {
|
||||
console.log(` Code: ${problem.q.slice(0, 60)}...`);
|
||||
|
||||
// Baseline: single call
|
||||
const baseline = await callDeepSeek([systemMsg, { role: "user", content: problem.q }]);
|
||||
|
||||
// Pipeline: plan + execute + reflect + fix (code pipeline)
|
||||
const pipelineConfig = buildPipelineConfig(problem.q, "code");
|
||||
const pipelineStart = Date.now();
|
||||
const pipeline = await executePipeline(pipelineConfig, makeDeepSeekExecutor());
|
||||
|
||||
results.push({
|
||||
question: problem.q,
|
||||
baselineText: baseline.text,
|
||||
pipelineText: pipeline.text,
|
||||
baselineCorrect: problem.check(baseline.text),
|
||||
pipelineCorrect: problem.check(pipeline.text),
|
||||
baselineTokens: { input: baseline.inputTokens, output: baseline.outputTokens },
|
||||
pipelineTokens: {
|
||||
input: pipeline.stages.reduce((s, r) => s + (r.inputTokens ?? 0), 0),
|
||||
output: pipeline.stages.reduce((s, r) => s + (r.outputTokens ?? 0), 0),
|
||||
},
|
||||
baselineLatencyMs: baseline.latencyMs,
|
||||
pipelineLatencyMs: Date.now() - pipelineStart,
|
||||
stagesExecuted: pipeline.stages.length,
|
||||
});
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function main() {
|
||||
if (!API_KEY) {
|
||||
console.error("DEEPSEEK_API_KEY not set");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log("=== Smart Auto Pipeline Benchmark (Direct Execution) ===\n");
|
||||
console.log(`Provider: ${MODEL} via ${BASE_URL}`);
|
||||
console.log(`Pipeline stages: math=[execute→reflect], code=[plan→execute→reflect→fix]\n`);
|
||||
|
||||
console.log("--- Math Problems ---");
|
||||
const mathResults = await runMathBenchmark();
|
||||
|
||||
console.log("\n--- Coding Problems ---");
|
||||
const codingResults = await runCodingBenchmark();
|
||||
|
||||
const allResults = [...mathResults, ...codingResults];
|
||||
|
||||
// Print details
|
||||
console.log("\n=== Detailed Results ===\n");
|
||||
for (const r of allResults) {
|
||||
const type = mathResults.includes(r) ? "MATH" : "CODE";
|
||||
console.log(`[${type}] ${r.question.slice(0, 55)}...`);
|
||||
console.log(
|
||||
` Baseline: ${r.baselineCorrect ? "CORRECT" : "WRONG"} | ${r.baselineTokens.input + r.baselineTokens.output} tok | ${r.baselineLatencyMs}ms`
|
||||
);
|
||||
console.log(
|
||||
` Pipeline: ${r.pipelineCorrect ? "CORRECT" : "WRONG"} | ${r.pipelineTokens.input + r.pipelineTokens.output} tok | ${r.pipelineLatencyMs}ms | ${r.stagesExecuted} stages`
|
||||
);
|
||||
}
|
||||
|
||||
// Aggregates
|
||||
const mathBC = mathResults.filter((r) => r.baselineCorrect).length;
|
||||
const mathPC = mathResults.filter((r) => r.pipelineCorrect).length;
|
||||
const codeBC = codingResults.filter((r) => r.baselineCorrect).length;
|
||||
const codePC = codingResults.filter((r) => r.pipelineCorrect).length;
|
||||
|
||||
const bTokens = allResults.reduce(
|
||||
(s, r) => s + r.baselineTokens.input + r.baselineTokens.output,
|
||||
0
|
||||
);
|
||||
const pTokens = allResults.reduce(
|
||||
(s, r) => s + r.pipelineTokens.input + r.pipelineTokens.output,
|
||||
0
|
||||
);
|
||||
const bCost = (bTokens / 1_000_000) * COST_INPUT_PER_M;
|
||||
const pCost = (pTokens / 1_000_000) * COST_INPUT_PER_M;
|
||||
const bLatency = Math.round(
|
||||
allResults.reduce((s, r) => s + r.baselineLatencyMs, 0) / allResults.length
|
||||
);
|
||||
const pLatency = Math.round(
|
||||
allResults.reduce((s, r) => s + r.pipelineLatencyMs, 0) / allResults.length
|
||||
);
|
||||
|
||||
console.log("\n=== Summary ===\n");
|
||||
console.log(
|
||||
`Math accuracy: Baseline ${mathBC}/${MATH_PROBLEMS.length} | Pipeline ${mathPC}/${MATH_PROBLEMS.length}`
|
||||
);
|
||||
console.log(
|
||||
`Coding accuracy: Baseline ${codeBC}/${CODING_PROBLEMS.length} | Pipeline ${codePC}/${CODING_PROBLEMS.length}`
|
||||
);
|
||||
console.log(
|
||||
`Total tokens: Baseline ${bTokens} | Pipeline ${pTokens} (${(pTokens / bTokens).toFixed(1)}x)`
|
||||
);
|
||||
console.log(
|
||||
`Estimated cost: Baseline $${bCost.toFixed(4)} | Pipeline $${pCost.toFixed(4)} (${(pCost / bCost).toFixed(1)}x)`
|
||||
);
|
||||
console.log(
|
||||
`Avg latency: Baseline ${bLatency}ms | Pipeline ${pLatency}ms (${(pLatency / bLatency).toFixed(1)}x)`
|
||||
);
|
||||
console.log(`\nNote: Single provider (DeepSeek) for all stages. Multi-provider routing`);
|
||||
console.log(`would reduce cost by using cheap providers for execute/fix stages.`);
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
Reference in New Issue
Block a user