wehub-resource-sync
283 lines
10 KiB
TypeScript
283 lines
10 KiB
TypeScript
/**
|
|
* Pipeline Benchmark — Direct Pipeline Execution
|
|
*
|
|
* Tests Smart Auto Pipeline accuracy and cost by calling executePipeline() directly
|
|
* with DeepSeek API as the stage executor. Bypasses combo routing overhead.
|
|
*
|
|
* Measures: accuracy, token usage, latency, cost — baseline (single call) vs pipeline
|
|
*/
|
|
|
|
import {
|
|
buildPipelineConfig,
|
|
executePipeline,
|
|
type StageExecutor,
|
|
type StageExecutorResult,
|
|
type FitnessTier,
|
|
} from "../../src/domain/pipeline.ts";
|
|
|
|
const API_KEY = process.env.DEEPSEEK_API_KEY;
|
|
const BASE_URL = "https://api.deepseek.com/v1";
|
|
const MODEL = "deepseek-chat";
|
|
|
|
const COST_INPUT_PER_M = 0.14;
|
|
const COST_OUTPUT_PER_M = 0.28;
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Test cases
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const MATH_PROBLEMS = [
|
|
{ q: "What is 17 * 23?", expected: "391" },
|
|
{ q: "Solve for x: 2x + 5 = 13", expected: "4" },
|
|
{ q: "What is the derivative of x^3 + 2x?", expected: "3x^2 + 2" },
|
|
{ q: "What is the integral of 2x dx?", expected: "x^2" },
|
|
{ q: "If a triangle has sides 3, 4, 5, what is its area?", expected: "6" },
|
|
];
|
|
|
|
const CODING_PROBLEMS = [
|
|
{
|
|
q: "Write a JavaScript function that returns the factorial of n.",
|
|
check: (r: string) =>
|
|
/function\s+\w*factorial|factorial\s*=|const\s+factorial/i.test(r) &&
|
|
/return|n\s*\*/i.test(r),
|
|
},
|
|
{
|
|
q: "Write a Python function that checks if a string is a palindrome.",
|
|
check: (r: string) => /def\s+\w*pali|pali.*def/i.test(r) && /return|==/i.test(r),
|
|
},
|
|
{
|
|
q: "Write a TypeScript function that reverses an array without mutating it.",
|
|
check: (r: string) => /reverse|slice|spread|\.\.\./i.test(r),
|
|
},
|
|
{
|
|
q: "Write a SQL query to find the second highest salary from an employees table.",
|
|
check: (r: string) =>
|
|
/SELECT|select/i.test(r) && /salary|LIMIT|OFFSET|DENSE_RANK|ROW_NUMBER/i.test(r),
|
|
},
|
|
{
|
|
q: "Write a bash one-liner to count the number of lines in all .ts files recursively.",
|
|
check: (r: string) => /find|grep|wc|cat/i.test(r) && /-l|lines|count/i.test(r),
|
|
},
|
|
];
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// API call helper
|
|
// ---------------------------------------------------------------------------
|
|
|
|
interface CallResult {
|
|
text: string;
|
|
inputTokens: number;
|
|
outputTokens: number;
|
|
latencyMs: number;
|
|
}
|
|
|
|
async function callDeepSeek(
|
|
messages: Array<{ role: string; content: string }>
|
|
): Promise<CallResult> {
|
|
const start = Date.now();
|
|
const res = await fetch(`${BASE_URL}/chat/completions`, {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json", Authorization: `Bearer ${API_KEY}` },
|
|
body: JSON.stringify({ model: MODEL, messages, stream: false }),
|
|
});
|
|
if (!res.ok) throw new Error(`DeepSeek error ${res.status}: ${await res.text()}`);
|
|
const data = (await res.json()) as Record<string, unknown>;
|
|
const usage = data.usage as Record<string, number> | undefined;
|
|
const msg = (data.choices as Array<Record<string, unknown>>)?.[0]?.message as
|
|
| Record<string, unknown>
|
|
| undefined;
|
|
return {
|
|
text: (msg?.content as string) ?? "",
|
|
inputTokens: usage?.prompt_tokens ?? 0,
|
|
outputTokens: usage?.completion_tokens ?? 0,
|
|
latencyMs: Date.now() - start,
|
|
};
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Stage executor — calls DeepSeek directly
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function makeDeepSeekExecutor(): StageExecutor {
|
|
return async ({ messages }): Promise<StageExecutorResult> => {
|
|
const result = await callDeepSeek(messages);
|
|
return {
|
|
text: result.text,
|
|
inputTokens: result.inputTokens,
|
|
outputTokens: result.outputTokens,
|
|
provider: "deepseek",
|
|
};
|
|
};
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Benchmark runner
|
|
// ---------------------------------------------------------------------------
|
|
|
|
interface BenchResult {
|
|
question: string;
|
|
baselineText: string;
|
|
pipelineText: string;
|
|
baselineCorrect: boolean;
|
|
pipelineCorrect: boolean;
|
|
baselineTokens: { input: number; output: number };
|
|
pipelineTokens: { input: number; output: number };
|
|
baselineLatencyMs: number;
|
|
pipelineLatencyMs: number;
|
|
stagesExecuted: number;
|
|
}
|
|
|
|
async function runMathBenchmark(): Promise<BenchResult[]> {
|
|
const results: BenchResult[] = [];
|
|
const systemMsg = {
|
|
role: "system",
|
|
content: "You are a math expert. Answer concisely with just the final answer.",
|
|
};
|
|
|
|
for (const problem of MATH_PROBLEMS) {
|
|
console.log(` Math: ${problem.q}`);
|
|
|
|
// Baseline: single call
|
|
const baseline = await callDeepSeek([systemMsg, { role: "user", content: problem.q }]);
|
|
|
|
// Pipeline: execute + reflect (math pipeline)
|
|
const pipelineConfig = buildPipelineConfig(problem.q, "math");
|
|
const pipelineStart = Date.now();
|
|
const pipeline = await executePipeline(pipelineConfig, makeDeepSeekExecutor());
|
|
|
|
results.push({
|
|
question: problem.q,
|
|
baselineText: baseline.text,
|
|
pipelineText: pipeline.text,
|
|
baselineCorrect: baseline.text.includes(problem.expected),
|
|
pipelineCorrect: pipeline.text.includes(problem.expected),
|
|
baselineTokens: { input: baseline.inputTokens, output: baseline.outputTokens },
|
|
pipelineTokens: {
|
|
input: pipeline.stages.reduce((s, r) => s + (r.inputTokens ?? 0), 0),
|
|
output: pipeline.stages.reduce((s, r) => s + (r.outputTokens ?? 0), 0),
|
|
},
|
|
baselineLatencyMs: baseline.latencyMs,
|
|
pipelineLatencyMs: Date.now() - pipelineStart,
|
|
stagesExecuted: pipeline.stages.length,
|
|
});
|
|
}
|
|
return results;
|
|
}
|
|
|
|
async function runCodingBenchmark(): Promise<BenchResult[]> {
|
|
const results: BenchResult[] = [];
|
|
const systemMsg = {
|
|
role: "system",
|
|
content: "You are an expert programmer. Write clean, working code.",
|
|
};
|
|
|
|
for (const problem of CODING_PROBLEMS) {
|
|
console.log(` Code: ${problem.q.slice(0, 60)}...`);
|
|
|
|
// Baseline: single call
|
|
const baseline = await callDeepSeek([systemMsg, { role: "user", content: problem.q }]);
|
|
|
|
// Pipeline: plan + execute + reflect + fix (code pipeline)
|
|
const pipelineConfig = buildPipelineConfig(problem.q, "code");
|
|
const pipelineStart = Date.now();
|
|
const pipeline = await executePipeline(pipelineConfig, makeDeepSeekExecutor());
|
|
|
|
results.push({
|
|
question: problem.q,
|
|
baselineText: baseline.text,
|
|
pipelineText: pipeline.text,
|
|
baselineCorrect: problem.check(baseline.text),
|
|
pipelineCorrect: problem.check(pipeline.text),
|
|
baselineTokens: { input: baseline.inputTokens, output: baseline.outputTokens },
|
|
pipelineTokens: {
|
|
input: pipeline.stages.reduce((s, r) => s + (r.inputTokens ?? 0), 0),
|
|
output: pipeline.stages.reduce((s, r) => s + (r.outputTokens ?? 0), 0),
|
|
},
|
|
baselineLatencyMs: baseline.latencyMs,
|
|
pipelineLatencyMs: Date.now() - pipelineStart,
|
|
stagesExecuted: pipeline.stages.length,
|
|
});
|
|
}
|
|
return results;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Main
|
|
// ---------------------------------------------------------------------------
|
|
|
|
async function main() {
|
|
if (!API_KEY) {
|
|
console.error("DEEPSEEK_API_KEY not set");
|
|
process.exit(1);
|
|
}
|
|
|
|
console.log("=== Smart Auto Pipeline Benchmark (Direct Execution) ===\n");
|
|
console.log(`Provider: ${MODEL} via ${BASE_URL}`);
|
|
console.log(`Pipeline stages: math=[execute→reflect], code=[plan→execute→reflect→fix]\n`);
|
|
|
|
console.log("--- Math Problems ---");
|
|
const mathResults = await runMathBenchmark();
|
|
|
|
console.log("\n--- Coding Problems ---");
|
|
const codingResults = await runCodingBenchmark();
|
|
|
|
const allResults = [...mathResults, ...codingResults];
|
|
|
|
// Print details
|
|
console.log("\n=== Detailed Results ===\n");
|
|
for (const r of allResults) {
|
|
const type = mathResults.includes(r) ? "MATH" : "CODE";
|
|
console.log(`[${type}] ${r.question.slice(0, 55)}...`);
|
|
console.log(
|
|
` Baseline: ${r.baselineCorrect ? "CORRECT" : "WRONG"} | ${r.baselineTokens.input + r.baselineTokens.output} tok | ${r.baselineLatencyMs}ms`
|
|
);
|
|
console.log(
|
|
` Pipeline: ${r.pipelineCorrect ? "CORRECT" : "WRONG"} | ${r.pipelineTokens.input + r.pipelineTokens.output} tok | ${r.pipelineLatencyMs}ms | ${r.stagesExecuted} stages`
|
|
);
|
|
}
|
|
|
|
// Aggregates
|
|
const mathBC = mathResults.filter((r) => r.baselineCorrect).length;
|
|
const mathPC = mathResults.filter((r) => r.pipelineCorrect).length;
|
|
const codeBC = codingResults.filter((r) => r.baselineCorrect).length;
|
|
const codePC = codingResults.filter((r) => r.pipelineCorrect).length;
|
|
|
|
const bTokens = allResults.reduce(
|
|
(s, r) => s + r.baselineTokens.input + r.baselineTokens.output,
|
|
0
|
|
);
|
|
const pTokens = allResults.reduce(
|
|
(s, r) => s + r.pipelineTokens.input + r.pipelineTokens.output,
|
|
0
|
|
);
|
|
const bCost = (bTokens / 1_000_000) * COST_INPUT_PER_M;
|
|
const pCost = (pTokens / 1_000_000) * COST_INPUT_PER_M;
|
|
const bLatency = Math.round(
|
|
allResults.reduce((s, r) => s + r.baselineLatencyMs, 0) / allResults.length
|
|
);
|
|
const pLatency = Math.round(
|
|
allResults.reduce((s, r) => s + r.pipelineLatencyMs, 0) / allResults.length
|
|
);
|
|
|
|
console.log("\n=== Summary ===\n");
|
|
console.log(
|
|
`Math accuracy: Baseline ${mathBC}/${MATH_PROBLEMS.length} | Pipeline ${mathPC}/${MATH_PROBLEMS.length}`
|
|
);
|
|
console.log(
|
|
`Coding accuracy: Baseline ${codeBC}/${CODING_PROBLEMS.length} | Pipeline ${codePC}/${CODING_PROBLEMS.length}`
|
|
);
|
|
console.log(
|
|
`Total tokens: Baseline ${bTokens} | Pipeline ${pTokens} (${(pTokens / bTokens).toFixed(1)}x)`
|
|
);
|
|
console.log(
|
|
`Estimated cost: Baseline $${bCost.toFixed(4)} | Pipeline $${pCost.toFixed(4)} (${(pCost / bCost).toFixed(1)}x)`
|
|
);
|
|
console.log(
|
|
`Avg latency: Baseline ${bLatency}ms | Pipeline ${pLatency}ms (${(pLatency / bLatency).toFixed(1)}x)`
|
|
);
|
|
console.log(`\nNote: Single provider (DeepSeek) for all stages. Multi-provider routing`);
|
|
console.log(`would reduce cost by using cheap providers for execute/fix stages.`);
|
|
}
|
|
|
|
main().catch(console.error);
|