23f7624596
ADR-166 MCP Bridge Security Lock / Static-source security lock (push) Failing after 0s
ADR-166 MCP Bridge Security Lock / Compose default binds loopback + Mongo has auth (push) Failing after 2s
CodeQL Advanced / Analyze (rust) (push) Failing after 0s
ADR-166 MCP Bridge Security Lock / plugin-agent-federation bindHost default (push) Failing after 1s
ADR-166 MCP Bridge Security Lock / Runtime behavior — 401 + terminal gate + fail-closed (push) Failing after 4s
business-pods-smoke / smoke (push) Failing after 1s
all-plugins-smoke / smoke-all (push) Failing after 2s
CI/CD Pipeline / Security & Code Quality (push) Failing after 1s
CI/CD Pipeline / Test Suite (ubuntu-latest) (push) Failing after 1s
CI/CD Pipeline / Build & Package (macos-latest) (push) Has been skipped
CI/CD Pipeline / Build & Package (ubuntu-latest) (push) Has been skipped
CI/CD Pipeline / Build & Package (windows-latest) (push) Has been skipped
CI/CD Pipeline / Documentation & Examples (push) Failing after 1s
Clone Tracker (14-day rolling) / Snapshot clones for ruflo ecosystem (push) Failing after 1s
CodeQL Advanced / Analyze (actions) (push) Failing after 1s
CodeQL Advanced / Analyze (javascript-typescript) (push) Failing after 1s
federation-peer-rust / stable-noop (push) Failing after 1s
metaharness-ci / score (push) Failing after 1s
metaharness-ci / router-compat (push) Failing after 0s
metaharness-ci / similarity-tests (push) Failing after 0s
no-agentbbs-smoke / smoke-without-agentbbs (push) Failing after 1s
V3 CI/CD Pipeline / Build V3 (windows-latest) (push) Has been skipped
codex-integration-audit / Codex integration audit (push) Failing after 1s
helpers-manifest-guard / guard (push) Failing after 1s
🔗 Cross-Agent Integration Tests / 🤝 Agent Coordination Tests (push) Has been skipped
🔗 Cross-Agent Integration Tests / 🧠 Memory Sharing Integration (push) Has been skipped
🔗 Cross-Agent Integration Tests / 🛡️ Fault Tolerance Tests (push) Has been skipped
🔗 Cross-Agent Integration Tests / ⚡ Performance Integration Tests (push) Has been skipped
metaharness-ci / mcp-scan (push) Failing after 1s
metaharness-ci / eject-dryrun (push) Failing after 1s
metaharness-ci / metaharness-real-data (push) Failing after 0s
no-cli-optdep-bloat-2561 / guard (push) Failing after 1s
no-metaharness-smoke / smoke-without-metaharness (push) Failing after 1s
no-phantom-agentic-flow-subpath / guard (push) Failing after 1s
🔄 Automated Rollback Manager / 🚨 Failure Detection (push) Failing after 1s
V3 CI/CD Pipeline / Plugin hooks smoke / ubuntu-latest / Node 22 (push) Failing after 1s
V3 CI/CD Pipeline / ruflo-graph-intelligence build + test smoke (#2044, ADR-123) (push) Failing after 1s
CVE Audit Gate / Audit root (critical-blocking) (push) Failing after 2s
cost-tracker-smoke / smoke (push) Failing after 3s
oia-audit-weekly / audit (push) Failing after 2s
ruflo-agent-smoke / ruflo-agent structural smoke (push) Failing after 1s
📊 Status Badges Update / 📊 Update Status Badges (push) Failing after 1s
V3 CI/CD Pipeline / Static regression guards (#2267 YAML + (push) Failing after 1s
V3 CI/CD Pipeline / Test V3 Packages (push) Failing after 0s
V3 CI/CD Pipeline / agent_execute provider routing smoke (#2042) (push) Failing after 0s
CVE Audit Gate / Audit v3 (critical-blocking) (push) Failing after 1s
federation-peer-rust / stable-native (push) Failing after 2s
🔗 Cross-Agent Integration Tests / 🚀 Integration Test Setup (push) Failing after 2s
neural-trader-smoke / runtime-smoke (push) Failing after 1s
V3 CI/CD Pipeline / Build V3 (macos-latest) (push) Has been skipped
V3 CI/CD Pipeline / Build V3 (ubuntu-latest) (push) Has been skipped
V3 CI/CD Pipeline / Type Check V3 (push) Failing after 1s
V3 CI/CD Pipeline / Smoke (no better-sqlite3) / ubuntu-latest / Node 24 (push) Failing after 1s
V3 CI/CD Pipeline / Smoke (no better-sqlite3) / ubuntu-latest / Node 22 (push) Failing after 2s
V3 CI/CD Pipeline / browser rvf create flag smoke (#2015) (push) Failing after 0s
V3 CI/CD Pipeline / Dependency review (#2046) (push) Has been skipped
V3 CI/CD Pipeline / Supply-chain audit (#2046) (push) Failing after 0s
V3 CI/CD Pipeline / witness marker drift smoke (#2021) (push) Failing after 1s
V3 CI/CD Pipeline / neural-trader portfolio CG smoke (#2068, ADR-126 Phase 3) (push) Failing after 1s
V3 CI/CD Pipeline / neural-trader backtest signing smoke (#2068, ADR-126 Phase 4) (push) Failing after 1s
V3 CI/CD Pipeline / kg-extract type-import classification smoke (#2049) (push) Failing after 0s
V3 CI/CD Pipeline / witness verify precondition smoke (#1880) (push) Failing after 2s
V3 CI/CD Pipeline / neural-trader pipeline risk-gate smoke (#2068, ADR-126 Phase 5) (push) Failing after 0s
V3 CI/CD Pipeline / neural-trader feature attribution smoke (#2068, ADR-126 Phase 6) (push) Failing after 0s
V3 CI/CD Pipeline / plugin-registry signature verification smoke (#1922, CWE-347) (push) Failing after 4s
V3 CI/CD Pipeline / memory stats legacy-DB smoke (#2120) (push) Failing after 4s
V3 CI/CD Pipeline / github deprecated actions smoke (#2089, ADR-127 Phase 3) (push) Failing after 1s
V3 CI/CD Pipeline / graph query + pathfinder smoke (ADR-130 P2+P5) (push) Has been skipped
V3 CI/CD Pipeline / graph trajectory hooks smoke (ADR-130 P3) (push) Has been skipped
V3 CI/CD Pipeline / graph plugin adapter smoke (ADR-130 P4) (push) Has been skipped
V3 CI/CD Pipeline / graph benchmark (ADR-130 P6) (push) Has been skipped
V3 CI/CD Pipeline / statusline generator delegation smoke (#2195) (push) Failing after 1s
V3 CI/CD Pipeline / wizard init regression guard (#2206 (push) Failing after 1s
V3 CI/CD Pipeline / memory no-stray-db smoke (ADR-125 P7) (push) Failing after 1s
V3 CI/CD Pipeline / github-safe injection smoke (#2089, ADR-127 Phase 1) (push) Failing after 1s
V3 CI/CD Pipeline / github actions pin smoke (#2089, ADR-127 Phase 1) (push) Failing after 1s
V3 CI/CD Pipeline / github attribution opt-in smoke (#2089, ADR-127 Phase 4) (push) Failing after 1s
V3 CI/CD Pipeline / pre-bash hook safety smoke (#2017) (push) Failing after 1s
V3 CI/CD Pipeline / Memory import smoke / ubuntu-latest (push) Failing after 0s
V3 CI/CD Pipeline / MCP protocol smoke / ubuntu-latest (push) Failing after 2s
V3 CI/CD Pipeline / ruvllm WASM auto-init smoke (#2086) (push) Failing after 4s
V3 CI/CD Pipeline / MCP paired-tool round-trip smoke (#1889) (push) Failing after 1s
V3 CI/CD Pipeline / Plugin package install-safety (#1902/#1903/#1904) (push) Failing after 1s
V3 CI/CD Pipeline / Tool description discoverability (ADR-112) (push) Failing after 3s
V3 CI/CD Pipeline / CLI npx-install smoke (#1147 / (22) (push) Failing after 1s
V3 CI/CD Pipeline / CLI npx-install smoke (#1147 / (24) (push) Failing after 1s
V3 CI/CD Pipeline / Windows hook shim smoke (#2132) / ubuntu-latest (push) Failing after 2s
V3 CI/CD Pipeline / Windows hook execution smoke (#2132) / ubuntu-latest (push) Failing after 1s
V3 CI/CD Pipeline / Windows init hooks smoke (#2132) / ubuntu-latest (push) Failing after 1s
V3 CI/CD Pipeline / Vector-index dimension audit (#1947) (push) Failing after 0s
V3 CI/CD Pipeline / Hook-command install safety (#1921) (push) Failing after 1s
V3 CI/CD Pipeline / ToolOutputGuardrail smoke (ADR-131, (push) Failing after 1s
V3 CI/CD Pipeline / init-bundle invariants smoke (#2095, ADR-128 Phase 5) (push) Failing after 1s
V3 CI/CD Pipeline / wasm provider bridge smoke (ADR-129 P1) (push) Failing after 2s
V3 CI/CD Pipeline / wasm gallery CRUD smoke (ADR-129 P3) (push) Failing after 1s
V3 CI/CD Pipeline / wasm plugin bridge smoke (ADR-129 P4) (push) Failing after 0s
V3 CI/CD Pipeline / wasm compose smoke (ADR-129 P2) (push) Failing after 4s
V3 CI/CD Pipeline / graph schema smoke (ADR-130 P1) (push) Failing after 0s
Validate Marketplace / validate (push) Failing after 1s
🔍 Verification Pipeline / 🚀 Setup Verification (push) Failing after 1s
🔍 Verification Pipeline / 🛡️ Security Verification (push) Has been skipped
🔍 Verification Pipeline / 📝 Code Quality (push) Has been skipped
🔍 Verification Pipeline / 🧪 Test Verification (${{ matrix.os }}, Node ${{ matrix.node }}) (push) Has been skipped
🔍 Verification Pipeline / 🏗️ Build Verification (push) Has been skipped
🔍 Verification Pipeline / 📚 Documentation Verification (push) Has been skipped
CVE Audit Gate / High-severity report (warn only) (push) Has been cancelled
🔄 Automated Rollback Manager / 🔄 Execute Rollback (push) Has been cancelled
🔄 Automated Rollback Manager / ✅ Post-Rollback Verification (push) Has been cancelled
🔄 Automated Rollback Manager / 📊 Rollback Monitoring (push) Has been cancelled
V3 CI/CD Pipeline / Windows init hooks smoke (#2132) / windows-latest (push) Has been cancelled
V3 CI/CD Pipeline / Windows hook execution smoke (#2132) / macos-latest (push) Has been cancelled
V3 CI/CD Pipeline / Windows hook execution smoke (#2132) / windows-latest (push) Has been cancelled
🔄 Automated Rollback Manager / ⏳ Manual Rollback Approval (push) Has been cancelled
V3 CI/CD Pipeline / MCP protocol smoke / macos-latest (push) Has been cancelled
V3 CI/CD Pipeline / Memory import smoke / macos-latest (push) Has been cancelled
V3 CI/CD Pipeline / Windows hook shim smoke (#2132) / macos-latest (push) Has been cancelled
V3 CI/CD Pipeline / Windows hook shim smoke (#2132) / windows-latest (push) Has been cancelled
V3 CI/CD Pipeline / Windows init hooks smoke (#2132) / macos-latest (push) Has been cancelled
V3 CI/CD Pipeline / Witness verify (signed manifest) / macos-latest (push) Has been cancelled
V3 CI/CD Pipeline / Witness verify (signed manifest) / ubuntu-latest (push) Has been cancelled
V3 CI/CD Pipeline / Witness verify (signed manifest) / windows-latest (push) Has been cancelled
V3 CI/CD Pipeline / Publish to npm (alpha) (push) Has been cancelled
V3 CI/CD Pipeline / Smoke (no better-sqlite3) / macos-latest / Node 22 (push) Has been cancelled
V3 CI/CD Pipeline / Plugin hooks smoke / macos-latest / Node 22 (push) Has been cancelled
CI/CD Pipeline / Deploy & Release (push) Has been cancelled
CI/CD Pipeline / CI Status (push) Has been cancelled
🔗 Cross-Agent Integration Tests / 📊 Integration Test Report (push) Has been cancelled
🔄 Automated Rollback Manager / 🔍 Pre-Rollback Validation (push) Has been cancelled
🔍 Verification Pipeline / ⚡ Performance Verification (push) Has been cancelled
🔍 Verification Pipeline / 📊 Verification Report (push) Has been cancelled
761 lines
26 KiB
TypeScript
761 lines
26 KiB
TypeScript
/**
|
|
* Tests for the Evolution Pipeline
|
|
*/
|
|
import { describe, it, expect, beforeEach } from 'vitest';
|
|
import {
|
|
EvolutionPipeline,
|
|
createEvolutionPipeline,
|
|
} from '../src/evolution.js';
|
|
import type {
|
|
ChangeProposal,
|
|
SimulationResult,
|
|
StagedRollout,
|
|
TraceEvaluator,
|
|
RolloutStage,
|
|
} from '../src/evolution.js';
|
|
|
|
// ============================================================================
|
|
// Helpers
|
|
// ============================================================================
|
|
|
|
function makeProposalParams(overrides: Record<string, unknown> = {}) {
|
|
return {
|
|
kind: 'rule-modify' as const,
|
|
title: 'Improve rule R001',
|
|
description: 'Clarify wording for R001 to reduce violations',
|
|
author: 'optimizer-agent',
|
|
targetPath: 'rules.R001.text',
|
|
diff: { before: 'old text', after: 'new text' },
|
|
rationale: 'R001 is violated 12 times per cycle',
|
|
riskAssessment: { level: 'low' as const, factors: ['wording-only change'] },
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Evaluator that returns identical results for both configs (zero divergence).
|
|
*/
|
|
const identicalEvaluator: TraceEvaluator = (trace, config) => ({
|
|
traceHash: 'aaa',
|
|
metrics: { accuracy: 0.95, latency: 100 },
|
|
decisions: [{ action: 'allow' }, { action: 'allow' }],
|
|
});
|
|
|
|
/**
|
|
* Evaluator that returns completely different results per config (high divergence).
|
|
*/
|
|
const divergentEvaluator: TraceEvaluator = (trace, config) => {
|
|
if (config === 'baseline') {
|
|
return {
|
|
traceHash: 'baseline-hash',
|
|
metrics: { accuracy: 0.95, latency: 100 },
|
|
decisions: [
|
|
{ action: 'allow' },
|
|
{ action: 'allow' },
|
|
{ action: 'warn' },
|
|
],
|
|
};
|
|
}
|
|
return {
|
|
traceHash: 'candidate-hash',
|
|
metrics: { accuracy: 0.60, latency: 200 },
|
|
decisions: [
|
|
{ action: 'block' },
|
|
{ action: 'block' },
|
|
{ action: 'block' },
|
|
{ action: 'block' },
|
|
{ action: 'block' },
|
|
],
|
|
};
|
|
};
|
|
|
|
/**
|
|
* Evaluator with slightly different decisions but within threshold.
|
|
*/
|
|
const slightlyDivergentEvaluator: TraceEvaluator = (trace, config) => {
|
|
if (config === 'baseline') {
|
|
return {
|
|
traceHash: 'baseline-hash',
|
|
metrics: { accuracy: 0.90, latency: 100 },
|
|
decisions: [{ action: 'allow' }, { action: 'warn' }],
|
|
};
|
|
}
|
|
return {
|
|
traceHash: 'candidate-hash',
|
|
metrics: { accuracy: 0.92, latency: 95 },
|
|
decisions: [{ action: 'allow' }, { action: 'allow' }],
|
|
};
|
|
};
|
|
|
|
// ============================================================================
|
|
// Tests
|
|
// ============================================================================
|
|
|
|
describe('EvolutionPipeline', () => {
|
|
let pipeline: EvolutionPipeline;
|
|
|
|
beforeEach(() => {
|
|
pipeline = createEvolutionPipeline({
|
|
signingKey: 'test-evolution-key',
|
|
maxDivergence: 0.3,
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// propose
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('propose', () => {
|
|
it('should create a signed proposal with a UUID', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
|
|
expect(proposal.proposalId).toBeTruthy();
|
|
expect(proposal.proposalId).toMatch(
|
|
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/,
|
|
);
|
|
});
|
|
|
|
it('should produce a valid HMAC-SHA256 signature', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
|
|
expect(proposal.signature).toMatch(/^[a-f0-9]{64}$/);
|
|
});
|
|
|
|
it('should set status to signed after creation', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
|
|
expect(proposal.status).toBe('signed');
|
|
});
|
|
|
|
it('should preserve all proposal fields', () => {
|
|
const params = makeProposalParams({
|
|
kind: 'rule-add',
|
|
title: 'Add rule R099',
|
|
author: 'human-1',
|
|
targetPath: 'rules.R099',
|
|
});
|
|
const proposal = pipeline.propose(params);
|
|
|
|
expect(proposal.kind).toBe('rule-add');
|
|
expect(proposal.title).toBe('Add rule R099');
|
|
expect(proposal.author).toBe('human-1');
|
|
expect(proposal.targetPath).toBe('rules.R099');
|
|
expect(proposal.diff).toEqual({ before: 'old text', after: 'new text' });
|
|
expect(proposal.riskAssessment.level).toBe('low');
|
|
expect(proposal.createdAt).toBeGreaterThan(0);
|
|
});
|
|
|
|
it('should produce different signatures for different proposals', () => {
|
|
const p1 = pipeline.propose(makeProposalParams({ title: 'Change A' }));
|
|
const p2 = pipeline.propose(makeProposalParams({ title: 'Change B' }));
|
|
|
|
expect(p1.signature).not.toBe(p2.signature);
|
|
});
|
|
|
|
it('should store the proposal for later retrieval', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
|
|
expect(pipeline.getProposal(proposal.proposalId)).toBe(proposal);
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// simulate
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('simulate', () => {
|
|
it('should compute zero divergence for identical evaluators', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
const goldenTraces = [{ input: 'test-1' }, { input: 'test-2' }];
|
|
|
|
const result = pipeline.simulate(proposal.proposalId, goldenTraces, identicalEvaluator);
|
|
|
|
expect(result.divergenceScore).toBe(0);
|
|
expect(result.passed).toBe(true);
|
|
expect(result.decisionDiffs).toHaveLength(0);
|
|
});
|
|
|
|
it('should compute non-zero divergence when decisions differ', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
const goldenTraces = [{ input: 'test-1' }];
|
|
|
|
const result = pipeline.simulate(
|
|
proposal.proposalId,
|
|
goldenTraces,
|
|
slightlyDivergentEvaluator,
|
|
);
|
|
|
|
expect(result.divergenceScore).toBeGreaterThan(0);
|
|
expect(result.decisionDiffs.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
it('should include baseline and candidate trace hashes', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
const result = pipeline.simulate(
|
|
proposal.proposalId,
|
|
[{ input: 'trace' }],
|
|
divergentEvaluator,
|
|
);
|
|
|
|
expect(result.baselineTraceHash).toMatch(/^[a-f0-9]{64}$/);
|
|
expect(result.candidateTraceHash).toMatch(/^[a-f0-9]{64}$/);
|
|
expect(result.baselineTraceHash).not.toBe(result.candidateTraceHash);
|
|
});
|
|
|
|
it('should aggregate metrics from all traces', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
const result = pipeline.simulate(
|
|
proposal.proposalId,
|
|
[{ input: '1' }, { input: '2' }],
|
|
identicalEvaluator,
|
|
);
|
|
|
|
expect(result.metricsComparison.baseline.accuracy).toBe(0.95);
|
|
expect(result.metricsComparison.candidate.accuracy).toBe(0.95);
|
|
});
|
|
|
|
it('should set proposal status to simulating during evaluation', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
|
|
pipeline.simulate(proposal.proposalId, [{ input: '1' }], identicalEvaluator);
|
|
|
|
// After simulation completes, status may still be 'simulating' until compare
|
|
// is called, but the simulate method itself sets it.
|
|
const fetched = pipeline.getProposal(proposal.proposalId);
|
|
expect(fetched?.status).toBe('simulating');
|
|
});
|
|
|
|
it('should throw for unknown proposal', () => {
|
|
expect(() =>
|
|
pipeline.simulate('nonexistent-id', [], identicalEvaluator),
|
|
).toThrow('Proposal not found');
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// compare
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('compare', () => {
|
|
it('should approve low-divergence changes', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
const simResult = pipeline.simulate(
|
|
proposal.proposalId,
|
|
[{ input: '1' }],
|
|
identicalEvaluator,
|
|
);
|
|
|
|
const comparison = pipeline.compare(proposal.proposalId, simResult);
|
|
|
|
expect(comparison.approved).toBe(true);
|
|
expect(comparison.reason).toContain('within threshold');
|
|
});
|
|
|
|
it('should reject high-divergence changes', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
const simResult = pipeline.simulate(
|
|
proposal.proposalId,
|
|
[{ input: '1' }, { input: '2' }, { input: '3' }],
|
|
divergentEvaluator,
|
|
);
|
|
|
|
// Force a high divergence score for the test
|
|
const highDivResult: SimulationResult = {
|
|
...simResult,
|
|
divergenceScore: 0.8,
|
|
};
|
|
|
|
const comparison = pipeline.compare(proposal.proposalId, highDivResult);
|
|
|
|
expect(comparison.approved).toBe(false);
|
|
expect(comparison.reason).toContain('exceeds threshold');
|
|
});
|
|
|
|
it('should reject changes with significant metric regressions', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
|
|
const simResult: SimulationResult = {
|
|
proposalId: proposal.proposalId,
|
|
baselineTraceHash: 'a'.repeat(64),
|
|
candidateTraceHash: 'b'.repeat(64),
|
|
divergenceScore: 0.1, // low divergence, should pass that check
|
|
decisionDiffs: [],
|
|
metricsComparison: {
|
|
baseline: { accuracy: 1.0, latency: 100 },
|
|
candidate: { accuracy: 0.5, latency: 100 }, // 50% regression in accuracy
|
|
},
|
|
passed: true,
|
|
reason: 'test',
|
|
};
|
|
|
|
const comparison = pipeline.compare(proposal.proposalId, simResult);
|
|
|
|
expect(comparison.approved).toBe(false);
|
|
expect(comparison.reason).toContain('regressed');
|
|
});
|
|
|
|
it('should set proposal status to compared on approval', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
const simResult = pipeline.simulate(
|
|
proposal.proposalId,
|
|
[{ input: '1' }],
|
|
identicalEvaluator,
|
|
);
|
|
|
|
pipeline.compare(proposal.proposalId, simResult);
|
|
|
|
expect(pipeline.getProposal(proposal.proposalId)?.status).toBe('compared');
|
|
});
|
|
|
|
it('should set proposal status to rejected on rejection', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
|
|
const simResult: SimulationResult = {
|
|
proposalId: proposal.proposalId,
|
|
baselineTraceHash: 'a'.repeat(64),
|
|
candidateTraceHash: 'b'.repeat(64),
|
|
divergenceScore: 0.9,
|
|
decisionDiffs: [],
|
|
metricsComparison: { baseline: {}, candidate: {} },
|
|
passed: false,
|
|
reason: 'too divergent',
|
|
};
|
|
|
|
pipeline.compare(proposal.proposalId, simResult);
|
|
|
|
expect(pipeline.getProposal(proposal.proposalId)?.status).toBe('rejected');
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// stage
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('stage', () => {
|
|
it('should create a staged rollout plan with default stages', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
|
|
const rollout = pipeline.stage(proposal.proposalId);
|
|
|
|
expect(rollout.rolloutId).toBeTruthy();
|
|
expect(rollout.proposalId).toBe(proposal.proposalId);
|
|
expect(rollout.stages).toHaveLength(3);
|
|
expect(rollout.stages[0].name).toBe('canary');
|
|
expect(rollout.stages[1].name).toBe('partial');
|
|
expect(rollout.stages[2].name).toBe('full');
|
|
expect(rollout.currentStage).toBe(0);
|
|
expect(rollout.status).toBe('in-progress');
|
|
expect(rollout.startedAt).toBeGreaterThan(0);
|
|
expect(rollout.completedAt).toBeNull();
|
|
});
|
|
|
|
it('should start the first stage immediately', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
const rollout = pipeline.stage(proposal.proposalId);
|
|
|
|
expect(rollout.stages[0].startedAt).toBeGreaterThan(0);
|
|
expect(rollout.stages[1].startedAt).toBeNull();
|
|
});
|
|
|
|
it('should set proposal status to staged', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
pipeline.stage(proposal.proposalId);
|
|
|
|
expect(pipeline.getProposal(proposal.proposalId)?.status).toBe('staged');
|
|
});
|
|
|
|
it('should use custom stages when provided', () => {
|
|
const customStages: RolloutStage[] = [
|
|
{
|
|
name: 'alpha',
|
|
percentage: 1,
|
|
durationMs: 10_000,
|
|
metrics: {},
|
|
divergenceThreshold: 0.1,
|
|
passed: null,
|
|
startedAt: null,
|
|
completedAt: null,
|
|
},
|
|
{
|
|
name: 'beta',
|
|
percentage: 100,
|
|
durationMs: 30_000,
|
|
metrics: {},
|
|
divergenceThreshold: 0.2,
|
|
passed: null,
|
|
startedAt: null,
|
|
completedAt: null,
|
|
},
|
|
];
|
|
|
|
const customPipeline = createEvolutionPipeline({
|
|
signingKey: 'test-key',
|
|
stages: customStages,
|
|
});
|
|
const proposal = customPipeline.propose(makeProposalParams());
|
|
const rollout = customPipeline.stage(proposal.proposalId);
|
|
|
|
expect(rollout.stages).toHaveLength(2);
|
|
expect(rollout.stages[0].name).toBe('alpha');
|
|
expect(rollout.stages[1].name).toBe('beta');
|
|
});
|
|
|
|
it('should store rollout for later retrieval', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
const rollout = pipeline.stage(proposal.proposalId);
|
|
|
|
expect(pipeline.getRollout(rollout.rolloutId)).toBe(rollout);
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// advanceStage
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('advanceStage', () => {
|
|
it('should progress to the next stage when metrics pass', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
const rollout = pipeline.stage(proposal.proposalId);
|
|
|
|
const result = pipeline.advanceStage(rollout.rolloutId, {
|
|
divergence: 0.05,
|
|
accuracy: 0.95,
|
|
});
|
|
|
|
expect(result.advanced).toBe(true);
|
|
expect(result.rolledBack).toBe(false);
|
|
expect(result.reason).toContain('partial');
|
|
|
|
const updated = pipeline.getRollout(rollout.rolloutId)!;
|
|
expect(updated.currentStage).toBe(1);
|
|
expect(updated.stages[0].passed).toBe(true);
|
|
expect(updated.stages[0].completedAt).toBeGreaterThan(0);
|
|
expect(updated.stages[1].startedAt).toBeGreaterThan(0);
|
|
});
|
|
|
|
it('should auto-rollback when divergence exceeds threshold', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
const rollout = pipeline.stage(proposal.proposalId);
|
|
|
|
const result = pipeline.advanceStage(rollout.rolloutId, {
|
|
divergence: 0.99, // way over the 0.2 canary threshold
|
|
});
|
|
|
|
expect(result.advanced).toBe(false);
|
|
expect(result.rolledBack).toBe(true);
|
|
expect(result.reason).toContain('Auto-rollback');
|
|
|
|
const updated = pipeline.getRollout(rollout.rolloutId)!;
|
|
expect(updated.status).toBe('rolled-back');
|
|
expect(updated.stages[0].passed).toBe(false);
|
|
});
|
|
|
|
it('should promote after all stages pass', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
const rollout = pipeline.stage(proposal.proposalId);
|
|
|
|
// Advance through all 3 stages
|
|
pipeline.advanceStage(rollout.rolloutId, { divergence: 0.05 });
|
|
pipeline.advanceStage(rollout.rolloutId, { divergence: 0.10 });
|
|
const final = pipeline.advanceStage(rollout.rolloutId, { divergence: 0.15 });
|
|
|
|
expect(final.advanced).toBe(true);
|
|
expect(final.reason).toContain('promoted');
|
|
|
|
const updated = pipeline.getRollout(rollout.rolloutId)!;
|
|
expect(updated.status).toBe('completed');
|
|
expect(updated.completedAt).toBeGreaterThan(0);
|
|
expect(pipeline.getProposal(proposal.proposalId)?.status).toBe('promoted');
|
|
});
|
|
|
|
it('should record metrics on the current stage', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
const rollout = pipeline.stage(proposal.proposalId);
|
|
|
|
pipeline.advanceStage(rollout.rolloutId, {
|
|
divergence: 0.01,
|
|
accuracy: 0.98,
|
|
latency: 55,
|
|
});
|
|
|
|
const updated = pipeline.getRollout(rollout.rolloutId)!;
|
|
expect(updated.stages[0].metrics).toEqual({
|
|
divergence: 0.01,
|
|
accuracy: 0.98,
|
|
latency: 55,
|
|
});
|
|
});
|
|
|
|
it('should reject advance on non-in-progress rollout', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
const rollout = pipeline.stage(proposal.proposalId);
|
|
|
|
// Rollback first
|
|
pipeline.rollback(rollout.rolloutId, 'manual rollback');
|
|
|
|
const result = pipeline.advanceStage(rollout.rolloutId, { divergence: 0 });
|
|
|
|
expect(result.advanced).toBe(false);
|
|
expect(result.rolledBack).toBe(false);
|
|
expect(result.reason).toContain('rolled-back');
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// rollback
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('rollback', () => {
|
|
it('should mark rollout as rolled-back', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
const rollout = pipeline.stage(proposal.proposalId);
|
|
|
|
pipeline.rollback(rollout.rolloutId, 'detected regression in production');
|
|
|
|
const updated = pipeline.getRollout(rollout.rolloutId)!;
|
|
expect(updated.status).toBe('rolled-back');
|
|
expect(updated.completedAt).toBeGreaterThan(0);
|
|
});
|
|
|
|
it('should mark the proposal as rolled-back', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
const rollout = pipeline.stage(proposal.proposalId);
|
|
|
|
pipeline.rollback(rollout.rolloutId, 'manual');
|
|
|
|
expect(pipeline.getProposal(proposal.proposalId)?.status).toBe('rolled-back');
|
|
});
|
|
|
|
it('should throw for unknown rollout', () => {
|
|
expect(() => pipeline.rollback('nonexistent', 'reason')).toThrow(
|
|
'Rollout not found',
|
|
);
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// promote
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('promote', () => {
|
|
it('should mark rollout as completed', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
const rollout = pipeline.stage(proposal.proposalId);
|
|
|
|
pipeline.promote(rollout.rolloutId);
|
|
|
|
const updated = pipeline.getRollout(rollout.rolloutId)!;
|
|
expect(updated.status).toBe('completed');
|
|
expect(updated.completedAt).toBeGreaterThan(0);
|
|
});
|
|
|
|
it('should mark proposal as promoted', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
const rollout = pipeline.stage(proposal.proposalId);
|
|
|
|
pipeline.promote(rollout.rolloutId);
|
|
|
|
expect(pipeline.getProposal(proposal.proposalId)?.status).toBe('promoted');
|
|
});
|
|
|
|
it('should throw for unknown rollout', () => {
|
|
expect(() => pipeline.promote('nonexistent')).toThrow('Rollout not found');
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// getProposals
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('getProposals', () => {
|
|
it('should return all proposals when no filter is given', () => {
|
|
pipeline.propose(makeProposalParams({ title: 'A' }));
|
|
pipeline.propose(makeProposalParams({ title: 'B' }));
|
|
pipeline.propose(makeProposalParams({ title: 'C' }));
|
|
|
|
expect(pipeline.getProposals()).toHaveLength(3);
|
|
});
|
|
|
|
it('should filter proposals by status', () => {
|
|
const p1 = pipeline.propose(makeProposalParams({ title: 'A' }));
|
|
const p2 = pipeline.propose(makeProposalParams({ title: 'B' }));
|
|
|
|
// Stage p1 (changes status to 'staged')
|
|
pipeline.stage(p1.proposalId);
|
|
|
|
const signed = pipeline.getProposals('signed');
|
|
const staged = pipeline.getProposals('staged');
|
|
|
|
expect(signed).toHaveLength(1);
|
|
expect(signed[0].title).toBe('B');
|
|
expect(staged).toHaveLength(1);
|
|
expect(staged[0].title).toBe('A');
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// getHistory
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('getHistory', () => {
|
|
it('should return empty history for fresh pipeline', () => {
|
|
expect(pipeline.getHistory()).toHaveLength(0);
|
|
});
|
|
|
|
it('should return full timeline for a proposal through all stages', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
const simResult = pipeline.simulate(
|
|
proposal.proposalId,
|
|
[{ input: '1' }],
|
|
identicalEvaluator,
|
|
);
|
|
pipeline.compare(proposal.proposalId, simResult);
|
|
const rollout = pipeline.stage(proposal.proposalId);
|
|
pipeline.advanceStage(rollout.rolloutId, { divergence: 0.01 });
|
|
pipeline.advanceStage(rollout.rolloutId, { divergence: 0.01 });
|
|
pipeline.advanceStage(rollout.rolloutId, { divergence: 0.01 });
|
|
|
|
const history = pipeline.getHistory();
|
|
|
|
expect(history).toHaveLength(1);
|
|
expect(history[0].proposal.proposalId).toBe(proposal.proposalId);
|
|
expect(history[0].simulation).toBeDefined();
|
|
expect(history[0].simulation?.proposalId).toBe(proposal.proposalId);
|
|
expect(history[0].rollout).toBeDefined();
|
|
expect(history[0].rollout?.rolloutId).toBe(rollout.rolloutId);
|
|
expect(history[0].outcome).toBe('promoted');
|
|
});
|
|
|
|
it('should include proposals without simulations or rollouts', () => {
|
|
pipeline.propose(makeProposalParams());
|
|
|
|
const history = pipeline.getHistory();
|
|
|
|
expect(history).toHaveLength(1);
|
|
expect(history[0].simulation).toBeUndefined();
|
|
expect(history[0].rollout).toBeUndefined();
|
|
expect(history[0].outcome).toBe('signed');
|
|
});
|
|
|
|
it('should track multiple proposals independently', () => {
|
|
const p1 = pipeline.propose(makeProposalParams({ title: 'Change 1' }));
|
|
const p2 = pipeline.propose(makeProposalParams({ title: 'Change 2' }));
|
|
|
|
// Only simulate p1
|
|
pipeline.simulate(p1.proposalId, [{ input: '1' }], identicalEvaluator);
|
|
|
|
const history = pipeline.getHistory();
|
|
|
|
expect(history).toHaveLength(2);
|
|
|
|
const h1 = history.find(h => h.proposal.proposalId === p1.proposalId)!;
|
|
const h2 = history.find(h => h.proposal.proposalId === p2.proposalId)!;
|
|
|
|
expect(h1.simulation).toBeDefined();
|
|
expect(h2.simulation).toBeUndefined();
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// Factory function
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('createEvolutionPipeline', () => {
|
|
it('should throw when no signingKey is provided', () => {
|
|
expect(() => createEvolutionPipeline()).toThrow('requires an explicit signingKey');
|
|
});
|
|
|
|
it('should create a pipeline with an explicit signing key', () => {
|
|
const pipeline = createEvolutionPipeline({ signingKey: 'test-key' });
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
|
|
expect(proposal.signature).toMatch(/^[a-f0-9]{64}$/);
|
|
expect(proposal.status).toBe('signed');
|
|
});
|
|
|
|
it('should create a pipeline with custom config', () => {
|
|
const customPipeline = createEvolutionPipeline({
|
|
signingKey: 'custom-key',
|
|
maxDivergence: 0.5,
|
|
});
|
|
|
|
const proposal = customPipeline.propose(makeProposalParams());
|
|
expect(proposal.signature).toBeTruthy();
|
|
|
|
// Higher threshold should accept more divergence
|
|
const simResult = customPipeline.simulate(
|
|
proposal.proposalId,
|
|
[{ input: '1' }],
|
|
slightlyDivergentEvaluator,
|
|
);
|
|
|
|
const comparison = customPipeline.compare(proposal.proposalId, simResult);
|
|
expect(comparison.approved).toBe(true);
|
|
});
|
|
});
|
|
|
|
// --------------------------------------------------------------------------
|
|
// End-to-end lifecycle
|
|
// --------------------------------------------------------------------------
|
|
|
|
describe('end-to-end lifecycle', () => {
|
|
it('should handle propose -> simulate -> compare -> stage -> advance -> promote', () => {
|
|
// 1. Propose
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
expect(proposal.status).toBe('signed');
|
|
|
|
// 2. Simulate
|
|
const simResult = pipeline.simulate(
|
|
proposal.proposalId,
|
|
[{ input: 'golden-1' }, { input: 'golden-2' }],
|
|
identicalEvaluator,
|
|
);
|
|
expect(simResult.passed).toBe(true);
|
|
|
|
// 3. Compare
|
|
const comparison = pipeline.compare(proposal.proposalId, simResult);
|
|
expect(comparison.approved).toBe(true);
|
|
expect(pipeline.getProposal(proposal.proposalId)?.status).toBe('compared');
|
|
|
|
// 4. Stage
|
|
const rollout = pipeline.stage(proposal.proposalId);
|
|
expect(rollout.status).toBe('in-progress');
|
|
expect(pipeline.getProposal(proposal.proposalId)?.status).toBe('staged');
|
|
|
|
// 5. Advance through all stages
|
|
const r1 = pipeline.advanceStage(rollout.rolloutId, { divergence: 0.01 });
|
|
expect(r1.advanced).toBe(true);
|
|
|
|
const r2 = pipeline.advanceStage(rollout.rolloutId, { divergence: 0.02 });
|
|
expect(r2.advanced).toBe(true);
|
|
|
|
const r3 = pipeline.advanceStage(rollout.rolloutId, { divergence: 0.03 });
|
|
expect(r3.advanced).toBe(true);
|
|
expect(r3.reason).toContain('promoted');
|
|
|
|
// 6. Verify final state
|
|
expect(pipeline.getProposal(proposal.proposalId)?.status).toBe('promoted');
|
|
expect(pipeline.getRollout(rollout.rolloutId)?.status).toBe('completed');
|
|
});
|
|
|
|
it('should handle propose -> simulate -> compare(reject) flow', () => {
|
|
const proposal = pipeline.propose(makeProposalParams());
|
|
|
|
const simResult: SimulationResult = {
|
|
proposalId: proposal.proposalId,
|
|
baselineTraceHash: 'a'.repeat(64),
|
|
candidateTraceHash: 'b'.repeat(64),
|
|
divergenceScore: 0.95,
|
|
decisionDiffs: [],
|
|
metricsComparison: { baseline: {}, candidate: {} },
|
|
passed: false,
|
|
reason: 'too divergent',
|
|
};
|
|
|
|
const comparison = pipeline.compare(proposal.proposalId, simResult);
|
|
expect(comparison.approved).toBe(false);
|
|
expect(pipeline.getProposal(proposal.proposalId)?.status).toBe('rejected');
|
|
});
|
|
});
|
|
});
|