//! Conformance & reproducibility scorecard (`conformance-v1`, EPIC 12.17). //! //! A self-check any user or CI can run to prove this instance honors its own //! contracts and that its extension surface behaves. It exercises three areas: //! //! * **contracts** — every machine-verified contract version is present. //! * **reproducibility** — the public discovery documents (`/v1/capabilities`, //! `/v1/openapi.json`) are deterministic (same bytes across two builds). //! * **extensions** — every registered compressor / chunker / read-mode in the //! [`extension_registry`](super::extension_registry) satisfies the stable //! invariants the engine relies on (determinism, budget honoring, coverage). //! //! The output is a [`Scorecard`]: a flat list of [`Check`]s plus a pass count. //! It is data, not prose, so it can be rendered (CLI), shared (JSON), or gated //! on (`all_passed()` in `tests/conformance_suite.rs`). use serde::Serialize; use serde_json::{Value, json}; /// A representative corpus the extension invariants run against. Mixes blank /// lines, multibyte UTF-8, and paragraph boundaries to stress edge cases. const CORPUS: &[&str] = &[ "", "single line", "a\n\n\n\nb \n", "para one\n\npara two\n\n\npara three", "mültibyte ä ö ü 漢字 \n\n end", ]; /// One conformance check result. #[derive(Debug, Clone, Serialize)] pub struct Check { pub name: String, pub category: String, pub passed: bool, pub detail: String, } impl Check { fn pass(category: &str, name: impl Into) -> Self { Self { name: name.into(), category: category.to_string(), passed: true, detail: String::new(), } } fn fail(category: &str, name: impl Into, detail: impl Into) -> Self { Self { name: name.into(), category: category.to_string(), passed: false, detail: detail.into(), } } fn from_bool(category: &str, name: impl Into, ok: bool, fail_detail: &str) -> Self { if ok { Self::pass(category, name) } else { Self::fail(category, name, fail_detail) } } } /// The full result of a conformance run. #[derive(Debug, Clone, Serialize)] pub struct Scorecard { pub version: u32, pub checks: Vec, } impl Scorecard { #[must_use] pub fn passed(&self) -> usize { self.checks.iter().filter(|c| c.passed).count() } #[must_use] pub fn total(&self) -> usize { self.checks.len() } #[must_use] pub fn all_passed(&self) -> bool { self.checks.iter().all(|c| c.passed) } #[must_use] pub fn failures(&self) -> Vec<&Check> { self.checks.iter().filter(|c| !c.passed).collect() } #[must_use] pub fn to_json(&self) -> Value { json!({ "version": self.version, "passed": self.passed(), "total": self.total(), "all_passed": self.all_passed(), "checks": self.checks, }) } } /// Run the full conformance suite against this instance. #[must_use] pub fn run() -> Scorecard { let mut checks = Vec::new(); checks.extend(contract_checks()); checks.extend(reproducibility_checks()); checks.extend(extension_checks()); checks.extend(accuracy_checks()); checks.extend(a2a_checks()); Scorecard { version: 1, checks } } // --------------------------------------------------------------------------- // A2A: agent card and JSON-RPC contract conformance (GL#449). // --------------------------------------------------------------------------- /// Fields the A2A spec requires on a published agent card. const A2A_CARD_REQUIRED: &[&str] = &[ "name", "description", "version", "protocolVersion", "capabilities", "skills", "defaultInputModes", "defaultOutputModes", "authentication", ]; fn a2a_checks() -> Vec { let mut checks = Vec::new(); let card = crate::core::a2a::agent_card::build_agent_card("conformance"); let missing: Vec<&&str> = A2A_CARD_REQUIRED .iter() .filter(|f| card.get(**f).is_none()) .collect(); checks.push(Check::from_bool( "a2a", "agent_card_required_fields", missing.is_empty(), &format!("agent card missing fields: {missing:?}"), )); checks.push(Check::from_bool( "a2a", "agent_card_deterministic", card == crate::core::a2a::agent_card::build_agent_card("conformance"), "two agent card builds differ", )); let skills_ok = card .get("skills") .and_then(serde_json::Value::as_array) .is_some_and(|skills| { !skills.is_empty() && skills.iter().all(|s| { s.get("id").is_some() && s.get("name").is_some() && s.get("description").is_some() }) }); checks.push(Check::from_bool( "a2a", "agent_card_skills_complete", skills_ok, "skills missing id/name/description", )); // JSON-RPC error contract: wrong version → -32600, unknown method → -32601. let bad_version = crate::core::a2a::a2a_compat::handle_a2a_jsonrpc( &crate::core::a2a::a2a_compat::JsonRpcRequest { jsonrpc: "1.0".to_string(), id: serde_json::Value::Number(1.into()), method: "tasks/get".to_string(), params: serde_json::Value::Null, }, ); checks.push(Check::from_bool( "a2a", "jsonrpc_rejects_bad_version", bad_version.error.as_ref().is_some_and(|e| e.code == -32600), "jsonrpc 1.0 not rejected with -32600", )); let unknown_method = crate::core::a2a::a2a_compat::handle_a2a_jsonrpc( &crate::core::a2a::a2a_compat::JsonRpcRequest { jsonrpc: "2.0".to_string(), id: serde_json::Value::Number(2.into()), method: "tasks/nonexistent".to_string(), params: serde_json::Value::Null, }, ); checks.push(Check::from_bool( "a2a", "jsonrpc_unknown_method_code", unknown_method .error .as_ref() .is_some_and(|e| e.code == -32601), "unknown method not rejected with -32601", )); checks } // --------------------------------------------------------------------------- // Accuracy: structural invariants of the lossy read modes (GL#441). // // Byte-golden snapshots would break on every intentional format improvement; // these checks instead pin down what each mode must *preserve* (symbols, // deps) and must *drop* (bodies), plus determinism and size bounds — the // properties an agent's correctness actually depends on. // --------------------------------------------------------------------------- /// A stable Rust fixture exercising pub fns, a struct, imports, and a body /// secret that lossy modes must strip. const ACCURACY_FIXTURE: &str = r"use std::collections::HashMap; use std::path::PathBuf; pub struct Inventory { items: HashMap, } pub fn add_item(inv: &mut Inventory, name: &str, qty: u32) { let body_secret_alpha = qty + 1; inv.items.insert(name.to_string(), body_secret_alpha); } pub fn total_count(inv: &Inventory) -> u32 { let body_secret_beta: u32 = inv.items.values().sum(); body_secret_beta } fn internal_rebalance(inv: &mut Inventory) { inv.items.retain(|_, qty| *qty > 0); } "; /// Symbols every lossy structural mode must keep visible. const MUST_KEEP_SYMBOLS: &[&str] = &["add_item", "total_count", "Inventory"]; /// Body-local identifiers `signatures`/`map` must strip. const MUST_DROP_BODIES: &[&str] = &["body_secret_alpha", "body_secret_beta"]; fn render_mode(mode: &str) -> String { render_mode_full(mode).0 } /// Rendered view plus its pre-footer body token count. The compression-size /// invariant measures `.1` (the body), since the reactive recovery footer and the /// savings line are orthogonal affordances appended after compression, not part of /// the compressed content. fn render_mode_full(mode: &str) -> (String, usize) { crate::tools::ctx_read::render::process_mode( ACCURACY_FIXTURE, mode, "", "fixture.rs", "rs", crate::core::tokens::count_tokens(ACCURACY_FIXTURE), crate::tools::CrpMode::Off, "conformance/fixture.rs", None, ) } fn accuracy_checks() -> Vec { let mut checks = Vec::new(); for mode in ["map", "signatures", "aggressive", "entropy"] { checks.push(Check::from_bool( "accuracy", format!("read_mode_deterministic:{mode}"), render_mode(mode) == render_mode(mode), "two renders of the same fixture differ", )); } for mode in ["map", "signatures"] { let out = render_mode(mode); let missing: Vec<&&str> = MUST_KEEP_SYMBOLS .iter() .filter(|s| !out.contains(**s)) .collect(); checks.push(Check::from_bool( "accuracy", format!("read_mode_keeps_symbols:{mode}"), missing.is_empty(), &format!("symbols lost: {missing:?}"), )); let leaked: Vec<&&str> = MUST_DROP_BODIES .iter() .filter(|s| out.contains(**s)) .collect(); checks.push(Check::from_bool( "accuracy", format!("read_mode_strips_bodies:{mode}"), leaked.is_empty(), &format!("body content leaked: {leaked:?}"), )); } let fixture_tokens = crate::core::tokens::count_tokens(ACCURACY_FIXTURE); for mode in ["map", "signatures", "aggressive"] { let sent = render_mode_full(mode).1; checks.push(Check::from_bool( "accuracy", format!("read_mode_compresses:{mode}"), sent < fixture_tokens, &format!("no compression: {sent} >= {fixture_tokens} tokens"), )); } // Target-density mode (GL#444): the body (excluding header/savings lines) // must stay within the token budget, and the render must be deterministic. { let target = 0.4_f64; let result = crate::core::entropy::entropy_compress_to_density(ACCURACY_FIXTURE, target); let actual = result.compressed_tokens as f64 / fixture_tokens.max(1) as f64; checks.push(Check::from_bool( "accuracy", "density_respects_budget:0.4", actual <= target + 0.10, &format!("density {actual:.2} exceeds target {target:.2} (+0.10 tolerance)"), )); checks.push(Check::from_bool( "accuracy", "density_deterministic:0.4", render_mode("density:0.4") == render_mode("density:0.4"), "two density renders of the same fixture differ", )); } checks } fn contract_checks() -> Vec { let present = !crate::core::contracts::versions_kv().is_empty(); vec![Check::from_bool( "contracts", "contract_versions_present", present, "versions_kv() is empty", )] } fn reproducibility_checks() -> Vec { let caps_stable = crate::core::server_capabilities::capabilities_value() == crate::core::server_capabilities::capabilities_value(); let openapi_stable = crate::core::openapi::openapi_value() == crate::core::openapi::openapi_value(); vec![ Check::from_bool( "reproducibility", "capabilities_deterministic", caps_stable, "capabilities document differs across builds", ), Check::from_bool( "reproducibility", "openapi_deterministic", openapi_stable, "openapi document differs across builds", ), ] } fn extension_checks() -> Vec { let mut checks = Vec::new(); let Ok(reg) = crate::core::extension_registry::global().read() else { checks.push(Check::fail( "extensions", "registry_readable", "extension registry lock poisoned", )); return checks; }; for name in reg.compressor_names() { if let Some(c) = reg.compressor(&name) { checks.push(compressor_invariants(&name, c.as_ref())); } } for name in reg.chunker_names() { if let Some(c) = reg.chunker(&name) { checks.push(chunker_invariants(&name, c.as_ref())); } } for name in reg.read_mode_names() { if let Some(m) = reg.read_mode(&name) { checks.push(read_mode_invariants(&name, m.as_ref())); } } checks } fn compressor_invariants(name: &str, c: &dyn crate::core::extension_registry::Compressor) -> Check { for input in CORPUS { // Determinism. if c.compress(input, None) != c.compress(input, None) { return Check::fail( "extensions", format!("compressor:{name}"), "non-deterministic", ); } // Budget is a hard byte ceiling, never split mid-char (valid UTF-8). let budget = 4; let out = c.compress(input, Some(budget)); if out.len() > budget { return Check::fail( "extensions", format!("compressor:{name}"), format!("exceeded byte budget: {} > {budget}", out.len()), ); } } Check::pass("extensions", format!("compressor:{name}")) } fn chunker_invariants(name: &str, c: &dyn crate::core::extension_registry::Chunker) -> Check { // Empty input ⇒ no chunks. if !c.chunk("").is_empty() { return Check::fail( "extensions", format!("chunker:{name}"), "empty input produced chunks", ); } for input in CORPUS.iter().filter(|s| !s.trim().is_empty()) { // Determinism. if c.chunk(input) != c.chunk(input) { return Check::fail("extensions", format!("chunker:{name}"), "non-deterministic"); } let chunks = c.chunk(input); // Non-empty input ⇒ at least one chunk, none empty after trim. if chunks.is_empty() { return Check::fail( "extensions", format!("chunker:{name}"), "non-empty input produced no chunks", ); } if chunks.iter().any(|c| c.trim().is_empty()) { return Check::fail( "extensions", format!("chunker:{name}"), "produced an empty chunk", ); } } Check::pass("extensions", format!("chunker:{name}")) } fn read_mode_invariants(name: &str, m: &dyn crate::core::extension_registry::ReadMode) -> Check { for input in CORPUS { if m.render(input, "x.txt") != m.render(input, "x.txt") { return Check::fail( "extensions", format!("read_mode:{name}"), "non-deterministic", ); } } // The byte-faithful `full` mode must round-trip source verbatim. if name == "full" { let sample = "verbatim\nsource\n漢字"; if m.render(sample, "x.txt") != sample { return Check::fail("extensions", "read_mode:full", "full mode altered source"); } } Check::pass("extensions", format!("read_mode:{name}")) } #[cfg(test)] mod tests { use super::*; #[test] fn builtin_suite_passes() { let card = run(); assert!( card.all_passed(), "conformance failures: {:?}", card.failures() ); assert!(card.total() >= 6, "expected a meaningful number of checks"); } #[test] fn scorecard_json_shape() { let v = run().to_json(); assert_eq!(v["version"], 1); assert!(v["checks"].is_array()); // Shape only: `passed == total` is covered by builtin_suite_passes. // Asserting it here races with tests that register an intentionally // broken compressor in the global extension registry. let passed = v["passed"].as_u64().expect("passed is a number"); let total = v["total"].as_u64().expect("total is a number"); assert!(passed <= total); assert_eq!(v["checks"].as_array().map(|c| c.len() as u64), Some(total)); } #[test] fn detects_a_nondeterministic_compressor() { use std::sync::atomic::{AtomicU64, Ordering}; struct Flaky(AtomicU64); impl crate::core::extension_registry::Compressor for Flaky { #[allow(clippy::unnecessary_literal_bound)] fn name(&self) -> &str { "flaky" } fn compress(&self, _input: &str, _budget: Option) -> String { self.0.fetch_add(1, Ordering::SeqCst).to_string() } } let check = compressor_invariants("flaky", &Flaky(AtomicU64::new(0))); assert!(!check.passed); assert!(check.detail.contains("non-deterministic")); } }