From 78e07b7aa807a6a3377ea520be94823cb1f5c082 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Wed, 1 Apr 2026 09:55:55 +0800 Subject: [PATCH] feat: strengthen portability semantics --- Makefile | 10 +- README.md | 7 +- agents/interface.yaml | 14 ++ evals/packaging_expectations.json | 18 ++- references/client-compatibility.md | 16 ++- references/packaging-contracts.md | 14 ++ references/platform-capability-matrix.md | 19 ++- references/skill-template.md | 14 ++ reports/context_budget.json | 10 +- reports/context_budget.md | 2 +- reports/context_budget_summary.json | 4 +- reports/portability_score.json | 25 ++++ reports/portability_score.md | 21 +++ scripts/cross_packager.py | 125 ++++++++++++++++- scripts/init_skill.py | 14 ++ scripts/render_portability_report.py | 129 ++++++++++++++++++ scripts/validate_skill.py | 33 +++++ scripts/yao.py | 3 + .../agents/interface.yaml | 12 ++ tests/snapshots/claude_adapter.json | 12 +- tests/snapshots/generic_adapter.json | 12 +- tests/snapshots/openai_adapter.json | 12 +- tests/verify_adapter_snapshots.py | 10 ++ tests/verify_portability_report.py | 42 ++++++ tests/verify_yao_cli.py | 1 + 25 files changed, 552 insertions(+), 27 deletions(-) create mode 100644 reports/portability_score.json create mode 100644 reports/portability_score.md create mode 100644 scripts/render_portability_report.py create mode 100644 tests/verify_portability_report.py diff --git a/Makefile b/Makefile index e780d94..15b4342 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PYTHON ?= python3 -.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check description-drift-history iteration-ledger results-panel regression-history context-reports failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check test clean +.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -44,6 +44,12 @@ regression-history: context-reports: $(PYTHON) scripts/render_context_reports.py +portability-report: + $(PYTHON) scripts/render_portability_report.py + +portability-check: + $(PYTHON) tests/verify_portability_report.py + failure-regression-check: $(PYTHON) tests/verify_failure_regressions.py @@ -71,7 +77,7 @@ resource-boundary-check: quality-check: $(PYTHON) tests/verify_quality_checks.py -test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check description-drift-history iteration-ledger regression-history context-reports failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check +test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check clean: rm -rf dist tests/tmp tests/tmp_snapshot diff --git a/README.md b/README.md index 6e0d27f..1e39e2e 100644 --- a/README.md +++ b/README.md @@ -111,14 +111,15 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa - packaging validation: `openai`, `claude`, and `generic` targets pass contract checks +- portability score: `100/100` with neutral activation, execution, trust, and degradation metadata preserved across all exported targets - description optimization suite: root, team frontend review, and governed incident command pass blind and adversarial holdout gates; governed incident command still carries one visible holdout miss, and adversarial calibration plus family drift are now tracked separately - judge-backed blind eval: root, team frontend review, and governed incident command now pass an independent rubric judge on blind holdout prompts - packaging failure fixtures: invalid metadata, invalid YAML, and unsupported targets fail as expected - failure library regressions: anti-pattern families pass automated checks - governance and resource-boundary checks are part of the default test path - root governance maturity score: `90/100`; governed benchmark example: `95/100` -- context budgets: root `886/1000`, complex benchmark `790/1000`, governed benchmark `760/1000` -- quality density: root `146.7`, complex benchmark `164.6`, governed benchmark `171.1` +- context budgets: root `971/1000`, complex benchmark `790/1000`, governed benchmark `760/1000` +- quality density: root `133.9`, complex benchmark `164.6`, governed benchmark `171.1` - regression milestones are tracked in [reports/regression_history.md](reports/regression_history.md) - description drift history is tracked in [reports/description_drift_history.md](reports/description_drift_history.md) - route confusion is tracked in [reports/route_scorecard.md](reports/route_scorecard.md) @@ -126,6 +127,7 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa - promotion decisions are published in [reports/promotion_decisions.md](reports/promotion_decisions.md) - candidate lifecycle states are published in [reports/candidate_registry.md](reports/candidate_registry.md) - context budget summaries are tracked in [reports/context_budget.md](reports/context_budget.md) +- portability status is tracked in [reports/portability_score.md](reports/portability_score.md) ## What It Does @@ -213,6 +215,7 @@ Utility scripts that make the meta-skill operational: - `render_context_reports.py`: generates root and example context-budget reports plus a shared context summary - `render_regression_history.py`: turns milestone snapshots into a readable regression history report - `cross_packager.py`: builds client-specific export artifacts with explicit platform contracts and validation +- `render_portability_report.py`: scores cross-environment portability from neutral metadata, degradation rules, and consumer validation coverage - `init_skill.py`, `lint_skill.py`, `validate_skill.py`, `diff_eval.py`: minimal authoring toolchain ### `evals/` diff --git a/agents/interface.yaml b/agents/interface.yaml index 4b64a48..599591a 100644 --- a/agents/interface.yaml +++ b/agents/interface.yaml @@ -8,3 +8,17 @@ compatibility: - "openai" - "claude" - "generic" + activation: + mode: "manual" + paths: [] + execution: + context: "inline" + shell: "bash" + trust: + source_tier: "local" + remote_inline_execution: "forbid" + remote_metadata_policy: "allow-metadata-only" + degradation: + openai: "metadata-adapter" + claude: "neutral-source-plus-adapter" + generic: "neutral-source" diff --git a/evals/packaging_expectations.json b/evals/packaging_expectations.json index b93ee23..a03a6df 100644 --- a/evals/packaging_expectations.json +++ b/evals/packaging_expectations.json @@ -1,6 +1,22 @@ { "required_targets": ["openai", "claude", "generic"], - "required_fields": ["name", "description", "version", "display_name", "short_description", "default_prompt", "canonical_metadata"], + "required_fields": [ + "name", + "description", + "version", + "display_name", + "short_description", + "default_prompt", + "canonical_metadata", + "canonical_format", + "activation_mode", + "execution_context", + "shell", + "trust_level", + "remote_inline_execution", + "degradation_strategy", + "portability_profile" + ], "openai_required_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"], "claude_required_files": ["targets/claude/adapter.json", "targets/claude/README.md"], "generic_required_files": ["targets/generic/adapter.json"] diff --git a/references/client-compatibility.md b/references/client-compatibility.md index df6a7fe..3b6eb29 100644 --- a/references/client-compatibility.md +++ b/references/client-compatibility.md @@ -4,7 +4,15 @@ This skill package uses a neutral source-of-truth file: - `agents/interface.yaml` -That file is the canonical metadata source for display name, short description, default prompt, and adapter targets. +That file is the canonical metadata source for: + +- display metadata +- adapter targets +- activation mode +- execution context +- shell expectations +- trust boundaries +- degradation rules ## Compatibility Strategy @@ -13,10 +21,12 @@ Use a two-layer model: 1. **Canonical source** - Keep brand-neutral metadata in `agents/interface.yaml`. - Keep behavior in `SKILL.md`, `references/`, and `scripts/`. + - Keep activation, execution, and trust declarations in neutral metadata rather than target-specific files. 2. **Adapter outputs** - Generate client-specific metadata only when exporting or packaging. - Do not keep vendor-specific metadata files in the source tree unless a client strictly requires them. + - Preserve neutral execution and trust semantics inside generated adapters. ## Supported Targets @@ -29,6 +39,8 @@ The current adapter flow is designed for: ## Compatibility Rules - Keep `SKILL.md` as the primary behavior definition. -- Keep client metadata minimal and presentation-focused. +- Keep client metadata minimal but semantically meaningful. - Avoid putting client-specific logic in the main workflow. - Prefer packaging-time conversion over source-tree duplication. +- Prefer one neutral portability profile over multiple divergent vendor trees. +- Declare how remote or untrusted environments should behave before exporting adapters. diff --git a/references/packaging-contracts.md b/references/packaging-contracts.md index 7cc2978..8a9f6b5 100644 --- a/references/packaging-contracts.md +++ b/references/packaging-contracts.md @@ -15,6 +15,9 @@ Each target contract defines: - required output fields - required output files - field mapping from the neutral source metadata +- portable execution metadata +- trust-boundary metadata +- degradation strategy metadata ## Failure Handling @@ -32,3 +35,14 @@ The neutral source remains: - `agents/interface.yaml` Target-specific metadata is generated at packaging time. + +## Portability Model + +The packaging layer now preserves four portable semantics from the neutral source: + +- activation +- execution +- trust +- degradation + +This means portability is not just "can it export a file?" but also "does the exported target preserve the source package's activation and safety assumptions?" diff --git a/references/platform-capability-matrix.md b/references/platform-capability-matrix.md index e95d1e6..f06a259 100644 --- a/references/platform-capability-matrix.md +++ b/references/platform-capability-matrix.md @@ -2,11 +2,11 @@ This matrix describes the current packaging targets and their support level. -| Target | Metadata Adapter | Output Contract | Snapshot Test | Notes | -| --- | --- | --- | --- | --- | -| `openai` | Yes | Yes | Yes | Generates `targets/openai/agents/openai.yaml` | -| `claude` | Yes | Yes | Yes | Generates `targets/claude/README.md` plus adapter metadata | -| `generic` | Yes | Yes | Yes | Uses neutral adapter metadata only | +| Target | Metadata Adapter | Output Contract | Snapshot Test | Portability Semantics | Notes | +| --- | --- | --- | --- | --- | --- | +| `openai` | Yes | Yes | Yes | activation, execution, trust, degradation | Generates `targets/openai/agents/openai.yaml` | +| `claude` | Yes | Yes | Yes | activation, execution, trust, degradation | Generates `targets/claude/README.md` plus adapter metadata | +| `generic` | Yes | Yes | Yes | activation, execution, trust, degradation | Uses neutral adapter metadata only | ## Current Support Model @@ -14,6 +14,15 @@ This matrix describes the current packaging targets and their support level. - `claude`: lightweight compatibility adapter, behavior still relies mainly on neutral source files - `generic`: lowest-friction export for neutral Agent Skills consumers +## Portable Semantics + +Each target now preserves: + +- activation mode and optional path filters +- execution context and shell choice +- trust tier and remote inline-execution policy +- degradation strategy for unsupported client behavior + ## Explicit Non-Goals This project does not yet implement: diff --git a/references/skill-template.md b/references/skill-template.md index de87b67..14ef9d7 100644 --- a/references/skill-template.md +++ b/references/skill-template.md @@ -71,4 +71,18 @@ compatibility: - "openai" - "claude" - "generic" + activation: + mode: "manual" + paths: [] + execution: + context: "inline" + shell: "bash" + trust: + source_tier: "local" + remote_inline_execution: "forbid" + remote_metadata_policy: "allow-metadata-only" + degradation: + openai: "metadata-adapter" + claude: "neutral-source-plus-adapter" + generic: "neutral-source" ``` diff --git a/reports/context_budget.json b/reports/context_budget.json index 5b62f56..384bf28 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,12 +6,12 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 795, - "other_text_tokens": 259384, - "estimated_initial_load_tokens": 886, - "estimated_total_text_tokens": 260179, - "relevant_file_count": 132, + "other_text_tokens": 263995, + "estimated_initial_load_tokens": 971, + "estimated_total_text_tokens": 264790, + "relevant_file_count": 136, "unused_resource_dirs": [], "quality_signal_points": 130, - "quality_density": 146.7 + "quality_density": 133.9 } } diff --git a/reports/context_budget.md b/reports/context_budget.md index afff72d..e7552f8 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | -| root | `.` | `production` | 1000 | 886 | 795 | 146.7 | - | ok | +| root | `.` | `production` | 1000 | 971 | 795 | 133.9 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index 79721e2..020536b 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -6,9 +6,9 @@ "path": ".", "budget_tier": "production", "budget_limit": 1000, - "initial_tokens": 886, + "initial_tokens": 971, "skill_body_tokens": 795, - "quality_density": 146.7, + "quality_density": 133.9, "unused_resource_dirs": [], "ok": true, "warnings": [] diff --git a/reports/portability_score.json b/reports/portability_score.json new file mode 100644 index 0000000..b79bca6 --- /dev/null +++ b/reports/portability_score.json @@ -0,0 +1,25 @@ +{ + "ok": true, + "score": 100, + "band": "world-class", + "summary": { + "adapter_target_count": 3, + "activation_mode": "manual", + "execution_context": "inline", + "shell": "bash", + "trust_level": "local", + "remote_inline_execution": "forbid", + "degradation_coverage": 3, + "snapshot_count": 3 + }, + "breakdown": { + "canonical_neutral_source": 15, + "adapter_target_coverage": 15, + "activation_metadata": 10, + "execution_metadata": 10, + "trust_boundary_metadata": 15, + "degradation_strategy_coverage": 10, + "contract_and_expectation_coverage": 15, + "consumer_validation_and_snapshots": 10 + } +} diff --git a/reports/portability_score.md b/reports/portability_score.md new file mode 100644 index 0000000..4557c9f --- /dev/null +++ b/reports/portability_score.md @@ -0,0 +1,21 @@ +# Portability Score + +- score: `100/100` +- band: `world-class` +- adapter targets: `3` +- activation mode: `manual` +- execution context: `inline` +- shell: `bash` +- trust level: `local` +- remote inline execution: `forbid` + +| Component | Score | +| --- | ---: | +| `canonical_neutral_source` | 15 | +| `adapter_target_coverage` | 15 | +| `activation_metadata` | 10 | +| `execution_metadata` | 10 | +| `trust_boundary_metadata` | 15 | +| `degradation_strategy_coverage` | 10 | +| `contract_and_expectation_coverage` | 15 | +| `consumer_validation_and_snapshots` | 10 | diff --git a/scripts/cross_packager.py b/scripts/cross_packager.py index 363a6bf..5f82aca 100644 --- a/scripts/cross_packager.py +++ b/scripts/cross_packager.py @@ -37,13 +37,35 @@ def require_fields(payload: dict, fields: list[str], label: str) -> None: raise ValueError(f"Missing required {label} fields: {', '.join(missing)}") +def require_target_degradation( + degradation: dict, + targets: list[str], +) -> None: + missing = [target for target in targets if not degradation.get(target)] + if missing: + raise ValueError(f"Missing degradation entries for targets: {', '.join(missing)}") + + def build_manifest(skill_dir: Path, platform: str) -> dict: frontmatter = read_frontmatter(skill_dir / "SKILL.md") interface_doc = read_interface(skill_dir) interface = interface_doc.get("interface", {}) compatibility = interface_doc.get("compatibility", {}) + activation = compatibility.get("activation", {}) + execution = compatibility.get("execution", {}) + trust = compatibility.get("trust", {}) + degradation = compatibility.get("degradation", {}) require_fields(frontmatter, ["name", "description"], "frontmatter") require_fields(interface, ["display_name", "short_description", "default_prompt"], "interface") + require_fields(compatibility, ["canonical_format", "adapter_targets"], "compatibility") + require_fields(activation, ["mode"], "compatibility.activation") + require_fields(execution, ["context", "shell"], "compatibility.execution") + require_fields( + trust, + ["source_tier", "remote_inline_execution", "remote_metadata_policy"], + "compatibility.trust", + ) + require_target_degradation(degradation, compatibility.get("adapter_targets", [])) return { "name": frontmatter.get("name", skill_dir.name), "description": frontmatter.get("description", ""), @@ -54,36 +76,109 @@ def build_manifest(skill_dir: Path, platform: str) -> dict: "short_description": interface.get("short_description", ""), "default_prompt": interface.get("default_prompt", ""), "canonical_metadata": "agents/interface.yaml", + "canonical_format": compatibility.get("canonical_format", "agent-skills"), "adapter_targets": compatibility.get("adapter_targets", []), + "activation_mode": activation.get("mode", "manual"), + "activation_paths": activation.get("paths", []), + "execution_context": execution.get("context", "inline"), + "shell": execution.get("shell", "bash"), + "trust_level": trust.get("source_tier", "local"), + "remote_inline_execution": trust.get("remote_inline_execution", "forbid"), + "remote_metadata_policy": trust.get("remote_metadata_policy", "allow-metadata-only"), + "degradation_strategy": degradation.get(platform, "neutral-source"), + "portability_profile": { + "activation_mode": activation.get("mode", "manual"), + "activation_paths": activation.get("paths", []), + "execution_context": execution.get("context", "inline"), + "shell": execution.get("shell", "bash"), + "source_tier": trust.get("source_tier", "local"), + "remote_inline_execution": trust.get("remote_inline_execution", "forbid"), + "remote_metadata_policy": trust.get("remote_metadata_policy", "allow-metadata-only"), + "degradation_strategy": degradation.get(platform, "neutral-source"), + }, } PLATFORM_CONTRACTS = { "openai": { - "required_fields": ["name", "description", "version", "display_name", "short_description", "default_prompt", "canonical_metadata"], + "required_fields": [ + "name", + "description", + "version", + "display_name", + "short_description", + "default_prompt", + "canonical_metadata", + "canonical_format", + "activation_mode", + "execution_context", + "shell", + "trust_level", + "remote_inline_execution", + "degradation_strategy", + "portability_profile", + ], "required_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"], "field_mapping": { "display_name": "interface.display_name", "short_description": "interface.short_description", "default_prompt": "interface.default_prompt", + "execution_context": "compatibility.execution.context", + "shell": "compatibility.execution.shell", }, }, "claude": { - "required_fields": ["name", "description", "version", "display_name", "short_description", "default_prompt", "canonical_metadata"], + "required_fields": [ + "name", + "description", + "version", + "display_name", + "short_description", + "default_prompt", + "canonical_metadata", + "canonical_format", + "activation_mode", + "execution_context", + "shell", + "trust_level", + "remote_inline_execution", + "degradation_strategy", + "portability_profile", + ], "required_files": ["targets/claude/adapter.json", "targets/claude/README.md"], "field_mapping": { "display_name": "adapter.display_name", "short_description": "adapter.short_description", "default_prompt": "adapter.default_prompt", + "execution_context": "compatibility.execution.context", + "shell": "compatibility.execution.shell", }, }, "generic": { - "required_fields": ["name", "description", "version", "display_name", "short_description", "default_prompt", "canonical_metadata"], + "required_fields": [ + "name", + "description", + "version", + "display_name", + "short_description", + "default_prompt", + "canonical_metadata", + "canonical_format", + "activation_mode", + "execution_context", + "shell", + "trust_level", + "remote_inline_execution", + "degradation_strategy", + "portability_profile", + ], "required_files": ["targets/generic/adapter.json"], "field_mapping": { "display_name": "adapter.display_name", "short_description": "adapter.short_description", "default_prompt": "adapter.default_prompt", + "execution_context": "compatibility.execution.context", + "shell": "compatibility.execution.shell", }, }, } @@ -91,10 +186,23 @@ PLATFORM_CONTRACTS = { def write_yaml_like(path: Path, payload: dict) -> None: interface = payload.get("interface", {}) + compatibility = payload.get("compatibility", {}) lines = ["interface:"] for key in ("display_name", "short_description", "default_prompt"): value = interface.get(key, "") lines.append(f' {key}: "{value}"') + lines.extend( + [ + "compatibility:", + f' canonical_format: "{compatibility.get("canonical_format", "")}"', + f' activation_mode: "{compatibility.get("activation_mode", "")}"', + f' execution_context: "{compatibility.get("execution_context", "")}"', + f' shell: "{compatibility.get("shell", "")}"', + f' trust_level: "{compatibility.get("trust_level", "")}"', + f' remote_inline_execution: "{compatibility.get("remote_inline_execution", "")}"', + f' degradation_strategy: "{compatibility.get("degradation_strategy", "")}"', + ] + ) path.write_text("\n".join(lines) + "\n", encoding="utf-8") @@ -114,7 +222,16 @@ def write_adapter(skill_dir: Path, out_dir: Path, platform: str) -> Path: "display_name": payload["display_name"], "short_description": payload["short_description"], "default_prompt": payload["default_prompt"], - } + }, + "compatibility": { + "canonical_format": payload["canonical_format"], + "activation_mode": payload["activation_mode"], + "execution_context": payload["execution_context"], + "shell": payload["shell"], + "trust_level": payload["trust_level"], + "remote_inline_execution": payload["remote_inline_execution"], + "degradation_strategy": payload["degradation_strategy"], + }, }, ) payload["install_hint"] = f"Use the packaged skill and include targets/openai/agents/openai.yaml when the client expects OpenAI-style interface metadata." diff --git a/scripts/init_skill.py b/scripts/init_skill.py index 32f41dd..dc8fcbb 100644 --- a/scripts/init_skill.py +++ b/scripts/init_skill.py @@ -28,6 +28,20 @@ compatibility: - "openai" - "claude" - "generic" + activation: + mode: "manual" + paths: [] + execution: + context: "inline" + shell: "bash" + trust: + source_tier: "local" + remote_inline_execution: "forbid" + remote_metadata_policy: "allow-metadata-only" + degradation: + openai: "metadata-adapter" + claude: "neutral-source-plus-adapter" + generic: "neutral-source" """ diff --git a/scripts/render_portability_report.py b/scripts/render_portability_report.py new file mode 100644 index 0000000..00246df --- /dev/null +++ b/scripts/render_portability_report.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +import argparse +import json +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parent.parent + + +def load_json(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def load_yaml(path: Path) -> dict: + return yaml.safe_load(path.read_text(encoding="utf-8")) or {} + + +def band_for(score: int) -> str: + if score >= 97: + return "world-class" + if score >= 93: + return "excellent" + if score >= 85: + return "strong" + return "developing" + + +def build_report(root: Path) -> dict: + interface_doc = load_yaml(root / "agents" / "interface.yaml") + compatibility = interface_doc.get("compatibility", {}) + activation = compatibility.get("activation", {}) + execution = compatibility.get("execution", {}) + trust = compatibility.get("trust", {}) + degradation = compatibility.get("degradation", {}) + targets = compatibility.get("adapter_targets", []) + expectations = load_json(root / "evals" / "packaging_expectations.json") + contracts_doc = (root / "references" / "packaging-contracts.md").exists() + matrix_doc = (root / "references" / "platform-capability-matrix.md").exists() + snapshot_files = list((root / "tests" / "snapshots").glob("*_adapter.json")) + + breakdown = { + "canonical_neutral_source": 15 if (root / "SKILL.md").exists() and (root / "agents" / "interface.yaml").exists() else 0, + "adapter_target_coverage": 15 if len(targets) >= 3 else 10 if len(targets) >= 2 else 5 if len(targets) >= 1 else 0, + "activation_metadata": 10 + if activation.get("mode") and activation.get("paths") is not None + else 5 + if activation.get("mode") + else 0, + "execution_metadata": 10 + if execution.get("context") and execution.get("shell") + else 0, + "trust_boundary_metadata": 15 + if trust.get("source_tier") and trust.get("remote_inline_execution") and trust.get("remote_metadata_policy") + else 0, + "degradation_strategy_coverage": 10 if all(degradation.get(target) for target in targets) else 0, + "contract_and_expectation_coverage": 15 + if contracts_doc and expectations.get("required_targets") == targets + else 5 + if contracts_doc + else 0, + } + + snapshot_score = 0 + if matrix_doc: + snapshot_score += 5 + if len(snapshot_files) >= len(targets): + snapshot_score += 5 + breakdown["consumer_validation_and_snapshots"] = snapshot_score + + score = sum(breakdown.values()) + return { + "ok": score >= 95, + "score": score, + "band": band_for(score), + "summary": { + "adapter_target_count": len(targets), + "activation_mode": activation.get("mode"), + "execution_context": execution.get("context"), + "shell": execution.get("shell"), + "trust_level": trust.get("source_tier"), + "remote_inline_execution": trust.get("remote_inline_execution"), + "degradation_coverage": sum(1 for target in targets if degradation.get(target)), + "snapshot_count": len(snapshot_files), + }, + "breakdown": breakdown, + } + + +def render_markdown(report: dict) -> str: + lines = [ + "# Portability Score", + "", + f"- score: `{report['score']}/100`", + f"- band: `{report['band']}`", + f"- adapter targets: `{report['summary']['adapter_target_count']}`", + f"- activation mode: `{report['summary']['activation_mode']}`", + f"- execution context: `{report['summary']['execution_context']}`", + f"- shell: `{report['summary']['shell']}`", + f"- trust level: `{report['summary']['trust_level']}`", + f"- remote inline execution: `{report['summary']['remote_inline_execution']}`", + "", + "| Component | Score |", + "| --- | ---: |", + ] + for name, value in report["breakdown"].items(): + lines.append(f"| `{name}` | {value} |") + return "\n".join(lines) + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render a portability score from neutral metadata, contracts, and snapshots.") + parser.add_argument("--output-json", default="reports/portability_score.json") + parser.add_argument("--output-md", default="reports/portability_score.md") + args = parser.parse_args() + + report = build_report(ROOT) + output_json = ROOT / args.output_json + output_md = ROOT / args.output_md + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + if not report["ok"]: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/validate_skill.py b/scripts/validate_skill.py index 835d89b..2b11a20 100644 --- a/scripts/validate_skill.py +++ b/scripts/validate_skill.py @@ -5,6 +5,11 @@ from pathlib import Path import yaml +VALID_EXECUTION_CONTEXTS = {"inline", "fork"} +VALID_SHELLS = {"bash", "powershell"} +VALID_SOURCE_TIERS = {"local", "managed", "plugin", "remote"} + + def main() -> None: parser = argparse.ArgumentParser(description="Validate a skill package.") parser.add_argument("skill_dir") @@ -36,9 +41,37 @@ def main() -> None: if interface.exists(): data = yaml.safe_load(interface.read_text(encoding="utf-8")) or {} meta = data.get("interface", {}) + compatibility = data.get("compatibility", {}) + activation = compatibility.get("activation", {}) + execution = compatibility.get("execution", {}) + trust = compatibility.get("trust", {}) + degradation = compatibility.get("degradation", {}) for field in ("display_name", "short_description", "default_prompt"): if not meta.get(field): failures.append(f"Missing interface field: {field}") + if not compatibility.get("canonical_format"): + failures.append("Missing compatibility field: canonical_format") + adapter_targets = compatibility.get("adapter_targets") + if not adapter_targets or not isinstance(adapter_targets, list): + failures.append("Missing compatibility field: adapter_targets") + adapter_targets = [] + if not activation.get("mode"): + failures.append("Missing compatibility.activation.mode") + if activation.get("mode") == "path_scoped" and not activation.get("paths"): + failures.append("path_scoped activation requires compatibility.activation.paths") + if execution.get("context") not in VALID_EXECUTION_CONTEXTS: + failures.append("Invalid compatibility.execution.context") + if execution.get("shell") not in VALID_SHELLS: + failures.append("Invalid compatibility.execution.shell") + if trust.get("source_tier") not in VALID_SOURCE_TIERS: + failures.append("Invalid compatibility.trust.source_tier") + if trust.get("remote_inline_execution") not in {"forbid", "allow"}: + failures.append("Invalid compatibility.trust.remote_inline_execution") + if not trust.get("remote_metadata_policy"): + failures.append("Missing compatibility.trust.remote_metadata_policy") + for target in adapter_targets: + if target not in degradation: + failures.append(f"Missing compatibility.degradation entry for target: {target}") if manifest.exists(): data = json.loads(manifest.read_text(encoding="utf-8")) diff --git a/scripts/yao.py b/scripts/yao.py index 0ab7217..ea36ec7 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -193,6 +193,7 @@ def command_report(args: argparse.Namespace) -> int: run_script("render_iteration_ledger.py", []), run_script("render_regression_history.py", []), run_script("render_context_reports.py", []), + run_script("render_portability_report.py", []), ] ) report = { @@ -205,6 +206,7 @@ def command_report(args: argparse.Namespace) -> int: "iteration_ledger": "reports/iteration_ledger.md", "regression_history": "reports/regression_history.md", "context_budget": "reports/context_budget.json", + "portability_score": "reports/portability_score.json", }, } print(json.dumps(report, ensure_ascii=False, indent=2)) @@ -269,6 +271,7 @@ def command_workspace_flow(args: argparse.Namespace) -> int: {"phase": "report-refresh", "result": run_script("render_iteration_ledger.py", [])}, {"phase": "report-refresh", "result": run_script("render_regression_history.py", [])}, {"phase": "report-refresh", "result": run_script("render_context_reports.py", [])}, + {"phase": "report-refresh", "result": run_script("render_portability_report.py", [])}, ] ) diff --git a/tests/fixtures/package_missing_interface_field/agents/interface.yaml b/tests/fixtures/package_missing_interface_field/agents/interface.yaml index f0f06b2..122311d 100644 --- a/tests/fixtures/package_missing_interface_field/agents/interface.yaml +++ b/tests/fixtures/package_missing_interface_field/agents/interface.yaml @@ -5,3 +5,15 @@ compatibility: canonical_format: "agent-skills" adapter_targets: - "openai" + activation: + mode: "manual" + paths: [] + execution: + context: "inline" + shell: "bash" + trust: + source_tier: "local" + remote_inline_execution: "forbid" + remote_metadata_policy: "allow-metadata-only" + degradation: + openai: "metadata-adapter" diff --git a/tests/snapshots/claude_adapter.json b/tests/snapshots/claude_adapter.json index 4ba9f56..dc0757f 100644 --- a/tests/snapshots/claude_adapter.json +++ b/tests/snapshots/claude_adapter.json @@ -1,5 +1,15 @@ { "platform": "claude", "canonical_metadata": "agents/interface.yaml", - "required_generated_file": "targets/claude/README.md" + "required_generated_file": "targets/claude/README.md", + "required_fields": [ + "canonical_format", + "activation_mode", + "execution_context", + "shell", + "trust_level", + "remote_inline_execution", + "degradation_strategy", + "portability_profile" + ] } diff --git a/tests/snapshots/generic_adapter.json b/tests/snapshots/generic_adapter.json index 7727b9b..83e5dae 100644 --- a/tests/snapshots/generic_adapter.json +++ b/tests/snapshots/generic_adapter.json @@ -1,5 +1,15 @@ { "platform": "generic", "canonical_metadata": "agents/interface.yaml", - "required_generated_file": "targets/generic/adapter.json" + "required_generated_file": "targets/generic/adapter.json", + "required_fields": [ + "canonical_format", + "activation_mode", + "execution_context", + "shell", + "trust_level", + "remote_inline_execution", + "degradation_strategy", + "portability_profile" + ] } diff --git a/tests/snapshots/openai_adapter.json b/tests/snapshots/openai_adapter.json index 0a0691e..6bc9499 100644 --- a/tests/snapshots/openai_adapter.json +++ b/tests/snapshots/openai_adapter.json @@ -1,5 +1,15 @@ { "platform": "openai", "canonical_metadata": "agents/interface.yaml", - "required_generated_file": "targets/openai/agents/openai.yaml" + "required_generated_file": "targets/openai/agents/openai.yaml", + "required_fields": [ + "canonical_format", + "activation_mode", + "execution_context", + "shell", + "trust_level", + "remote_inline_execution", + "degradation_strategy", + "portability_profile" + ] } diff --git a/tests/verify_adapter_snapshots.py b/tests/verify_adapter_snapshots.py index c294438..51cb910 100644 --- a/tests/verify_adapter_snapshots.py +++ b/tests/verify_adapter_snapshots.py @@ -4,6 +4,7 @@ import shutil import subprocess import sys from pathlib import Path +import yaml ROOT = Path(__file__).resolve().parent.parent @@ -49,8 +50,17 @@ def main() -> None: failures.append(f"{name}: platform mismatch") if adapter.get("canonical_metadata") != snapshot["canonical_metadata"]: failures.append(f"{name}: canonical metadata mismatch") + for field in snapshot.get("required_fields", []): + if field not in adapter: + failures.append(f"{name}: missing required adapter field {field}") if not (TMP / snapshot["required_generated_file"]).exists(): failures.append(f"{name}: missing generated file {snapshot['required_generated_file']}") + if name == "openai": + meta = yaml.safe_load((TMP / "targets" / "openai" / "agents" / "openai.yaml").read_text(encoding="utf-8")) or {} + compatibility = meta.get("compatibility", {}) + for field in ("canonical_format", "activation_mode", "execution_context", "shell", "trust_level", "remote_inline_execution", "degradation_strategy"): + if not compatibility.get(field): + failures.append(f"{name}: missing portability metadata in generated openai.yaml: {field}") report = {"ok": not failures, "failures": failures} print(json.dumps(report, ensure_ascii=False, indent=2)) diff --git a/tests/verify_portability_report.py b/tests/verify_portability_report.py new file mode 100644 index 0000000..7a73e35 --- /dev/null +++ b/tests/verify_portability_report.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_portability_report.py" + + +def main() -> None: + proc = subprocess.run( + [sys.executable, str(SCRIPT)], + cwd=ROOT, + capture_output=True, + text=True, + ) + if proc.returncode != 0: + print(proc.stdout) + print(proc.stderr) + raise SystemExit(proc.returncode) + + payload = json.loads(proc.stdout) + failures = [] + if payload.get("score", 0) < 95: + failures.append(f"portability score too low: {payload.get('score')}") + if payload.get("summary", {}).get("adapter_target_count", 0) < 3: + failures.append("adapter target coverage too low") + if payload.get("summary", {}).get("degradation_coverage", 0) < 3: + failures.append("degradation coverage too low") + if payload.get("summary", {}).get("snapshot_count", 0) < 3: + failures.append("snapshot coverage too low") + + report = {"ok": not failures, "failures": failures, "payload": payload} + print(json.dumps(report, ensure_ascii=False, indent=2)) + if failures: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_yao_cli.py b/tests/verify_yao_cli.py index 133adcb..3b0c4e9 100644 --- a/tests/verify_yao_cli.py +++ b/tests/verify_yao_cli.py @@ -65,6 +65,7 @@ def main() -> None: report_result = run("report") assert report_result["ok"], report_result assert "iteration_ledger" in report_result["payload"]["artifacts"], report_result + assert "portability_score" in report_result["payload"]["artifacts"], report_result package_dir = tmp_root / "dist" package_result = run("package", ".", "--platform", "generic", "--output-dir", str(package_dir))