feat: strengthen portability semantics
This commit is contained in:
+121
-4
@@ -37,13 +37,35 @@ def require_fields(payload: dict, fields: list[str], label: str) -> None:
|
||||
raise ValueError(f"Missing required {label} fields: {', '.join(missing)}")
|
||||
|
||||
|
||||
def require_target_degradation(
|
||||
degradation: dict,
|
||||
targets: list[str],
|
||||
) -> None:
|
||||
missing = [target for target in targets if not degradation.get(target)]
|
||||
if missing:
|
||||
raise ValueError(f"Missing degradation entries for targets: {', '.join(missing)}")
|
||||
|
||||
|
||||
def build_manifest(skill_dir: Path, platform: str) -> dict:
|
||||
frontmatter = read_frontmatter(skill_dir / "SKILL.md")
|
||||
interface_doc = read_interface(skill_dir)
|
||||
interface = interface_doc.get("interface", {})
|
||||
compatibility = interface_doc.get("compatibility", {})
|
||||
activation = compatibility.get("activation", {})
|
||||
execution = compatibility.get("execution", {})
|
||||
trust = compatibility.get("trust", {})
|
||||
degradation = compatibility.get("degradation", {})
|
||||
require_fields(frontmatter, ["name", "description"], "frontmatter")
|
||||
require_fields(interface, ["display_name", "short_description", "default_prompt"], "interface")
|
||||
require_fields(compatibility, ["canonical_format", "adapter_targets"], "compatibility")
|
||||
require_fields(activation, ["mode"], "compatibility.activation")
|
||||
require_fields(execution, ["context", "shell"], "compatibility.execution")
|
||||
require_fields(
|
||||
trust,
|
||||
["source_tier", "remote_inline_execution", "remote_metadata_policy"],
|
||||
"compatibility.trust",
|
||||
)
|
||||
require_target_degradation(degradation, compatibility.get("adapter_targets", []))
|
||||
return {
|
||||
"name": frontmatter.get("name", skill_dir.name),
|
||||
"description": frontmatter.get("description", ""),
|
||||
@@ -54,36 +76,109 @@ def build_manifest(skill_dir: Path, platform: str) -> dict:
|
||||
"short_description": interface.get("short_description", ""),
|
||||
"default_prompt": interface.get("default_prompt", ""),
|
||||
"canonical_metadata": "agents/interface.yaml",
|
||||
"canonical_format": compatibility.get("canonical_format", "agent-skills"),
|
||||
"adapter_targets": compatibility.get("adapter_targets", []),
|
||||
"activation_mode": activation.get("mode", "manual"),
|
||||
"activation_paths": activation.get("paths", []),
|
||||
"execution_context": execution.get("context", "inline"),
|
||||
"shell": execution.get("shell", "bash"),
|
||||
"trust_level": trust.get("source_tier", "local"),
|
||||
"remote_inline_execution": trust.get("remote_inline_execution", "forbid"),
|
||||
"remote_metadata_policy": trust.get("remote_metadata_policy", "allow-metadata-only"),
|
||||
"degradation_strategy": degradation.get(platform, "neutral-source"),
|
||||
"portability_profile": {
|
||||
"activation_mode": activation.get("mode", "manual"),
|
||||
"activation_paths": activation.get("paths", []),
|
||||
"execution_context": execution.get("context", "inline"),
|
||||
"shell": execution.get("shell", "bash"),
|
||||
"source_tier": trust.get("source_tier", "local"),
|
||||
"remote_inline_execution": trust.get("remote_inline_execution", "forbid"),
|
||||
"remote_metadata_policy": trust.get("remote_metadata_policy", "allow-metadata-only"),
|
||||
"degradation_strategy": degradation.get(platform, "neutral-source"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
PLATFORM_CONTRACTS = {
|
||||
"openai": {
|
||||
"required_fields": ["name", "description", "version", "display_name", "short_description", "default_prompt", "canonical_metadata"],
|
||||
"required_fields": [
|
||||
"name",
|
||||
"description",
|
||||
"version",
|
||||
"display_name",
|
||||
"short_description",
|
||||
"default_prompt",
|
||||
"canonical_metadata",
|
||||
"canonical_format",
|
||||
"activation_mode",
|
||||
"execution_context",
|
||||
"shell",
|
||||
"trust_level",
|
||||
"remote_inline_execution",
|
||||
"degradation_strategy",
|
||||
"portability_profile",
|
||||
],
|
||||
"required_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"],
|
||||
"field_mapping": {
|
||||
"display_name": "interface.display_name",
|
||||
"short_description": "interface.short_description",
|
||||
"default_prompt": "interface.default_prompt",
|
||||
"execution_context": "compatibility.execution.context",
|
||||
"shell": "compatibility.execution.shell",
|
||||
},
|
||||
},
|
||||
"claude": {
|
||||
"required_fields": ["name", "description", "version", "display_name", "short_description", "default_prompt", "canonical_metadata"],
|
||||
"required_fields": [
|
||||
"name",
|
||||
"description",
|
||||
"version",
|
||||
"display_name",
|
||||
"short_description",
|
||||
"default_prompt",
|
||||
"canonical_metadata",
|
||||
"canonical_format",
|
||||
"activation_mode",
|
||||
"execution_context",
|
||||
"shell",
|
||||
"trust_level",
|
||||
"remote_inline_execution",
|
||||
"degradation_strategy",
|
||||
"portability_profile",
|
||||
],
|
||||
"required_files": ["targets/claude/adapter.json", "targets/claude/README.md"],
|
||||
"field_mapping": {
|
||||
"display_name": "adapter.display_name",
|
||||
"short_description": "adapter.short_description",
|
||||
"default_prompt": "adapter.default_prompt",
|
||||
"execution_context": "compatibility.execution.context",
|
||||
"shell": "compatibility.execution.shell",
|
||||
},
|
||||
},
|
||||
"generic": {
|
||||
"required_fields": ["name", "description", "version", "display_name", "short_description", "default_prompt", "canonical_metadata"],
|
||||
"required_fields": [
|
||||
"name",
|
||||
"description",
|
||||
"version",
|
||||
"display_name",
|
||||
"short_description",
|
||||
"default_prompt",
|
||||
"canonical_metadata",
|
||||
"canonical_format",
|
||||
"activation_mode",
|
||||
"execution_context",
|
||||
"shell",
|
||||
"trust_level",
|
||||
"remote_inline_execution",
|
||||
"degradation_strategy",
|
||||
"portability_profile",
|
||||
],
|
||||
"required_files": ["targets/generic/adapter.json"],
|
||||
"field_mapping": {
|
||||
"display_name": "adapter.display_name",
|
||||
"short_description": "adapter.short_description",
|
||||
"default_prompt": "adapter.default_prompt",
|
||||
"execution_context": "compatibility.execution.context",
|
||||
"shell": "compatibility.execution.shell",
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -91,10 +186,23 @@ PLATFORM_CONTRACTS = {
|
||||
|
||||
def write_yaml_like(path: Path, payload: dict) -> None:
|
||||
interface = payload.get("interface", {})
|
||||
compatibility = payload.get("compatibility", {})
|
||||
lines = ["interface:"]
|
||||
for key in ("display_name", "short_description", "default_prompt"):
|
||||
value = interface.get(key, "")
|
||||
lines.append(f' {key}: "{value}"')
|
||||
lines.extend(
|
||||
[
|
||||
"compatibility:",
|
||||
f' canonical_format: "{compatibility.get("canonical_format", "")}"',
|
||||
f' activation_mode: "{compatibility.get("activation_mode", "")}"',
|
||||
f' execution_context: "{compatibility.get("execution_context", "")}"',
|
||||
f' shell: "{compatibility.get("shell", "")}"',
|
||||
f' trust_level: "{compatibility.get("trust_level", "")}"',
|
||||
f' remote_inline_execution: "{compatibility.get("remote_inline_execution", "")}"',
|
||||
f' degradation_strategy: "{compatibility.get("degradation_strategy", "")}"',
|
||||
]
|
||||
)
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
@@ -114,7 +222,16 @@ def write_adapter(skill_dir: Path, out_dir: Path, platform: str) -> Path:
|
||||
"display_name": payload["display_name"],
|
||||
"short_description": payload["short_description"],
|
||||
"default_prompt": payload["default_prompt"],
|
||||
}
|
||||
},
|
||||
"compatibility": {
|
||||
"canonical_format": payload["canonical_format"],
|
||||
"activation_mode": payload["activation_mode"],
|
||||
"execution_context": payload["execution_context"],
|
||||
"shell": payload["shell"],
|
||||
"trust_level": payload["trust_level"],
|
||||
"remote_inline_execution": payload["remote_inline_execution"],
|
||||
"degradation_strategy": payload["degradation_strategy"],
|
||||
},
|
||||
},
|
||||
)
|
||||
payload["install_hint"] = f"Use the packaged skill and include targets/openai/agents/openai.yaml when the client expects OpenAI-style interface metadata."
|
||||
|
||||
@@ -28,6 +28,20 @@ compatibility:
|
||||
- "openai"
|
||||
- "claude"
|
||||
- "generic"
|
||||
activation:
|
||||
mode: "manual"
|
||||
paths: []
|
||||
execution:
|
||||
context: "inline"
|
||||
shell: "bash"
|
||||
trust:
|
||||
source_tier: "local"
|
||||
remote_inline_execution: "forbid"
|
||||
remote_metadata_policy: "allow-metadata-only"
|
||||
degradation:
|
||||
openai: "metadata-adapter"
|
||||
claude: "neutral-source-plus-adapter"
|
||||
generic: "neutral-source"
|
||||
"""
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def load_yaml(path: Path) -> dict:
|
||||
return yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
||||
|
||||
|
||||
def band_for(score: int) -> str:
|
||||
if score >= 97:
|
||||
return "world-class"
|
||||
if score >= 93:
|
||||
return "excellent"
|
||||
if score >= 85:
|
||||
return "strong"
|
||||
return "developing"
|
||||
|
||||
|
||||
def build_report(root: Path) -> dict:
|
||||
interface_doc = load_yaml(root / "agents" / "interface.yaml")
|
||||
compatibility = interface_doc.get("compatibility", {})
|
||||
activation = compatibility.get("activation", {})
|
||||
execution = compatibility.get("execution", {})
|
||||
trust = compatibility.get("trust", {})
|
||||
degradation = compatibility.get("degradation", {})
|
||||
targets = compatibility.get("adapter_targets", [])
|
||||
expectations = load_json(root / "evals" / "packaging_expectations.json")
|
||||
contracts_doc = (root / "references" / "packaging-contracts.md").exists()
|
||||
matrix_doc = (root / "references" / "platform-capability-matrix.md").exists()
|
||||
snapshot_files = list((root / "tests" / "snapshots").glob("*_adapter.json"))
|
||||
|
||||
breakdown = {
|
||||
"canonical_neutral_source": 15 if (root / "SKILL.md").exists() and (root / "agents" / "interface.yaml").exists() else 0,
|
||||
"adapter_target_coverage": 15 if len(targets) >= 3 else 10 if len(targets) >= 2 else 5 if len(targets) >= 1 else 0,
|
||||
"activation_metadata": 10
|
||||
if activation.get("mode") and activation.get("paths") is not None
|
||||
else 5
|
||||
if activation.get("mode")
|
||||
else 0,
|
||||
"execution_metadata": 10
|
||||
if execution.get("context") and execution.get("shell")
|
||||
else 0,
|
||||
"trust_boundary_metadata": 15
|
||||
if trust.get("source_tier") and trust.get("remote_inline_execution") and trust.get("remote_metadata_policy")
|
||||
else 0,
|
||||
"degradation_strategy_coverage": 10 if all(degradation.get(target) for target in targets) else 0,
|
||||
"contract_and_expectation_coverage": 15
|
||||
if contracts_doc and expectations.get("required_targets") == targets
|
||||
else 5
|
||||
if contracts_doc
|
||||
else 0,
|
||||
}
|
||||
|
||||
snapshot_score = 0
|
||||
if matrix_doc:
|
||||
snapshot_score += 5
|
||||
if len(snapshot_files) >= len(targets):
|
||||
snapshot_score += 5
|
||||
breakdown["consumer_validation_and_snapshots"] = snapshot_score
|
||||
|
||||
score = sum(breakdown.values())
|
||||
return {
|
||||
"ok": score >= 95,
|
||||
"score": score,
|
||||
"band": band_for(score),
|
||||
"summary": {
|
||||
"adapter_target_count": len(targets),
|
||||
"activation_mode": activation.get("mode"),
|
||||
"execution_context": execution.get("context"),
|
||||
"shell": execution.get("shell"),
|
||||
"trust_level": trust.get("source_tier"),
|
||||
"remote_inline_execution": trust.get("remote_inline_execution"),
|
||||
"degradation_coverage": sum(1 for target in targets if degradation.get(target)),
|
||||
"snapshot_count": len(snapshot_files),
|
||||
},
|
||||
"breakdown": breakdown,
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(report: dict) -> str:
|
||||
lines = [
|
||||
"# Portability Score",
|
||||
"",
|
||||
f"- score: `{report['score']}/100`",
|
||||
f"- band: `{report['band']}`",
|
||||
f"- adapter targets: `{report['summary']['adapter_target_count']}`",
|
||||
f"- activation mode: `{report['summary']['activation_mode']}`",
|
||||
f"- execution context: `{report['summary']['execution_context']}`",
|
||||
f"- shell: `{report['summary']['shell']}`",
|
||||
f"- trust level: `{report['summary']['trust_level']}`",
|
||||
f"- remote inline execution: `{report['summary']['remote_inline_execution']}`",
|
||||
"",
|
||||
"| Component | Score |",
|
||||
"| --- | ---: |",
|
||||
]
|
||||
for name, value in report["breakdown"].items():
|
||||
lines.append(f"| `{name}` | {value} |")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Render a portability score from neutral metadata, contracts, and snapshots.")
|
||||
parser.add_argument("--output-json", default="reports/portability_score.json")
|
||||
parser.add_argument("--output-md", default="reports/portability_score.md")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = build_report(ROOT)
|
||||
output_json = ROOT / args.output_json
|
||||
output_md = ROOT / args.output_md
|
||||
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_markdown(report), encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if not report["ok"]:
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -5,6 +5,11 @@ from pathlib import Path
|
||||
import yaml
|
||||
|
||||
|
||||
VALID_EXECUTION_CONTEXTS = {"inline", "fork"}
|
||||
VALID_SHELLS = {"bash", "powershell"}
|
||||
VALID_SOURCE_TIERS = {"local", "managed", "plugin", "remote"}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Validate a skill package.")
|
||||
parser.add_argument("skill_dir")
|
||||
@@ -36,9 +41,37 @@ def main() -> None:
|
||||
if interface.exists():
|
||||
data = yaml.safe_load(interface.read_text(encoding="utf-8")) or {}
|
||||
meta = data.get("interface", {})
|
||||
compatibility = data.get("compatibility", {})
|
||||
activation = compatibility.get("activation", {})
|
||||
execution = compatibility.get("execution", {})
|
||||
trust = compatibility.get("trust", {})
|
||||
degradation = compatibility.get("degradation", {})
|
||||
for field in ("display_name", "short_description", "default_prompt"):
|
||||
if not meta.get(field):
|
||||
failures.append(f"Missing interface field: {field}")
|
||||
if not compatibility.get("canonical_format"):
|
||||
failures.append("Missing compatibility field: canonical_format")
|
||||
adapter_targets = compatibility.get("adapter_targets")
|
||||
if not adapter_targets or not isinstance(adapter_targets, list):
|
||||
failures.append("Missing compatibility field: adapter_targets")
|
||||
adapter_targets = []
|
||||
if not activation.get("mode"):
|
||||
failures.append("Missing compatibility.activation.mode")
|
||||
if activation.get("mode") == "path_scoped" and not activation.get("paths"):
|
||||
failures.append("path_scoped activation requires compatibility.activation.paths")
|
||||
if execution.get("context") not in VALID_EXECUTION_CONTEXTS:
|
||||
failures.append("Invalid compatibility.execution.context")
|
||||
if execution.get("shell") not in VALID_SHELLS:
|
||||
failures.append("Invalid compatibility.execution.shell")
|
||||
if trust.get("source_tier") not in VALID_SOURCE_TIERS:
|
||||
failures.append("Invalid compatibility.trust.source_tier")
|
||||
if trust.get("remote_inline_execution") not in {"forbid", "allow"}:
|
||||
failures.append("Invalid compatibility.trust.remote_inline_execution")
|
||||
if not trust.get("remote_metadata_policy"):
|
||||
failures.append("Missing compatibility.trust.remote_metadata_policy")
|
||||
for target in adapter_targets:
|
||||
if target not in degradation:
|
||||
failures.append(f"Missing compatibility.degradation entry for target: {target}")
|
||||
|
||||
if manifest.exists():
|
||||
data = json.loads(manifest.read_text(encoding="utf-8"))
|
||||
|
||||
@@ -193,6 +193,7 @@ def command_report(args: argparse.Namespace) -> int:
|
||||
run_script("render_iteration_ledger.py", []),
|
||||
run_script("render_regression_history.py", []),
|
||||
run_script("render_context_reports.py", []),
|
||||
run_script("render_portability_report.py", []),
|
||||
]
|
||||
)
|
||||
report = {
|
||||
@@ -205,6 +206,7 @@ def command_report(args: argparse.Namespace) -> int:
|
||||
"iteration_ledger": "reports/iteration_ledger.md",
|
||||
"regression_history": "reports/regression_history.md",
|
||||
"context_budget": "reports/context_budget.json",
|
||||
"portability_score": "reports/portability_score.json",
|
||||
},
|
||||
}
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
@@ -269,6 +271,7 @@ def command_workspace_flow(args: argparse.Namespace) -> int:
|
||||
{"phase": "report-refresh", "result": run_script("render_iteration_ledger.py", [])},
|
||||
{"phase": "report-refresh", "result": run_script("render_regression_history.py", [])},
|
||||
{"phase": "report-refresh", "result": run_script("render_context_reports.py", [])},
|
||||
{"phase": "report-refresh", "result": run_script("render_portability_report.py", [])},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user