#!/usr/bin/env python3 import json import shutil import subprocess import sys from pathlib import Path ROOT = Path(__file__).resolve().parent.parent SCRIPT = ROOT / "scripts" / "trust_check.py" INTERFACE = """interface: display_name: "Trust Demo" short_description: "Trust check demo" default_prompt: "Use trust demo." compatibility: canonical_format: "agent-skills" adapter_targets: - "generic" activation: mode: "manual" paths: [] execution: context: "inline" shell: "bash" trust: source_tier: "local" remote_inline_execution: "forbid" remote_metadata_policy: "allow-metadata-only" degradation: generic: "neutral-source" """ def main() -> None: tmp_root = ROOT / "tests" / "tmp_trust" shutil.rmtree(tmp_root, ignore_errors=True) tmp_root.mkdir(parents=True, exist_ok=True) output_json = tmp_root / "security_trust_report.json" output_md = tmp_root / "security_trust_report.md" proc = subprocess.run( [ sys.executable, str(SCRIPT), str(ROOT), "--output-json", str(output_json), "--output-md", str(output_md), ], cwd=ROOT, capture_output=True, text=True, check=True, ) payload = json.loads(proc.stdout) assert payload["ok"], payload assert payload["summary"]["secret_findings"] == 0, payload assert payload["summary"]["package_hash_scope"] == "source-contract-without-generated-reports", payload assert payload["summary"]["package_hash_file_count"] == payload["summary"]["scanned_files"], payload assert payload["summary"]["package_sha256"], payload assert payload["summary"]["internal_module_count"] >= 3, payload assert payload["summary"]["network_script_count"] == 3, payload assert payload["summary"]["network_policy_covered_count"] == payload["summary"]["network_script_count"], payload assert payload["summary"]["network_policy_missing_count"] == 0, payload assert payload["summary"]["permission_required_count"] >= 3, payload assert payload["summary"]["permission_approved_count"] == payload["summary"]["permission_required_count"], payload assert payload["summary"]["permission_missing_count"] == 0, payload assert payload["summary"]["permission_invalid_count"] == 0, payload assert payload["summary"]["permission_expired_count"] == 0, payload assert payload["summary"]["help_smoke_checked_count"] > 0, payload assert payload["summary"]["help_smoke_failed_count"] == 0, payload assert payload["help_smoke"]["enabled"], payload["help_smoke"] assert payload["help_smoke"]["checked_count"] == payload["help_smoke"]["passed_count"], payload["help_smoke"] assert payload["help_smoke"]["failed_scripts"] == [], payload["help_smoke"] assert payload["help_smoke"]["skipped_count"] == payload["summary"]["internal_module_count"], payload["help_smoke"] assert payload["network_policy"]["present"], payload["network_policy"] assert payload["network_policy"]["missing_scripts"] == [], payload["network_policy"] assert payload["network_policy"]["mismatches"] == [], payload["network_policy"] assert payload["permission_governance"]["present"], payload["permission_governance"] assert {"network", "file_write", "subprocess"} <= set(payload["permission_governance"]["approved_capabilities"]), payload["permission_governance"] assert payload["permission_governance"]["missing_capabilities"] == [], payload["permission_governance"] assert payload["permission_governance"]["invalid_capabilities"] == [], payload["permission_governance"] assert payload["permission_governance"]["expired_capabilities"] == [], payload["permission_governance"] assert "scripts/check_update.py" in payload["network_policy"]["covered_scripts"], payload["network_policy"] assert "scripts/github_benchmark_scan.py" in payload["network_policy"]["covered_scripts"], payload["network_policy"] assert "scripts/provider_output_eval_runner.py" in payload["network_policy"]["covered_scripts"], payload["network_policy"] script_map = {item["path"]: item for item in payload["scripts"]} for internal_module in [ "scripts/review_studio_formatting.py", "scripts/review_studio_gates.py", "scripts/review_studio_layout.py", "scripts/skill_report_charts.py", "scripts/skill_report_i18n.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", "scripts/skill_report_world_class.py", "scripts/world_class_evidence_contract.py", "scripts/world_class_source_checks.py", "scripts/yao_cli_config.py", "scripts/yao_cli_parser.py", "scripts/yao_cli_telemetry.py", ]: assert script_map[internal_module]["interface"] == "internal-module", script_map[internal_module] assert script_map[internal_module]["interface_declared"], script_map[internal_module] warning_text = "\n".join(payload["warnings"]) assert "review_studio_formatting.py" not in warning_text, payload["warnings"] assert "review_studio_gates.py" not in warning_text, payload["warnings"] assert "review_studio_layout.py" not in warning_text, payload["warnings"] assert "skill_report_charts.py" not in warning_text, payload["warnings"] assert "skill_report_i18n.py" not in warning_text, payload["warnings"] assert "skill_report_layout.py" not in warning_text, payload["warnings"] assert "skill_report_metrics.py" not in warning_text, payload["warnings"] assert "skill_report_model.py" not in warning_text, payload["warnings"] assert "skill_report_world_class.py" not in warning_text, payload["warnings"] assert "world_class_evidence_contract.py" not in warning_text, payload["warnings"] assert "world_class_source_checks.py" not in warning_text, payload["warnings"] assert "yao_cli_config.py" not in warning_text, payload["warnings"] assert "yao_cli_parser.py" not in warning_text, payload["warnings"] assert "yao_cli_telemetry.py" not in warning_text, payload["warnings"] assert "render_context_reports.py" not in warning_text, payload["warnings"] assert "render_social_preview.py" not in warning_text, payload["warnings"] assert "Network-capable scripts require bounded host policy" not in warning_text, payload["warnings"] assert "CLI help smoke failed" not in warning_text, payload["warnings"] assert "Permission approvals" not in warning_text, payload["warnings"] assert output_json.exists(), output_json assert output_md.exists(), output_md assert "Security Trust Report" in output_md.read_text(encoding="utf-8") assert "Package hash scope" in output_md.read_text(encoding="utf-8") secret_skill = tmp_root / "secret-skill" (secret_skill / "agents").mkdir(parents=True, exist_ok=True) (secret_skill / "scripts").mkdir(parents=True, exist_ok=True) (secret_skill / "SKILL.md").write_text( "---\nname: secret-skill\ndescription: Secret demo.\n---\n\n# Secret\n", encoding="utf-8", ) (secret_skill / "agents" / "interface.yaml").write_text(INTERFACE, encoding="utf-8") (secret_skill / "scripts" / "leaky.py").write_text( "TOKEN = 'ghp_1234567890abcdefghijklmnopqrstuv'\n", encoding="utf-8", ) secret_proc = subprocess.run( [sys.executable, str(SCRIPT), str(secret_skill)], cwd=ROOT, capture_output=True, text=True, ) assert secret_proc.returncode == 2, secret_proc.stdout secret_payload = json.loads(secret_proc.stdout) assert not secret_payload["ok"], secret_payload assert secret_payload["summary"]["secret_findings"] == 1, secret_payload assert secret_payload["secrets"][0]["type"] == "github_token", secret_payload network_skill = tmp_root / "network-skill" (network_skill / "agents").mkdir(parents=True, exist_ok=True) (network_skill / "scripts").mkdir(parents=True, exist_ok=True) (network_skill / "SKILL.md").write_text( "---\nname: network-skill\ndescription: Network demo.\n---\n\n# Network\n", encoding="utf-8", ) (network_skill / "agents" / "interface.yaml").write_text(INTERFACE, encoding="utf-8") (network_skill / "scripts" / "probe.py").write_text( "from urllib.request import urlopen\n\nurlopen('https://example.com/status')\n", encoding="utf-8", ) missing_policy_proc = subprocess.run( [sys.executable, str(SCRIPT), str(network_skill)], cwd=ROOT, capture_output=True, text=True, check=True, ) missing_policy_payload = json.loads(missing_policy_proc.stdout) assert missing_policy_payload["summary"]["network_script_count"] == 1, missing_policy_payload assert missing_policy_payload["summary"]["network_policy_missing_count"] == 1, missing_policy_payload assert any("Network-capable scripts require bounded host policy" in item for item in missing_policy_payload["warnings"]), missing_policy_payload (network_skill / "security").mkdir(parents=True, exist_ok=True) (network_skill / "security" / "network_policy.json").write_text( json.dumps( { "schema_version": "1.0", "scripts": { "scripts/probe.py": { "purpose": "Test policy coverage.", "allowed_hosts": ["example.com"], "allowed_path_prefixes": ["/"], "requires_https": True, "custom_url_policy": "not supported", } }, }, ensure_ascii=False, indent=2, ) + "\n", encoding="utf-8", ) covered_policy_proc = subprocess.run( [sys.executable, str(SCRIPT), str(network_skill)], cwd=ROOT, capture_output=True, text=True, check=True, ) covered_policy_payload = json.loads(covered_policy_proc.stdout) assert covered_policy_payload["summary"]["network_policy_covered_count"] == 1, covered_policy_payload assert covered_policy_payload["summary"]["network_policy_missing_count"] == 0, covered_policy_payload assert not any("Network-capable scripts require bounded host policy" in item for item in covered_policy_payload["warnings"]), covered_policy_payload assert covered_policy_payload["summary"]["permission_missing_count"] == 1, covered_policy_payload assert covered_policy_payload["permission_governance"]["missing_capabilities"] == ["network"], covered_policy_payload assert any("Permission approvals missing: network" in item for item in covered_policy_payload["warnings"]), covered_policy_payload (network_skill / "security" / "permission_policy.json").write_text( json.dumps( { "schema_version": "1.0", "capabilities": { "network": { "decision": "approved", "reviewer": "Yao Team", "scope": "Test network probe.", "reason": "Network access is bounded to example.com for this isolated test fixture.", "expires_at": "2026-09-30", "target_enforcement": { "openai": "metadata-only", "claude": "adapter metadata", "generic": "adapter metadata", "vscode": "workspace trust note", }, } }, }, ensure_ascii=False, indent=2, ) + "\n", encoding="utf-8", ) approved_policy_proc = subprocess.run( [sys.executable, str(SCRIPT), str(network_skill)], cwd=ROOT, capture_output=True, text=True, check=True, ) approved_policy_payload = json.loads(approved_policy_proc.stdout) assert approved_policy_payload["summary"]["permission_approved_count"] == 1, approved_policy_payload assert approved_policy_payload["summary"]["permission_missing_count"] == 0, approved_policy_payload assert not any("Permission approvals missing" in item for item in approved_policy_payload["warnings"]), approved_policy_payload help_smoke_skill = tmp_root / "help-smoke-skill" (help_smoke_skill / "agents").mkdir(parents=True, exist_ok=True) (help_smoke_skill / "scripts").mkdir(parents=True, exist_ok=True) (help_smoke_skill / "SKILL.md").write_text( "---\nname: help-smoke-skill\ndescription: Help smoke demo.\n---\n\n# Help Smoke\n", encoding="utf-8", ) (help_smoke_skill / "agents" / "interface.yaml").write_text(INTERFACE, encoding="utf-8") (help_smoke_skill / "scripts" / "bad_help.py").write_text( "import argparse\n\nraise RuntimeError('help import failed')\n", encoding="utf-8", ) help_smoke_proc = subprocess.run( [sys.executable, str(SCRIPT), str(help_smoke_skill)], cwd=ROOT, capture_output=True, text=True, check=True, ) help_smoke_payload = json.loads(help_smoke_proc.stdout) assert help_smoke_payload["summary"]["help_smoke_checked_count"] == 1, help_smoke_payload assert help_smoke_payload["summary"]["help_smoke_failed_count"] == 1, help_smoke_payload assert help_smoke_payload["help_smoke"]["failed_scripts"] == ["scripts/bad_help.py"], help_smoke_payload assert any("CLI help smoke failed: scripts/bad_help.py" in item for item in help_smoke_payload["warnings"]), help_smoke_payload print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) if __name__ == "__main__": main()