From f37e78def3464bc818429c47e1ecc9e033c7356c Mon Sep 17 00:00:00 2001 From: yaojingang Date: Sat, 20 Jun 2026 20:34:30 +0800 Subject: [PATCH] Add operator UX diagnostics --- AGENTS.md | 3 + Makefile | 9 +- README.md | 16 +- docs/README.zh-CN.md | 16 +- scripts/ci_test.py | 1 + scripts/yao.py | 5 + scripts/yao_cli_operator_commands.py | 424 +++++++++++++++++++++++++++ scripts/yao_cli_parser.py | 2 + scripts/yao_cli_parser_operator.py | 58 ++++ tests/verify_operator_ux.py | 198 +++++++++++++ tests/verify_trust_check.py | 4 + tests/yao_cli_helpers.py | 5 + 12 files changed, 736 insertions(+), 5 deletions(-) create mode 100644 scripts/yao_cli_operator_commands.py create mode 100644 scripts/yao_cli_parser_operator.py create mode 100644 tests/verify_operator_ux.py diff --git a/AGENTS.md b/AGENTS.md index b7511deb..50eeb9d6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,6 +23,7 @@ Use `make ci-test` as the default full verification command before calling a cha Common focused checks: - CLI changes: `python3 tests/verify_yao_cli.py` +- Operator UX changes: `python3 tests/verify_operator_ux.py` - Skill overview report changes: `python3 tests/verify_skill_overview.py` - Review Studio changes: `python3 tests/verify_review_studio.py` - Trust or script inventory changes: `python3 tests/verify_trust_check.py` @@ -119,6 +120,8 @@ Clean test-only scratch directories after verification with `find tests -maxdept - `scripts/skill_report_model.py`, `scripts/skill_report_metrics.py`, `scripts/skill_report_charts.py`: skill overview data model, scoring, and inline SVG chart generation. - `scripts/yao_cli_config.py`: CLI target maps, archetype heuristics, diagnosis copy, and side-effect-free shaping helpers. - `scripts/yao_cli_parser.py`: CLI argparse command surface, flags, choices, and command handler binding. +- `scripts/yao_cli_operator_commands.py`: read-only operator diagnostics for active install status, localized homepage sync, and PR review reports. Keep install checks non-mutating, docs sync explicit, and PR review free of merge/write actions. +- `scripts/yao_cli_parser_operator.py`: argparse surface for operator UX commands. Keep command names stable and route all behavior into `yao_cli_operator_commands.py`. - `scripts/yao_cli_telemetry.py`: opt-in metadata-only CLI run telemetry. Keep it free of prompt, argument, output, transcript, note, or message capture. - `scripts/import_telemetry_events.py`: external telemetry importer. Validate the whole input before appending events, and keep raw prompt/output/transcript/message/note fields blocked. - `scripts/emit_telemetry_event.py`: external client telemetry emitter. It may append one normalized metadata event to a local spool, but must never accept or write raw prompt, output, transcript, message, note, argument, or private content. diff --git a/Makefile b/Makefile index a65d8589..9fae8216 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ REQUIRED_PYTHON_VERSION ?= 3.11 LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill -.PHONY: python-version-check eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check yao-cli-world-class-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-kit-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-kit-check world-class-preflight-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check skillops-opportunity-check daily-skillops-check weekly-curator-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean +.PHONY: python-version-check eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check yao-cli-world-class-check operator-ux-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-kit-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-kit-check world-class-preflight-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check skillops-opportunity-check daily-skillops-check weekly-curator-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean python-version-check: @$(PYTHON) -c 'import sys; required=tuple(map(int, "$(REQUIRED_PYTHON_VERSION)".split("."))); actual=sys.version_info[:2]; raise SystemExit(0 if actual >= required else "Python {}.{}+ is required; {}.{} is active. Set PYTHON=/path/to/python{} or create a Python {} virtualenv.".format(*required, *actual, "$(REQUIRED_PYTHON_VERSION)", "$(REQUIRED_PYTHON_VERSION)"))' @@ -44,6 +44,9 @@ yao-cli-check: yao-cli-world-class-check: $(PYTHON) tests/verify_yao_cli_world_class.py +operator-ux-check: + $(PYTHON) tests/verify_operator_ux.py + skill-overview-check: $(PYTHON) tests/verify_skill_overview.py @@ -270,11 +273,11 @@ sync-local-install: package-check sync-active-install: package-check $(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)" -test: python-version-check eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-kit-check world-class-preflight-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check skillops-opportunity-check daily-skillops-check weekly-curator-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check +test: python-version-check eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check operator-ux-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-kit-check world-class-preflight-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check skillops-opportunity-check daily-skillops-check weekly-curator-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check ci-test: python-version-check $(PYTHON) scripts/ci_test.py clean: - rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_python_compat tests/tmp_architecture_maintainability tests/tmp_skill_overview tests/tmp_skill_interpretation tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_skill_os2_audit tests/tmp_skill_os2_coverage tests/tmp_world_class_evidence tests/tmp_world_class_evidence_ledger tests/tmp_world_class_evidence_intake tests/tmp_world_class_submission_review tests/tmp_world_class_operator_runbook tests/tmp_world_class_claim_guard tests/tmp_benchmark_reproducibility tests/tmp_evidence_consistency tests/tmp_feedback tests/tmp_daily_skillops tests/tmp_weekly_curator tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_telemetry_emit tests/tmp_telemetry_hooks tests/tmp_telemetry_native_host tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md + rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_operator_ux tests/tmp_python_compat tests/tmp_architecture_maintainability tests/tmp_skill_overview tests/tmp_skill_interpretation tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_skill_os2_audit tests/tmp_skill_os2_coverage tests/tmp_world_class_evidence tests/tmp_world_class_evidence_ledger tests/tmp_world_class_evidence_intake tests/tmp_world_class_submission_review tests/tmp_world_class_operator_runbook tests/tmp_world_class_claim_guard tests/tmp_benchmark_reproducibility tests/tmp_evidence_consistency tests/tmp_feedback tests/tmp_daily_skillops tests/tmp_weekly_curator tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_telemetry_emit tests/tmp_telemetry_hooks tests/tmp_telemetry_native_host tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md find . -type d -name __pycache__ -prune -exec rm -rf {} + diff --git a/README.md b/README.md index 20e95e84..7e16b2b7 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ `yao-meta-skill` creates, evaluates, packages, and governs reusable agent skills. The 1.0 line focused on turning repeated workflows into installable, readable, cross-platform skill packages. The 2.0 line expands that factory into a Skill OS: a governed system for modeling a skill once, compiling it for multiple targets, testing its behavior, reviewing its release evidence, and tracking the next iteration. -[Quick Start](#quick-start) · [Skill OS 2.0](#skill-os-20-upgrade) · [1.0 vs 2.0](#from-10-to-20) · [Benchmark](#weighted-quality-benchmark) · [Examples](examples/README.md) · [Evals](evals/README.md) · [Failure Library](failures/README.md) · [Method Doctrine](#method-doctrine) +[Quick Start](#quick-start) · [Skill OS 2.0](#skill-os-20-upgrade) · [1.0 vs 2.0](#from-10-to-20) · [Operator UX](#operator-ux-commands) · [Benchmark](#weighted-quality-benchmark) · [Examples](examples/README.md) · [Evals](evals/README.md) · [Failure Library](failures/README.md) · [Method Doctrine](#method-doctrine) ## Skill OS 2.0 Upgrade @@ -53,6 +53,20 @@ See the companion artifacts: - **Keep a skill useful after release**: use metadata-only telemetry, adoption drift, feedback logs, SkillOps reports, and adaptive proposals to decide whether the next move should be documentation, an eval, a skill patch, or a governance update. - **Compare with other meta-skill approaches**: keep Anthropic/OpenAI-style conversational creation and lean instruction writing where they fit, then use `yao-meta-skill` when the package needs evidence, portability, release gates, and repeatable maintenance. +## Operator UX Commands + +These read-only helper commands turn common maintainer questions into repeatable diagnostics: + +```bash +python3 scripts/yao.py install-status --expected-source . +python3 scripts/yao.py localized-doc-sync-check +python3 scripts/yao.py pr-review-report 4 --repo yaojingang/yao-meta-skill +``` + +- `install-status` explains whether the active skill is coming from `.codex/skills`, `.agents/skills`, or the disabled mirror, and flags duplicate active installs. +- `localized-doc-sync-check` verifies that the Chinese README carries the public homepage sections that were added to the English README. +- `pr-review-report` reads GitHub PR metadata, changed files, status checks, and suggested local commands without merging or mutating the PR. + ## Capability Surface It turns rough workflows, transcripts, prompts, notes, and runbooks into reusable skill packages with: diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index 90293a1c..ec3e6132 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -4,7 +4,7 @@ `yao-meta-skill` 用来创建、评估、打包和治理可复用的 agent skill。1.0 的重点是把重复工作流整理成可安装、可阅读、可跨平台的 skill 包；2.0 进一步升级为 Skill OS，把建模、跨端编译、输出评测、评审工作台、证据账本、包体验证、发布门禁和后续迭代串成一套完整流程。 -[快速开始](#快速开始) · [Skill OS 2.0 升级](#skill-os-20-升级) · [从 1.0 到 2.0](#从-10-到-20) · [加权质量评测](#加权质量评测) · [与其他元 Skill 的适用差异](#与其他元-skill-的适用差异) +[快速开始](#快速开始) · [Skill OS 2.0 升级](#skill-os-20-升级) · [从 1.0 到 2.0](#从-10-到-20) · [Operator UX 命令](#operator-ux-命令) · [加权质量评测](#加权质量评测) · [与其他元 Skill 的适用差异](#与其他元-skill-的适用差异) ## Skill OS 2.0 升级 @@ -45,6 +45,20 @@ Skill OS 2.0 保留 `yao-meta-skill` 原来的轻量入口，但把 skill 的生 - **发布后继续迭代**：用元数据级遥测、采用漂移、feedback log、SkillOps 报告和自适应建议判断下一步是补文档、补 eval、改 skill，还是调整治理规则。 - **与其他元 skill 搭配使用**：保留 Anthropic/OpenAI 式对话创建和精简写作方法的优势，在需要证据、可移植性、发布门禁和长期维护时用 `yao-meta-skill` 加固。 +## Operator UX 命令 + +这组只读辅助命令把维护者最常见的判断变成可复验诊断： + +```bash +python3 scripts/yao.py install-status --expected-source . +python3 scripts/yao.py localized-doc-sync-check +python3 scripts/yao.py pr-review-report 4 --repo yaojingang/yao-meta-skill +``` + +- `install-status` 会说明当前 skill 是从 `.codex/skills`、`.agents/skills` 还是 disabled mirror（禁用镜像）被发现，并提示是否存在重复激活。 +- `localized-doc-sync-check` 会检查中文 README 是否同步包含英文 README 首页新增的公开说明模块。 +- `pr-review-report` 会只读获取 GitHub PR 元数据、变更文件、状态检查和建议本地命令，不会合并或修改 PR。 + ## 能力面 它把粗糙的 workflow、transcript、prompt、notes 和 runbook 转成可复用的 skill 包，并具备： diff --git a/scripts/ci_test.py b/scripts/ci_test.py index 9ec03633..e86d06ec 100644 --- a/scripts/ci_test.py +++ b/scripts/ci_test.py @@ -34,6 +34,7 @@ DEFAULT_TARGETS = [ "architecture-maintainability-check", "yao-cli-check", "yao-cli-world-class-check", + "operator-ux-check", "skill-overview-check", "skill-interpretation-check", "skill-report-metrics-check", diff --git a/scripts/yao.py b/scripts/yao.py index 09ffb3e0..fc51f30c 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -38,6 +38,11 @@ from yao_cli_output_commands import ( command_output_review_import, command_output_review_kit, ) +from yao_cli_operator_commands import ( + command_install_status, + command_localized_doc_sync_check, + command_pr_review_report, +) from yao_cli_parser import build_parser as build_cli_parser from yao_cli_report_commands import ( command_artifact_design_profile, diff --git a/scripts/yao_cli_operator_commands.py b/scripts/yao_cli_operator_commands.py new file mode 100644 index 00000000..5dfe85ce --- /dev/null +++ b/scripts/yao_cli_operator_commands.py @@ -0,0 +1,424 @@ +"""Operator UX commands for installation, docs sync, and PR review evidence.""" + +import argparse +import json +import shutil +import subprocess +from pathlib import Path +from typing import Any + +from yao_cli_runtime import ROOT + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by yao.py for operator-facing install, localized docs, and PR review diagnostics." + + +DEFAULT_DOC_PAIRS = [ + { + "key": "skill-os-2-upgrade", + "source": "## Skill OS 2.0 Upgrade", + "localized": "## Skill OS 2.0 升级", + }, + { + "key": "from-1-to-2", + "source": "## From 1.0 to 2.0", + "localized": "## 从 1.0 到 2.0", + }, + { + "key": "use-cases", + "source": "## 2.0 Use Cases", + "localized": "## 2.0 使用场景", + }, + { + "key": "operator-ux", + "source": "## Operator UX Commands", + "localized": "## Operator UX 命令", + }, + { + "key": "architecture", + "source": "## Architecture", + "localized": "## 架构图", + }, + { + "key": "benchmark", + "source": "## Weighted Quality Benchmark", + "localized": "## 加权质量评测", + }, + { + "key": "claim-boundary", + "source": 'Current posture: the repository is ready for beta and external testing', + "localized": "当前发布口径：仓库已经适合进入测试版和外部试用", + }, +] + + +def _expand(path: str) -> Path: + return Path(path).expanduser().resolve() + + +def _display(path: Path) -> str: + try: + return str(path.relative_to(Path.home())) + except ValueError: + return str(path) + + +def _skill_path_status(root: Path, skill_name: str, expected_source: Path) -> dict[str, Any]: + path = root / skill_name + exists = path.exists() + resolved = path.resolve() if exists or path.is_symlink() else path + has_skill_md = (path / "SKILL.md").exists() + return { + "root": str(root), + "path": str(path), + "exists": exists, + "is_symlink": path.is_symlink(), + "resolved": str(resolved), + "has_skill_md": has_skill_md, + "points_to_expected_source": resolved == expected_source, + } + + +def _recommend_install_status(codex: dict[str, Any], agents: dict[str, Any], disabled: dict[str, Any]) -> list[str]: + recommendations = [] + if codex["has_skill_md"]: + recommendations.append("Codex/Cortex can discover this skill through .codex/skills; restart the app if the list looks stale.") + if agents["has_skill_md"]: + recommendations.append("The skill is also active under .agents/skills; expect a duplicate entry when this repo is open.") + if not codex["has_skill_md"] and not agents["has_skill_md"] and disabled["has_skill_md"]: + recommendations.append("Only the disabled mirror exists; run make sync-active-install or add a .codex/skills symlink to activate it.") + if not codex["has_skill_md"] and not agents["has_skill_md"] and not disabled["has_skill_md"]: + recommendations.append("No active or disabled install was found for this skill name.") + if codex["has_skill_md"] and not codex["points_to_expected_source"]: + recommendations.append("The .codex/skills entry points somewhere else; inspect the symlink before editing or syncing.") + if not recommendations: + recommendations.append("No action required.") + return recommendations + + +def command_install_status(args: argparse.Namespace) -> int: + skill_name = args.skill_name + expected_source = _expand(args.expected_source) + codex_root = _expand(args.codex_root) + agents_root = _expand(args.agents_root) + disabled_root = _expand(args.disabled_root) + + codex = _skill_path_status(codex_root, skill_name, expected_source) + agents = _skill_path_status(agents_root, skill_name, expected_source) + disabled = _skill_path_status(disabled_root, skill_name, expected_source) + active_locations = [name for name, item in (("codex", codex), ("agents", agents)) if item["has_skill_md"]] + report = { + "ok": True, + "skill_name": skill_name, + "expected_source": str(expected_source), + "summary": { + "codex_active": codex["has_skill_md"], + "agents_active": agents["has_skill_md"], + "disabled_mirror": disabled["has_skill_md"], + "active_location_count": len(active_locations), + "duplicate_active": len(active_locations) > 1, + "active_locations": active_locations, + }, + "locations": { + "codex": codex, + "agents": agents, + "disabled": disabled, + }, + "recommendations": _recommend_install_status(codex, agents, disabled), + } + print(json.dumps(report, ensure_ascii=False, indent=2)) + return 0 + + +def _load_pairs(raw_pairs: list[str]) -> list[dict[str, str]]: + if not raw_pairs: + return DEFAULT_DOC_PAIRS + pairs = [] + for raw in raw_pairs: + parts = raw.split("::") + if len(parts) != 3: + raise ValueError("--pair must use key::source-marker::localized-marker") + pairs.append({"key": parts[0], "source": parts[1], "localized": parts[2]}) + return pairs + + +def _write_json(path: str | None, payload: dict[str, Any]) -> None: + if not path: + return + target = _expand(path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def _write_docs_sync_md(path: str | None, payload: dict[str, Any]) -> None: + if not path: + return + target = _expand(path) + target.parent.mkdir(parents=True, exist_ok=True) + lines = [ + "# Localized Docs Sync Check", + "", + f"- ok: `{str(payload['ok']).lower()}`", + f"- source: `{payload['source']}`", + f"- localized: `{payload['localized']}`", + f"- checked: `{payload['summary']['checked_count']}`", + f"- missing: `{payload['summary']['missing_count']}`", + f"- skipped: `{payload['summary']['skipped_count']}`", + "", + "## Pairs", + "", + ] + for item in payload["pairs"]: + lines.append(f"- `{item['key']}`: `{item['status']}`") + target.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def command_localized_doc_sync_check(args: argparse.Namespace) -> int: + source_path = _expand(args.source) + localized_path = _expand(args.localized) + try: + pairs = _load_pairs(args.pair) + except ValueError as exc: + report = { + "ok": False, + "source": str(source_path), + "localized": str(localized_path), + "summary": {"checked_count": 0, "missing_count": 0, "skipped_count": 0}, + "pairs": [], + "missing": [], + "failures": [str(exc)], + } + _write_json(args.output_json, report) + _write_docs_sync_md(args.output_md, report) + print(json.dumps(report, ensure_ascii=False, indent=2)) + return 2 + missing_paths = [str(path) for path in (source_path, localized_path) if not path.exists()] + if missing_paths: + report = { + "ok": False, + "source": str(source_path), + "localized": str(localized_path), + "summary": {"checked_count": 0, "missing_count": 0, "skipped_count": 0}, + "pairs": [], + "missing": [], + "failures": [f"Missing docs file: {path}" for path in missing_paths], + } + _write_json(args.output_json, report) + _write_docs_sync_md(args.output_md, report) + print(json.dumps(report, ensure_ascii=False, indent=2)) + return 2 + source_text = source_path.read_text(encoding="utf-8") + localized_text = localized_path.read_text(encoding="utf-8") + results = [] + for pair in pairs: + source_present = pair["source"] in source_text + localized_present = pair["localized"] in localized_text + if not source_present: + status = "skipped-source-missing" + elif localized_present: + status = "pass" + else: + status = "missing-localized-marker" + results.append( + { + **pair, + "source_present": source_present, + "localized_present": localized_present, + "status": status, + } + ) + missing = [item for item in results if item["status"] == "missing-localized-marker"] + checked = [item for item in results if item["source_present"]] + report = { + "ok": not missing, + "source": str(source_path), + "localized": str(localized_path), + "summary": { + "checked_count": len(checked), + "missing_count": len(missing), + "skipped_count": len(results) - len(checked), + }, + "pairs": results, + "missing": missing, + } + _write_json(args.output_json, report) + _write_docs_sync_md(args.output_md, report) + print(json.dumps(report, ensure_ascii=False, indent=2)) + return 0 if report["ok"] else 2 + + +def _run_gh(args: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run(["gh", *args], capture_output=True, text=True) + + +def _gh_available() -> bool: + return shutil.which("gh") is not None + + +def _review_depth(changed_files: int, changed_lines: int) -> str: + if changed_files <= 5 and changed_lines < 100: + return "quick" + if changed_files <= 10 and changed_lines < 500: + return "standard" + return "deep" + + +def _check_summary(checks: list[dict[str, Any]]) -> dict[str, Any]: + pending = [] + failed = [] + passed = [] + for item in checks: + name = str(item.get("name") or item.get("workflowName") or "check") + status = str(item.get("status") or "").upper() + conclusion = str(item.get("conclusion") or "").upper() + if status and status != "COMPLETED": + pending.append(name) + elif conclusion in {"SUCCESS", "SKIPPED", "NEUTRAL"}: + passed.append(name) + else: + failed.append(name) + return { + "present": bool(checks), + "passed_count": len(passed), + "pending_count": len(pending), + "failed_count": len(failed), + "passed": passed, + "pending": pending, + "failed": failed, + } + + +def _pr_decision(view: dict[str, Any], checks: dict[str, Any], require_checks: bool) -> str: + if view.get("state") != "OPEN": + return "not-open" + if view.get("isDraft"): + return "draft" + if view.get("mergeable") not in {"MERGEABLE", "UNKNOWN"}: + return "not-mergeable" + if checks["failed_count"]: + return "fix-failing-checks" + if checks["pending_count"]: + return "wait-for-checks" + if require_checks and not checks["present"]: + return "checks-required" + if not checks["present"]: + return "local-verification-required" + return "mergeable-after-review" + + +def _write_pr_review_md(path: str | None, payload: dict[str, Any]) -> None: + if not path: + return + target = _expand(path) + target.parent.mkdir(parents=True, exist_ok=True) + lines = [ + "# PR Review Report", + "", + f"- ok: `{str(payload['ok']).lower()}`", + f"- PR: `{payload['pr']}`", + f"- repo: `{payload.get('repo') or 'current'}`", + f"- decision: `{payload['decision']}`", + f"- review depth: `{payload['review_depth']}`", + f"- changed files: `{payload['summary']['changed_files']}`", + f"- additions: `{payload['summary']['additions']}`", + f"- deletions: `{payload['summary']['deletions']}`", + f"- checks present: `{str(payload['checks']['present']).lower()}`", + "", + "## Files", + "", + ] + for path_item in payload["files"]: + lines.append(f"- `{path_item}`") + lines.extend(["", "## Suggested Commands", ""]) + for command in payload["suggested_commands"]: + lines.append(f"- `{command}`") + target.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def command_pr_review_report(args: argparse.Namespace) -> int: + if not _gh_available(): + report = { + "ok": False, + "pr": args.pr, + "repo": args.repo, + "decision": "missing-gh-cli", + "failures": ["GitHub CLI `gh` is not available on PATH."], + } + print(json.dumps(report, ensure_ascii=False, indent=2)) + return 2 + + fields = ",".join( + [ + "number", + "title", + "state", + "isDraft", + "mergeable", + "author", + "baseRefName", + "headRefName", + "url", + "additions", + "deletions", + "changedFiles", + "commits", + "statusCheckRollup", + "reviewDecision", + "maintainerCanModify", + ] + ) + view_args = ["pr", "view", args.pr, "--json", fields] + diff_args = ["pr", "diff", args.pr, "--name-only"] + if args.repo: + view_args.extend(["--repo", args.repo]) + diff_args.extend(["--repo", args.repo]) + view_proc = _run_gh(view_args) + if view_proc.returncode != 0: + report = { + "ok": False, + "pr": args.pr, + "repo": args.repo, + "decision": "gh-pr-view-failed", + "returncode": view_proc.returncode, + "stderr": view_proc.stderr.strip(), + } + print(json.dumps(report, ensure_ascii=False, indent=2)) + return 2 + view = json.loads(view_proc.stdout) + diff_proc = _run_gh(diff_args) + files = [line.strip() for line in diff_proc.stdout.splitlines() if line.strip()] if diff_proc.returncode == 0 else [] + checks = _check_summary(view.get("statusCheckRollup") or []) + changed_lines = int(view.get("additions") or 0) + int(view.get("deletions") or 0) + depth = _review_depth(int(view.get("changedFiles") or len(files)), changed_lines) + decision = _pr_decision(view, checks, args.require_checks) + report = { + "ok": decision not in {"missing-gh-cli", "gh-pr-view-failed", "checks-required", "fix-failing-checks", "not-mergeable", "draft", "not-open"}, + "pr": str(view.get("number", args.pr)), + "repo": args.repo, + "url": view.get("url", ""), + "title": view.get("title", ""), + "state": view.get("state", ""), + "mergeable": view.get("mergeable", ""), + "decision": decision, + "review_depth": depth, + "summary": { + "changed_files": int(view.get("changedFiles") or len(files)), + "additions": int(view.get("additions") or 0), + "deletions": int(view.get("deletions") or 0), + "commit_count": len(view.get("commits") or []), + "maintainer_can_modify": bool(view.get("maintainerCanModify")), + }, + "checks": checks, + "files": files, + "suggested_commands": [ + f"gh pr view {args.pr}{' --repo ' + args.repo if args.repo else ''} --json number,title,state,mergeable,statusCheckRollup", + f"gh pr diff {args.pr}{' --repo ' + args.repo if args.repo else ''} --name-only", + f"git fetch origin pull/{args.pr}/head:refs/tmp/pr-{args.pr}", + f"git diff --check origin/main...refs/tmp/pr-{args.pr}", + ], + } + _write_json(args.output_json, report) + _write_pr_review_md(args.output_md, report) + print(json.dumps(report, ensure_ascii=False, indent=2)) + return 0 if report["ok"] else 2 diff --git a/scripts/yao_cli_parser.py b/scripts/yao_cli_parser.py index 60e5e916..dab6a01f 100644 --- a/scripts/yao_cli_parser.py +++ b/scripts/yao_cli_parser.py @@ -5,6 +5,7 @@ import argparse from collections.abc import Callable from yao_cli_parser_evidence import add_evidence_commands +from yao_cli_parser_operator import add_operator_commands from yao_cli_parser_operations import add_operating_loop_commands @@ -299,6 +300,7 @@ def build_parser(command_handlers: dict[str, Callable[[argparse.Namespace], int] feedback_cmd.add_argument("--recommended-action", default="review") feedback_cmd.set_defaults(func=_handler(command_handlers, "command_feedback")) + add_operator_commands(subparsers, command_handlers) add_operating_loop_commands(subparsers, command_handlers) baseline_compare_cmd = subparsers.add_parser( diff --git a/scripts/yao_cli_parser_operator.py b/scripts/yao_cli_parser_operator.py new file mode 100644 index 00000000..795d927c --- /dev/null +++ b/scripts/yao_cli_parser_operator.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Operator UX command declarations for the Yao CLI parser.""" + +import argparse +from collections.abc import Callable + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by yao_cli_parser.py to keep operator UX command declarations out of the main parser module." + + +def _handler(command_handlers: dict[str, Callable[[argparse.Namespace], int]], name: str) -> Callable[[argparse.Namespace], int]: + if name not in command_handlers: + raise KeyError(f"Missing CLI command handler: {name}") + return command_handlers[name] + + +def add_operator_commands( + subparsers: argparse._SubParsersAction, + command_handlers: dict[str, Callable[[argparse.Namespace], int]], +) -> None: + install_status_cmd = subparsers.add_parser( + "install-status", + help="Diagnose whether yao-meta-skill is active in Codex/Cortex, .agents, or a disabled mirror.", + ) + install_status_cmd.add_argument("--skill-name", default="yao-meta-skill") + install_status_cmd.add_argument("--expected-source", default=".") + install_status_cmd.add_argument("--codex-root", default="~/.codex/skills") + install_status_cmd.add_argument("--agents-root", default="~/.agents/skills") + install_status_cmd.add_argument("--disabled-root", default="~/.agents/skills.disabled") + install_status_cmd.set_defaults(func=_handler(command_handlers, "command_install_status")) + + localized_docs_cmd = subparsers.add_parser( + "localized-doc-sync-check", + help="Check that localized README docs contain the public homepage sections introduced in README.md.", + ) + localized_docs_cmd.add_argument("--source", default="README.md") + localized_docs_cmd.add_argument("--localized", default="docs/README.zh-CN.md") + localized_docs_cmd.add_argument( + "--pair", + action="append", + default=[], + help="Custom sync marker in key::source-marker::localized-marker form. Replaces defaults when present.", + ) + localized_docs_cmd.add_argument("--output-json") + localized_docs_cmd.add_argument("--output-md") + localized_docs_cmd.set_defaults(func=_handler(command_handlers, "command_localized_doc_sync_check")) + + pr_review_cmd = subparsers.add_parser( + "pr-review-report", + help="Build a read-only GitHub PR review report with mergeability, checks, files, and suggested commands.", + ) + pr_review_cmd.add_argument("pr") + pr_review_cmd.add_argument("--repo") + pr_review_cmd.add_argument("--require-checks", action="store_true") + pr_review_cmd.add_argument("--output-json") + pr_review_cmd.add_argument("--output-md") + pr_review_cmd.set_defaults(func=_handler(command_handlers, "command_pr_review_report")) diff --git a/tests/verify_operator_ux.py b/tests/verify_operator_ux.py new file mode 100644 index 00000000..4f81ddbb --- /dev/null +++ b/tests/verify_operator_ux.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +import json +import os +import shutil +import stat +import textwrap +from pathlib import Path + +from yao_cli_helpers import ROOT, run, run_with_env + + +def write_skill(path: Path, description: str = "Operator UX test skill.") -> None: + path.mkdir(parents=True, exist_ok=True) + (path / "SKILL.md").write_text( + f"---\nname: yao-meta-skill\ndescription: {description}\n---\n\n# Test Skill\n", + encoding="utf-8", + ) + + +def write_fake_gh(bin_dir: Path) -> None: + bin_dir.mkdir(parents=True, exist_ok=True) + script = bin_dir / "gh" + script.write_text( + textwrap.dedent( + """\ + #!/usr/bin/env python3 + import json + import os + import sys + + args = sys.argv[1:] + if args[:2] == ["pr", "view"]: + checks = [] if os.environ.get("YAO_FAKE_GH_EMPTY_CHECKS") else [ + { + "__typename": "CheckRun", + "name": "test", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "workflowName": "test", + } + ] + print(json.dumps({ + "number": 4, + "title": "docs: clarify Python 3.11 local development", + "state": "OPEN", + "isDraft": False, + "mergeable": "MERGEABLE", + "author": {"login": "contributor"}, + "baseRefName": "main", + "headRefName": "docs-python-311", + "url": "https://github.com/example/repo/pull/4", + "additions": 23, + "deletions": 3, + "changedFiles": 2, + "commits": [{"oid": "abc123", "messageHeadline": "docs: clarify Python 3.11 local development"}], + "statusCheckRollup": checks, + "reviewDecision": "", + "maintainerCanModify": True, + })) + raise SystemExit(0) + if args[:2] == ["pr", "diff"]: + print("Makefile") + print("README.md") + raise SystemExit(0) + print("unexpected gh invocation", args, file=sys.stderr) + raise SystemExit(2) + """ + ), + encoding="utf-8", + ) + script.chmod(script.stat().st_mode | stat.S_IXUSR) + + +def main() -> None: + tmp_root = ROOT / "tests" / "tmp_operator_ux" + shutil.rmtree(tmp_root, ignore_errors=True) + tmp_root.mkdir(parents=True, exist_ok=True) + + source = tmp_root / "source" / "yao-meta-skill" + codex_root = tmp_root / "codex-skills" + agents_root = tmp_root / "agents-skills" + disabled_root = tmp_root / "disabled-skills" + codex_root.mkdir() + agents_root.mkdir() + disabled_root.mkdir() + write_skill(source) + (codex_root / "yao-meta-skill").symlink_to(source, target_is_directory=True) + write_skill(disabled_root / "yao-meta-skill", "Disabled mirror.") + + install_status = run( + "install-status", + "--expected-source", + str(source), + "--codex-root", + str(codex_root), + "--agents-root", + str(agents_root), + "--disabled-root", + str(disabled_root), + ) + assert install_status["ok"], install_status + assert install_status["payload"]["summary"]["codex_active"] is True, install_status + assert install_status["payload"]["summary"]["agents_active"] is False, install_status + assert install_status["payload"]["summary"]["disabled_mirror"] is True, install_status + assert install_status["payload"]["locations"]["codex"]["is_symlink"] is True, install_status + assert install_status["payload"]["locations"]["codex"]["points_to_expected_source"] is True, install_status + assert any("restart" in item.lower() for item in install_status["payload"]["recommendations"]), install_status + + sync_ok = run("localized-doc-sync-check") + assert sync_ok["ok"], sync_ok + assert sync_ok["payload"]["summary"]["missing_count"] == 0, sync_ok + assert any(item["key"] == "operator-ux" for item in sync_ok["payload"]["pairs"]), sync_ok + + source_readme = tmp_root / "README.md" + localized_readme = tmp_root / "README.zh-CN.md" + source_readme.write_text("## Skill OS 2.0 Upgrade\n## 2.0 Use Cases\n", encoding="utf-8") + localized_readme.write_text("## Skill OS 2.0 升级\n", encoding="utf-8") + sync_fail = run( + "localized-doc-sync-check", + "--source", + str(source_readme), + "--localized", + str(localized_readme), + "--pair", + "upgrade::## Skill OS 2.0 Upgrade::## Skill OS 2.0 升级", + "--pair", + "use-cases::## 2.0 Use Cases::## 2.0 使用场景", + "--output-json", + str(tmp_root / "sync.json"), + "--output-md", + str(tmp_root / "sync.md"), + ) + assert not sync_fail["ok"], sync_fail + assert sync_fail["returncode"] == 2, sync_fail + assert sync_fail["payload"]["summary"]["missing_count"] == 1, sync_fail + assert (tmp_root / "sync.json").exists(), sync_fail + assert "use-cases" in (tmp_root / "sync.md").read_text(encoding="utf-8"), sync_fail + + sync_missing_file = run( + "localized-doc-sync-check", + "--source", + str(tmp_root / "missing-README.md"), + "--localized", + str(localized_readme), + ) + assert not sync_missing_file["ok"], sync_missing_file + assert sync_missing_file["payload"]["failures"], sync_missing_file + assert "Missing docs file" in sync_missing_file["payload"]["failures"][0], sync_missing_file + + fake_bin = tmp_root / "bin" + write_fake_gh(fake_bin) + fake_env = {"PATH": f"{fake_bin}{os.pathsep}{os.environ.get('PATH', '')}"} + pr_report = run_with_env( + fake_env, + "pr-review-report", + "4", + "--repo", + "example/repo", + "--output-json", + str(tmp_root / "pr.json"), + "--output-md", + str(tmp_root / "pr.md"), + ) + assert pr_report["ok"], pr_report + assert pr_report["payload"]["decision"] == "mergeable-after-review", pr_report + assert pr_report["payload"]["review_depth"] == "quick", pr_report + assert pr_report["payload"]["checks"]["passed_count"] == 1, pr_report + assert pr_report["payload"]["files"] == ["Makefile", "README.md"], pr_report + assert (tmp_root / "pr.json").exists(), pr_report + assert "Suggested Commands" in (tmp_root / "pr.md").read_text(encoding="utf-8"), pr_report + + empty_check_env = dict(fake_env) + empty_check_env["YAO_FAKE_GH_EMPTY_CHECKS"] = "1" + pr_without_checks = run_with_env( + empty_check_env, + "pr-review-report", + "4", + "--repo", + "example/repo", + ) + assert pr_without_checks["ok"], pr_without_checks + assert pr_without_checks["payload"]["decision"] == "local-verification-required", pr_without_checks + pr_require_checks = run_with_env( + empty_check_env, + "pr-review-report", + "4", + "--repo", + "example/repo", + "--require-checks", + ) + assert not pr_require_checks["ok"], pr_require_checks + assert pr_require_checks["payload"]["decision"] == "checks-required", pr_require_checks + + print("operator ux checks passed") + + +if __name__ == "__main__": + main() diff --git a/tests/verify_trust_check.py b/tests/verify_trust_check.py index 42152e09..10ff49d6 100644 --- a/tests/verify_trust_check.py +++ b/tests/verify_trust_check.py @@ -130,8 +130,10 @@ def main() -> None: "scripts/yao_cli_config.py", "scripts/yao_cli_distribution_commands.py", "scripts/yao_cli_output_commands.py", + "scripts/yao_cli_operator_commands.py", "scripts/yao_cli_parser.py", "scripts/yao_cli_parser_evidence.py", + "scripts/yao_cli_parser_operator.py", "scripts/yao_cli_parser_operations.py", "scripts/yao_cli_report_commands.py", "scripts/yao_cli_telemetry.py", @@ -173,8 +175,10 @@ def main() -> None: assert "yao_cli_config.py" not in warning_text, payload["warnings"] assert "yao_cli_distribution_commands.py" not in warning_text, payload["warnings"] assert "yao_cli_output_commands.py" not in warning_text, payload["warnings"] + assert "yao_cli_operator_commands.py" not in warning_text, payload["warnings"] assert "yao_cli_parser.py" not in warning_text, payload["warnings"] assert "yao_cli_parser_evidence.py" not in warning_text, payload["warnings"] + assert "yao_cli_parser_operator.py" not in warning_text, payload["warnings"] assert "yao_cli_parser_operations.py" not in warning_text, payload["warnings"] assert "yao_cli_report_commands.py" not in warning_text, payload["warnings"] assert "yao_cli_telemetry.py" not in warning_text, payload["warnings"] diff --git a/tests/yao_cli_helpers.py b/tests/yao_cli_helpers.py index dae5daf1..bcf67871 100644 --- a/tests/yao_cli_helpers.py +++ b/tests/yao_cli_helpers.py @@ -17,7 +17,9 @@ import yao_cli_distribution_commands # noqa: E402 import yao_cli_output_commands # noqa: E402 import yao_cli_parser # noqa: E402 import yao_cli_parser_evidence # noqa: E402 +import yao_cli_parser_operator # noqa: E402 import yao_cli_parser_operations # noqa: E402 +import yao_cli_operator_commands # noqa: E402 import yao_cli_report_commands # noqa: E402 import yao_cli_runtime # noqa: E402 from yao_cli_report_refresh import refresh_root_report_consistency_inputs # noqa: E402 @@ -89,7 +91,9 @@ def assert_cli_module_contracts() -> None: yao_cli_adaptation_commands, yao_cli_distribution_commands, yao_cli_output_commands, + yao_cli_operator_commands, yao_cli_report_commands, + yao_cli_parser_operator, ): assert module.SCRIPT_INTERFACE == "internal-module" assert callable(yao_cli_module.command_review_studio) @@ -99,6 +103,7 @@ def assert_help_surface() -> None: parser_help = yao_cli_module.build_parser().format_help() expected_help = ( "quickstart skill-interpretation review-studio python-compat architecture-audit skill-os2-audit skill-os2-coverage " + "install-status localized-doc-sync-check pr-review-report " "world-class-evidence world-class-ledger world-class-intake world-class-preflight world-class-submission-kit world-class-submission-review world-class-runbook world-class-claim-guard " "benchmark-reproducibility evidence-consistency output-review-kit output-review-import adapt-scan adapt-propose adapt-apply telemetry-import telemetry-emit telemetry-hooks weekly-curator --record-cli-telemetry" ).split()