frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Harden world-class release gates

Browse Source
This commit is contained in:
yaojingang
2026-06-17 15:50:49 +08:00
parent e503cd6333
commit cece6f9b04
98 changed files with 3067 additions and 2714 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/provider_output_eval_runner.py
+107 -13
View File
@@ -11,9 +11,26 @@ from urllib.request import Request, urlopen
ROOT = Path(__file__).resolve().parent.parent
DEFAULT_BASE_URL = "https://api.openai.com/v1/responses"
DEFAULT_HOST = urlparse(DEFAULT_BASE_URL).hostname or "api.openai.com"
DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1/responses"
DEFAULT_DEEPSEEK_BASE_URL = "https://api.deepseek.com/chat/completions"
DEFAULT_BASE_URL = DEFAULT_OPENAI_BASE_URL
DEFAULT_HOST = urlparse(DEFAULT_OPENAI_BASE_URL).hostname or "api.openai.com"
DEFAULT_PROVIDER_CONFIGS = {
"openai": {
"base_url": DEFAULT_OPENAI_BASE_URL,
"api_format": "responses",
"api_key_env": "OPENAI_API_KEY",
"thinking": "",
},
"deepseek": {
"base_url": DEFAULT_DEEPSEEK_BASE_URL,
"api_format": "chat-completions",
"api_key_env": "DEEPSEEK_API_KEY",
"thinking": "disabled",
},
}
ALLOWED_PATH_PREFIX = "/v1/responses"
CHAT_COMPLETIONS_PATHS = {"/chat/completions", "/v1/chat/completions"}
LOCAL_HOSTS = {"127.0.0.1", "localhost", "::1"}
@@ -22,12 +39,23 @@ def fail(message: str) -> None:
raise SystemExit(2)
def validate_base_url(base_url: str, allow_insecure_localhost: bool, allow_custom_base_url: bool) -> None:
def validate_base_url(
base_url: str,
allow_insecure_localhost: bool,
allow_custom_base_url: bool,
api_format: str,
) -> None:
parsed = urlparse(base_url)
host = parsed.hostname or ""
if not parsed.path.startswith(ALLOWED_PATH_PREFIX):
if api_format == "responses" and not parsed.path.startswith(ALLOWED_PATH_PREFIX):
fail(f"provider endpoint path must start with {ALLOWED_PATH_PREFIX}")
if parsed.scheme == "https" and (host == DEFAULT_HOST or allow_custom_base_url):
if api_format == "chat-completions" and parsed.path not in CHAT_COMPLETIONS_PATHS:
fail("chat-completions provider endpoint path must be /chat/completions or /v1/chat/completions")
if parsed.scheme == "https" and (
base_url in {DEFAULT_OPENAI_BASE_URL, DEFAULT_DEEPSEEK_BASE_URL}
or host == DEFAULT_HOST
or allow_custom_base_url
):
return
if parsed.scheme == "https":
fail("custom provider host requires --allow-custom-base-url")
@@ -36,6 +64,10 @@ def validate_base_url(base_url: str, allow_insecure_localhost: bool, allow_custo
fail("provider runner requires HTTPS; use --allow-insecure-localhost only for local test servers")
def provider_defaults(provider: str) -> dict[str, str]:
return DEFAULT_PROVIDER_CONFIGS.get(provider, DEFAULT_PROVIDER_CONFIGS["openai"])
def load_request() -> dict[str, Any]:
raw = sys.stdin.read()
if not raw.strip():
@@ -173,8 +205,43 @@ def observed_usage(payload: dict[str, Any]) -> dict[str, Any]:
return result
def call_provider(base_url: str, api_key: str, model: str, provider_input: str, timeout_seconds: float) -> dict[str, Any]:
body = json.dumps({"model": model, "input": provider_input}, ensure_ascii=False).encode("utf-8")
def request_body(
model: str,
provider_input: str,
api_format: str,
thinking_type: str,
temperature: float | None,
) -> dict[str, Any]:
if api_format == "chat-completions":
body: dict[str, Any] = {
"model": model,
"messages": [{"role": "user", "content": provider_input}],
}
if thinking_type:
body["thinking"] = {"type": thinking_type}
if temperature is not None:
body["temperature"] = temperature
return body
body = {"model": model, "input": provider_input}
if temperature is not None:
body["temperature"] = temperature
return body
def call_provider(
base_url: str,
api_key: str,
model: str,
provider_input: str,
timeout_seconds: float,
api_format: str,
thinking_type: str,
temperature: float | None,
) -> dict[str, Any]:
body = json.dumps(
request_body(model, provider_input, api_format, thinking_type, temperature),
ensure_ascii=False,
).encode("utf-8")
request = Request(
base_url,
data=body,
@@ -209,9 +276,19 @@ def main() -> None:
)
)
parser.add_argument("--provider", default="openai", help="Provider label to write into execution evidence.")
parser.add_argument("--base-url", default=DEFAULT_BASE_URL, help="OpenAI Responses API compatible endpoint.")
parser.add_argument(
"--base-url",
help="Override the provider endpoint. Defaults are selected from --provider.",
)
parser.add_argument(
"--api-format",
choices=["responses", "chat-completions"],
help="Provider API shape. Defaults are selected from --provider.",
)
parser.add_argument("--thinking", choices=["enabled", "disabled"], help="Optional chat-completions thinking mode.")
parser.add_argument("--temperature", type=float, default=0.0, help="Sampling temperature for provider requests.")
parser.add_argument("--model", default=os.environ.get("YAO_OUTPUT_EVAL_MODEL", ""))
parser.add_argument("--api-key-env", default="OPENAI_API_KEY")
parser.add_argument("--api-key-env", help="Environment variable that contains the provider API key.")
parser.add_argument("--input-root", default=str(ROOT / "evals" / "output"))
parser.add_argument("--skill-file", default=str(ROOT / "SKILL.md"))
parser.add_argument("--timeout-seconds", type=float, default=60.0)
@@ -221,18 +298,35 @@ def main() -> None:
parser.add_argument("--allow-custom-base-url", action="store_true")
args = parser.parse_args()
validate_base_url(args.base_url, args.allow_insecure_localhost, args.allow_custom_base_url)
defaults = provider_defaults(args.provider)
base_url = args.base_url or defaults["base_url"]
api_format = args.api_format or defaults["api_format"]
thinking = args.thinking if args.thinking is not None else defaults["thinking"]
api_key_env = args.api_key_env or defaults["api_key_env"]
validate_base_url(base_url, args.allow_insecure_localhost, args.allow_custom_base_url, api_format)
if args.temperature is not None and not 0 <= args.temperature <= 2:
fail("--temperature must be between 0 and 2")
if not args.model:
fail("missing model; pass --model or set YAO_OUTPUT_EVAL_MODEL")
api_key = os.environ.get(args.api_key_env, "")
api_key = os.environ.get(api_key_env, "")
if not api_key:
fail(f"missing API key env: {args.api_key_env}")
fail(f"missing API key env: {api_key_env}")
request = load_request()
input_files = read_input_files(request.get("input_files", []), Path(args.input_root).resolve(), args.max_input_file_chars)
skill_text = read_skill_instructions(Path(args.skill_file).resolve(), args.max_skill_chars)
provider_input = build_provider_input(request, skill_text, input_files)
response = call_provider(args.base_url, api_key, args.model, provider_input, args.timeout_seconds)
response = call_provider(
base_url,
api_key,
args.model,
provider_input,
args.timeout_seconds,
api_format,
thinking,
args.temperature,
)
output = response_text(response)
if not output:
fail("provider response did not contain output text")
scripts/render_benchmark_reproducibility.py
+16 -6
View File
@@ -364,6 +364,21 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
world_class_source_blocked_count,
)
public_claim_ready = not claim_blockers
limitations = [
"The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.",
"Pending blind-review decisions are visible but do not count as human adjudication.",
"World-class readiness remains false until external and human evidence gaps close.",
]
if provider_evidence_complete:
limitations.insert(
1,
"Provider-backed model holdout source evidence is complete, but ledger acceptance still requires a valid independently reviewed submission packet.",
)
else:
limitations.insert(
1,
"Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.",
)
return {
"schema_version": "1.0",
"ok": local_reproducibility_ready,
@@ -422,12 +437,7 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
"case_count": failure_case_count,
"policy": "Keep representative failures visible and tied to regression checks.",
},
"limitations": [
"The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.",
"Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.",
"Pending blind-review decisions are visible but do not count as human adjudication.",
"World-class readiness remains false until external and human evidence gaps close.",
],
"limitations": limitations,
"artifacts": {
"json": "reports/benchmark_reproducibility.json",
"markdown": "reports/benchmark_reproducibility.md",
scripts/render_world_class_evidence_plan.py
+20 -6
View File
@@ -6,6 +6,7 @@ from pathlib import Path
from typing import Any
from render_skill_os2_audit import build_audit
from world_class_evidence_contract import load_json_with_status, validate_payload
ROOT = Path(__file__).resolve().parent.parent
@@ -17,9 +18,9 @@ TASK_TEMPLATES: dict[str, dict[str, Any]] = {
"owner": "operator with provider credentials",
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
"runbook": [
"Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.",
"export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}",
"python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
"Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.",
"For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60",
"For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120",
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
],
"success_checks": [
@@ -186,12 +187,25 @@ def build_task(item: dict[str, Any]) -> dict[str, Any]:
}
def build_plan(skill_dir: Path, generated_at: str) -> dict[str, Any]:
def has_accepted_ledger_submission(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) -> bool:
path = submissions_dir / f"{task['key']}.json"
payload, load_status = load_json_with_status(path)
if load_status != "present":
return False
validation = validate_payload(payload, task, path=path, root=skill_dir, template_expected=False)
return validation.get("status") == "pass"
def build_plan(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]:
audit = build_audit(skill_dir, generated_at)
submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions")
evidence_keys = set(TASK_TEMPLATES)
evidence_requirements = [build_task(item) for item in audit["items"] if item["key"] in evidence_keys]
tasks = [task for task in evidence_requirements if task["status"] != "pass"]
tasks.extend(build_task(item) for item in audit["items"] if item["status"] != "pass" and item["key"] not in evidence_keys)
tasks = [
task
for task in evidence_requirements
if task["status"] != "pass" or not has_accepted_ledger_submission(skill_dir, task, submissions_dir)
]
category_counts: dict[str, int] = {}
for task in tasks:
category_counts[task["category"]] = category_counts.get(task["category"], 0) + 1
scripts/render_world_class_operator_runbook.py
+12 -8
View File
@@ -42,6 +42,7 @@ def build_runbook_item(
review_item: dict[str, Any],
preflight_item: dict[str, Any],
) -> dict[str, Any]:
accepted = entry.get("status") == "accepted" or review_item.get("review_state") == "accepted"
commands = checklist.get("commands", {}) if isinstance(checklist.get("commands", {}), dict) else {}
must_collect = checklist.get("must_collect", {}) if isinstance(checklist.get("must_collect", {}), dict) else {}
source_checklist = review_item.get("source_checklist", [])
@@ -53,6 +54,13 @@ def build_runbook_item(
action = str(row.get("next_action", "")).strip()
if action and action not in next_source_actions:
next_source_actions.append(action)
repair_checklist = (
preflight_item.get("repair_checklist", []) if isinstance(preflight_item.get("repair_checklist", []), list) else []
)
phase_queue = preflight_item.get("phase_queue", []) if isinstance(preflight_item.get("phase_queue", []), list) else []
if accepted:
repair_checklist = []
phase_queue = []
return {
"evidence_key": entry.get("key", ""),
"label": entry.get("label", entry.get("key", "")),
@@ -85,21 +93,17 @@ def build_runbook_item(
"source_checklist": source_checklist,
"blocked_source_check_count": len(blocked_source_checks),
"next_source_actions": next_source_actions,
"repair_checklist": preflight_item.get("repair_checklist", [])
if isinstance(preflight_item.get("repair_checklist", []), list)
else [],
"repair_checklist": repair_checklist,
"repair_blocked_count": sum(
1
for row in preflight_item.get("repair_checklist", [])
for row in repair_checklist
if isinstance(row, dict) and row.get("status") != "ready"
),
"repair_counts_as_completion": False,
"phase_queue": preflight_item.get("phase_queue", [])
if isinstance(preflight_item.get("phase_queue", []), list)
else [],
"phase_queue": phase_queue,
"phase_queue_blocked_count": sum(
1
for row in preflight_item.get("phase_queue", [])
for row in phase_queue
if isinstance(row, dict) and row.get("status") != "ready"
),
"phase_queue_counts_as_completion": False,
scripts/render_world_class_preflight.py
+19 -6
View File
@@ -43,13 +43,13 @@ PREFLIGHT_SPECS: dict[str, list[dict[str, Any]]] = {
"next_action": "Use the provider runner instead of the local command runner for model-backed evidence.",
},
{
"key": "openai-api-key",
"key": "provider-api-key",
"label": "Provider credential",
"kind": "env",
"name": "OPENAI_API_KEY",
"kind": "env_any",
"names": ["OPENAI_API_KEY", "DEEPSEEK_API_KEY"],
"required": True,
"secret": True,
"next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.",
"next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.",
},
{
"key": "provider-model",
@@ -57,8 +57,8 @@ PREFLIGHT_SPECS: dict[str, list[dict[str, Any]]] = {
"kind": "env",
"name": "YAO_OUTPUT_EVAL_MODEL",
"required": False,
"default": "gpt-4.1-mini",
"next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL; the runbook defaults to gpt-4.1-mini.",
"default": "provider-specific model, or pass --provider-model",
"next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL, or pass --provider-model for the selected provider.",
},
],
"human-adjudication": [
@@ -245,6 +245,19 @@ def build_precheck(skill_dir: Path, evidence_key: str, spec: dict[str, Any]) ->
}
)
return row
if kind == "env_any":
names = [str(name) for name in spec.get("names", []) if str(name).strip()]
set_names = [name for name in names if env_present(name)]
row.update(
{
"env_any": names,
"status": "pass" if set_names else ("missing" if required else "optional"),
"actual": "set" if set_names else "not-set",
"set_count": len(set_names),
"set_envs": set_names,
}
)
return row
if kind == "human":
row.update(
{
scripts/world_class_operator_runbook_coordination.py
+2 -2
View File
@@ -19,10 +19,10 @@ COORDINATION_GUIDANCE: dict[str, dict[str, str]] = {
"provider-holdout": {
"phase": "provider-holdout",
"owner": "assistant + operator with provider credentials",
"user_action": "Provide the provider API key through an environment variable and confirm the model to use.",
"user_action": "Provide the selected provider API key through an environment variable and confirm the provider, model, endpoint, and API format to use.",
"assistant_action": "Run provider-backed output execution, verify aggregate timing and token metadata, then prepare the evidence packet.",
"external_dependency": "Valid provider credentials and a live provider endpoint.",
"command": "python3 scripts/yao.py output-exec . --provider-runner openai --provider-model <model> --timeout-seconds 60",
"command": "python3 scripts/yao.py output-exec . --provider-runner <openai|deepseek> --provider-model <model> --timeout-seconds 60",
"pass_condition": "reports/output_execution_runs.json has model_executed_count > 0 and token_observed_count > 0.",
"artifact": "reports/output_execution_runs.json",
"privacy_boundary": "Commit aggregate metadata only; do not commit API keys, raw prompts, raw outputs, or provider request payloads.",
scripts/world_class_repair_checklist.py
+5 -1
View File
@@ -55,7 +55,11 @@ def repair_verification_command(evidence_key: str, repair_type: str) -> str:
if repair_type == "unknown-key":
return "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
if evidence_key == "provider-holdout" and repair_type == "source-check":
return "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && " + preflight
return (
"python3 scripts/yao.py output-exec --provider-runner <openai|deepseek> "
"--provider-model <model> --timeout-seconds 60 && "
+ preflight
)
if evidence_key == "human-adjudication" and repair_type == "source-check":
return "python3 scripts/yao.py output-review && " + preflight
if evidence_key == "native-permission-enforcement" and repair_type == "source-check":
scripts/yao.py
+5 -2
View File
@@ -68,7 +68,7 @@ from yao_cli_report_commands import (
command_world_class_submission_kit,
command_world_class_submission_review,
)
from yao_cli_runtime import ROOT, run_adoption_drift_if_source_exists, run_script
from yao_cli_runtime import ROOT, allow_report_status, run_adoption_drift_if_source_exists, run_script
from yao_cli_telemetry import add_telemetry_args, maybe_record_cli_event
@@ -442,7 +442,10 @@ def command_workspace_flow(args: argparse.Namespace) -> int:
{"phase": "report-refresh", "result": run_script("render_world_class_evidence_plan.py", [str(ROOT)])},
{"phase": "report-refresh", "result": run_script("render_world_class_evidence_ledger.py", [str(ROOT)])},
{"phase": "report-refresh", "result": run_script("render_world_class_evidence_intake.py", [str(ROOT)])},
{"phase": "report-refresh", "result": run_script("render_world_class_submission_review.py", [str(ROOT)])},
{
"phase": "report-refresh",
"result": allow_report_status(run_script("render_world_class_submission_review.py", [str(ROOT)])),
},
{"phase": "report-refresh", "result": run_script("render_world_class_operator_runbook.py", [str(ROOT)])},
{"phase": "report-refresh", "result": run_script("render_world_class_claim_guard.py", [str(ROOT)])},
{"phase": "report-refresh", "result": run_script("render_skill_os2_coverage.py", [str(ROOT)])},
scripts/yao_cli_config.py
+13
View File
@@ -72,15 +72,28 @@ def provider_output_runner_command(
provider: str,
model: str | None = None,
base_url: str | None = None,
api_format: str | None = None,
thinking: str | None = None,
temperature: float | None = None,
api_key_env: str | None = None,
allow_insecure_localhost: bool = False,
allow_custom_base_url: bool = False,
) -> str:
command = ["python3", "scripts/provider_output_eval_runner.py", "--provider", provider]
if provider == "deepseek":
api_format = api_format or "chat-completions"
thinking = thinking or "disabled"
api_key_env = api_key_env or "DEEPSEEK_API_KEY"
if model:
command.extend(["--model", model])
if base_url:
command.extend(["--base-url", base_url])
if api_format:
command.extend(["--api-format", api_format])
if thinking:
command.extend(["--thinking", thinking])
if temperature is not None:
command.extend(["--temperature", str(temperature)])
if api_key_env:
command.extend(["--api-key-env", api_key_env])
if allow_insecure_localhost:
scripts/yao_cli_output_commands.py
+3
View File
@@ -57,6 +57,9 @@ def command_output_execution(args: argparse.Namespace) -> int:
args.provider_runner,
model=args.provider_model,
base_url=args.provider_base_url,
api_format=args.provider_api_format,
thinking=args.provider_thinking,
temperature=args.provider_temperature,
api_key_env=args.api_key_env,
allow_insecure_localhost=args.allow_insecure_localhost,
allow_custom_base_url=args.allow_custom_base_url,
scripts/yao_cli_parser.py
+17 -2
View File
@@ -356,12 +356,27 @@ def build_parser(command_handlers: dict[str, Callable[[argparse.Namespace], int]
output_execution_cmd.add_argument("--runner-command")
output_execution_cmd.add_argument(
"--provider-runner",
choices=["openai"],
choices=["openai", "deepseek"],
help="Use the bundled provider-backed runner instead of a custom runner command.",
)
output_execution_cmd.add_argument("--provider-model", help="Model for --provider-runner; otherwise use YAO_OUTPUT_EVAL_MODEL.")
output_execution_cmd.add_argument("--provider-base-url", help="Override provider endpoint for compatible APIs.")
output_execution_cmd.add_argument("--api-key-env", default="OPENAI_API_KEY")
output_execution_cmd.add_argument(
"--provider-api-format",
choices=["responses", "chat-completions"],
help="Provider API shape used by the bundled provider runner. Defaults are selected from --provider-runner.",
)
output_execution_cmd.add_argument(
"--provider-thinking",
choices=["enabled", "disabled"],
help="Optional thinking mode override for chat-completions providers that support it.",
)
output_execution_cmd.add_argument(
"--provider-temperature",
type=float,
help="Sampling temperature for the bundled provider runner. Defaults to 0 for reproducibility.",
)
output_execution_cmd.add_argument("--api-key-env")
output_execution_cmd.add_argument("--allow-insecure-localhost", action="store_true")
output_execution_cmd.add_argument("--allow-custom-base-url", action="store_true")
output_execution_cmd.add_argument("--timeout-seconds", type=float)
scripts/yao_cli_report_commands.py
+39 -3
View File
@@ -5,7 +5,7 @@ import json
from pathlib import Path
from yao_cli_config import baseline_compare_args, local_output_runner_command
from yao_cli_runtime import ROOT, run_adoption_drift_if_source_exists, run_script
from yao_cli_runtime import ROOT, allow_report_status, run_adoption_drift_if_source_exists, run_script
SCRIPT_INTERFACE = "internal-module"
@@ -17,6 +17,42 @@ def emit_result(result: dict) -> int:
return 0 if result["ok"] else 2
def load_json_file(path: Path) -> dict:
if not path.exists():
return {}
try:
return json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
return {}
def output_execution_has_provider_evidence() -> bool:
summary = load_json_file(ROOT / "reports" / "output_execution_runs.json").get("summary", {})
return (
int(summary.get("model_executed_count", 0) or 0) > 0
and int(summary.get("timing_observed_count", 0) or 0) > 0
and int(summary.get("token_observed_count", 0) or 0) > 0
)
def run_output_execution_refresh() -> dict:
command = f"run_output_execution.py --runner-command {local_output_runner_command()}"
if output_execution_has_provider_evidence():
return {
"command": f"{command} skipped: provider-backed evidence already present",
"returncode": 0,
"ok": True,
"stdout": "",
"stderr": "",
"payload": {
"ok": True,
"skipped": True,
"reason": "provider-backed output execution evidence already has observed model, timing, and token metadata",
},
}
return run_script("run_output_execution.py", ["--runner-command", local_output_runner_command()])
def resolved_skill_dir(args: argparse.Namespace) -> str:
return str(Path(args.skill_dir).resolve())
@@ -63,7 +99,7 @@ def command_report(args: argparse.Namespace) -> int:
run_script("render_system_model.py", [str(ROOT)]),
run_script("compile_skill.py", [str(ROOT)]),
run_script("run_output_eval.py", []),
run_script("run_output_execution.py", ["--runner-command", local_output_runner_command()]),
run_output_execution_refresh(),
run_script("prepare_output_review_kit.py", []),
run_script("adjudicate_output_review.py", []),
run_adoption_drift_if_source_exists(),
@@ -74,7 +110,7 @@ def command_report(args: argparse.Namespace) -> int:
run_script("render_world_class_evidence_ledger.py", [str(ROOT)]),
run_script("render_world_class_evidence_intake.py", [str(ROOT)]),
run_script("render_world_class_preflight.py", [str(ROOT)]),
run_script("render_world_class_submission_review.py", [str(ROOT)]),
allow_report_status(run_script("render_world_class_submission_review.py", [str(ROOT)])),
run_script("render_world_class_operator_runbook.py", [str(ROOT)]),
run_script("render_world_class_claim_guard.py", [str(ROOT)]),
run_script("render_daily_skillops_report.py", [str(ROOT)]),
scripts/yao_cli_runtime.py
+11
View File
@@ -45,6 +45,17 @@ def run_script(name: str, args: list[str], cwd: Path | None = None) -> dict:
}
def allow_report_status(result: dict, *, allowed_returncodes: set[int] | None = None) -> dict:
allowed = allowed_returncodes or {2}
if result.get("ok") or result.get("returncode") not in allowed or result.get("payload") is None:
return result
normalized = dict(result)
normalized["ok"] = True
normalized["soft_status_returncode"] = result.get("returncode")
normalized["soft_status_reason"] = "report generated with a non-passing evidence decision"
return normalized
def run_adoption_drift_if_source_exists(skill_dir: Path | None = None) -> dict:
target = (skill_dir or ROOT).resolve()
events_path = target / "reports" / "telemetry_events.jsonl"