Add SkillOps opportunity scoring
This commit is contained in:
yaojingang
@@ -2,7 +2,7 @@ PYTHON ?= python3
|
||||
LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill
|
||||
ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill
|
||||
|
||||
.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-kit-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-preflight-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check daily-skillops-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean
|
||||
.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-kit-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-preflight-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check skillops-opportunity-check daily-skillops-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean
|
||||
|
||||
eval:
|
||||
$(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt
|
||||
@@ -140,6 +140,9 @@ feedback-check:
|
||||
adaptation-safety-check:
|
||||
$(PYTHON) tests/verify_adaptation_safety.py
|
||||
|
||||
skillops-opportunity-check:
|
||||
$(PYTHON) tests/verify_skillops_opportunity.py
|
||||
|
||||
daily-skillops-check:
|
||||
$(PYTHON) tests/verify_daily_skillops.py
|
||||
|
||||
@@ -254,7 +257,7 @@ sync-local-install: package-check
|
||||
sync-active-install: package-check
|
||||
$(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)"
|
||||
|
||||
test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-preflight-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check daily-skillops-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check
|
||||
test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-preflight-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check skillops-opportunity-check daily-skillops-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check
|
||||
|
||||
ci-test:
|
||||
$(PYTHON) scripts/ci_test.py
|
||||
|
||||
@@ -45,6 +45,7 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl
|
||||
- a lightweight feedback log that does not require a full promotion cycle
|
||||
- a local-first metadata-only adoption and drift report that turns real usage signals into next iteration candidates, with optional `yao.py` CLI run capture, external client event emit hooks, hook recipes, and JSONL import that record command names and outcomes without arguments or raw content
|
||||
- an explicit-source adaptive proposal loop that summarizes redacted repeated user preferences and generates approval-gated adaptation proposals without scanning private logs or writing source files
|
||||
- a SkillOps opportunity scorer and decision policy that ranks redacted repeated signals, maps them to report-only, AGENTS update, existing-skill patch, or eval-addition actions, and keeps every durable write approval-gated
|
||||
- a Browser/Chrome Native Messaging telemetry host that can receive length-prefixed metadata-only client events and generate a local launcher plus manifest without storing raw content
|
||||
- a Skill Atlas drift layer that reads aggregate adoption reports and surfaces portfolio-level drift signals without packaging raw telemetry logs
|
||||
- a baseline compare report for with-skill vs baseline review
|
||||
@@ -445,6 +446,7 @@ Utility scripts that make the meta-skill operational:
|
||||
- `render_skill_os2_audit.py`: renders a requirement-by-requirement Skill OS 2.0 audit that separates landed local evidence from human-required and external-required gaps
|
||||
- `render_skill_os2_coverage.py`: maps the Skill OS 2.0 upgrade blueprint to local artifacts, commands, tests, and remaining evidence boundaries
|
||||
- `render_daily_skillops_report.py`: renders an explicit-source Daily SkillOps operations report that summarizes redacted user patterns, proposal-only adaptations, approval state, release evidence, and world-class evidence gaps without scanning private logs or applying patches
|
||||
- `skillops_opportunity.py`: scores redacted SkillOps opportunities and maps them to approval-gated action types such as report-only, AGENTS update, existing-skill patch, or eval addition
|
||||
- `render_world_class_evidence_plan.py`: renders executable evidence tasks for remaining world-class gaps without treating planned external work as completed evidence
|
||||
- `render_world_class_evidence_ledger.py`: renders a machine-checkable ledger for current world-class evidence acceptance, anti-overclaim guards, provenance requirements, and privacy contracts
|
||||
- `render_world_class_evidence_intake.py`: validates world-class external and human evidence packets against provenance, privacy, artifact, and anti-overclaim requirements before ledger review
|
||||
|
||||
@@ -33,7 +33,7 @@ Mode rules: [Method](references/skill-engineering-method.md), [Operating Modes](
|
||||
6. Add only folders and gates that earn their keep.
|
||||
7. Surface the top three next iteration directions.
|
||||
|
||||
Core playbooks: [Method](references/skill-engineering-method.md), [Intent](references/intent-dialogue.md), [Skill IR](references/skill-ir-method.md), [Output Eval](references/output-eval-method.md), [Registry](references/distribution-registry-method.md), [Review Studio](references/review-studio-method.md).
|
||||
Core playbooks: [Method](references/skill-engineering-method.md), [Intent](references/intent-dialogue.md), [Skill IR](references/skill-ir-method.md), [Output Eval](references/output-eval-method.md), [Registry](references/distribution-registry-method.md), [Review Studio](references/review-studio-method.md), [SkillOps Decision](references/skillops-decision-policy.md).
|
||||
|
||||
## Skill OS 2.0 Gates
|
||||
|
||||
@@ -57,4 +57,4 @@ Unless the user asks otherwise, produce a working skill directory with `SKILL.md
|
||||
|
||||
## Reference Map
|
||||
|
||||
Primary references: [Method](references/skill-engineering-method.md), [Artifact Design](references/artifact-design-doctrine.md), [Systems Thinking](references/systems-thinking-doctrine.md), [Governance](references/governance.md).
|
||||
Primary references: [Method](references/skill-engineering-method.md), [Artifact Design](references/artifact-design-doctrine.md), [Systems Thinking](references/systems-thinking-doctrine.md), [Governance](references/governance.md), [SkillOps Decision](references/skillops-decision-policy.md).
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
# SkillOps Decision Policy
|
||||
|
||||
Use this policy when turning explicit-source conversation evidence into SkillOps opportunities, proposals, or release work. The goal is to make repeated user signals actionable without letting automation write durable instructions, skills, scripts, or evals without review.
|
||||
|
||||
## Decision Order
|
||||
|
||||
1. Classify the signal as no action, report only, Memory, AGENTS.md, existing Skill patch, candidate Skill, script, eval, report, merge, split, or archive.
|
||||
2. Prefer the smallest durable surface that fixes the repeated friction.
|
||||
3. Require evidence for every write action.
|
||||
4. Require a proposal or approval ledger entry before any source-file write.
|
||||
5. Map every proposed change to at least one verification command.
|
||||
|
||||
## Score Bands
|
||||
|
||||
SkillOps opportunities use a `0-100` score. Scores are advisory and never bypass approval.
|
||||
|
||||
| Score | Decision |
|
||||
| ---: | --- |
|
||||
| `85-100` | Ready for approval review |
|
||||
| `70-84` | Proposal review |
|
||||
| `50-69` | Observe more evidence |
|
||||
| `0-49` | Report only or no action |
|
||||
|
||||
High-risk items stay proposal-only even when their score is high.
|
||||
|
||||
## Action Mapping
|
||||
|
||||
| Pattern | Default Action | Durable Surface |
|
||||
| --- | --- | --- |
|
||||
| `language_default` | Patch existing skill | Report template or artifact doctrine |
|
||||
| `report_ui` | Patch existing skill | Report renderer, artifact doctrine, visual test |
|
||||
| `approval_safety` | AGENTS update | Governance guidance or approval policy |
|
||||
| `delivery_format` | Patch existing skill | CLI output, README, generated summary copy |
|
||||
| `evidence_testing` | Add eval | Focused regression, report-quality, or release gate |
|
||||
| Unknown pattern | Report only | Manual review queue |
|
||||
|
||||
## Safety Rules
|
||||
|
||||
- Do not scan private logs implicitly; only use explicit user-supplied sources.
|
||||
- Do not store raw conversation content in reports; use redacted excerpts and aggregate counts.
|
||||
- Do not write source files from a daily report run.
|
||||
- Do not count SkillOps reports as public world-class evidence.
|
||||
- Do not treat planned work, draft submissions, or generated proposals as accepted evidence.
|
||||
|
||||
## Verification
|
||||
|
||||
Every implementation that changes this policy should run:
|
||||
|
||||
```bash
|
||||
python3 tests/verify_skillops_opportunity.py
|
||||
python3 tests/verify_daily_skillops.py
|
||||
python3 tests/verify_yao_cli.py
|
||||
```
|
||||
@@ -0,0 +1,112 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://yao.local/schemas/skillops-opportunity.schema.json",
|
||||
"title": "SkillOps Opportunity",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"opportunity_id",
|
||||
"pattern_id",
|
||||
"title",
|
||||
"opportunity_type",
|
||||
"action_type",
|
||||
"decision",
|
||||
"priority",
|
||||
"score",
|
||||
"risk_level",
|
||||
"requires_approval",
|
||||
"write_allowed_without_approval",
|
||||
"evidence_count",
|
||||
"target_files",
|
||||
"verification_commands"
|
||||
],
|
||||
"properties": {
|
||||
"opportunity_id": {
|
||||
"type": "string",
|
||||
"pattern": "^skillops-[a-f0-9]{10}$"
|
||||
},
|
||||
"proposal_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"pattern_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": {
|
||||
"type": "string"
|
||||
},
|
||||
"opportunity_type": {
|
||||
"type": "string"
|
||||
},
|
||||
"action_type": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"add_eval",
|
||||
"agents_update",
|
||||
"patch_existing_skill",
|
||||
"report_only"
|
||||
]
|
||||
},
|
||||
"decision": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"ready_for_approval_review",
|
||||
"proposal_review",
|
||||
"observe_more_evidence",
|
||||
"report_only",
|
||||
"no_action"
|
||||
]
|
||||
},
|
||||
"priority": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"high",
|
||||
"medium",
|
||||
"low"
|
||||
]
|
||||
},
|
||||
"score": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 100
|
||||
},
|
||||
"score_reasons": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"policy_reason": {
|
||||
"type": "string"
|
||||
},
|
||||
"risk_level": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"low",
|
||||
"medium",
|
||||
"high"
|
||||
]
|
||||
},
|
||||
"requires_approval": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"write_allowed_without_approval": {
|
||||
"const": false
|
||||
},
|
||||
"evidence_count": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"target_files": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"verification_commands": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": true
|
||||
}
|
||||
@@ -67,6 +67,7 @@ DEFAULT_TARGETS = [
|
||||
"evidence-consistency-check",
|
||||
"feedback-check",
|
||||
"adaptation-safety-check",
|
||||
"skillops-opportunity-check",
|
||||
"daily-skillops-check",
|
||||
"adoption-drift-check",
|
||||
"telemetry-import-check",
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import Any
|
||||
|
||||
from propose_adaptation import build_report as build_proposal_report
|
||||
from propose_adaptation import render_markdown as render_proposals_markdown
|
||||
from skillops_opportunity import build_opportunities, decision_policy_contract, summarize_opportunities
|
||||
from summarize_user_signals import build_report as build_pattern_report
|
||||
from summarize_user_signals import render_markdown as render_patterns_markdown
|
||||
|
||||
@@ -247,6 +248,9 @@ def build_report(
|
||||
proposal_contract = proposal_report.get("proposal_contract", {}) if isinstance(proposal_report.get("proposal_contract"), dict) else {}
|
||||
|
||||
proposal_rows = compact_proposals(proposal_report)
|
||||
pattern_rows = pattern_report.get("patterns", []) if isinstance(pattern_report.get("patterns"), list) else []
|
||||
opportunities = build_opportunities(pattern_rows, proposal_rows)
|
||||
opportunity_summary = summarize_opportunities(opportunities)
|
||||
failure_count = len(failures)
|
||||
summary = {
|
||||
"decision": "blocked" if failure_count else "proposal-review" if proposal_rows else "monitor",
|
||||
@@ -298,6 +302,9 @@ def build_report(
|
||||
"source": source_summary(pattern_report) if pattern_report else {},
|
||||
"patterns": pattern_report.get("patterns", []) if isinstance(pattern_report.get("patterns"), list) else [],
|
||||
"proposals": proposal_rows,
|
||||
"opportunity_summary": opportunity_summary,
|
||||
"opportunities": opportunities,
|
||||
"decision_policy": decision_policy_contract(),
|
||||
"approval": {
|
||||
"approval_count": summary["approval_count"],
|
||||
"pending_review_count": summary["pending_review_count"],
|
||||
@@ -374,6 +381,28 @@ def render_markdown(report: dict[str, Any]) -> str:
|
||||
lines.extend(["", "## Actions", ""])
|
||||
for action in report["actions"]:
|
||||
lines.append(f"- `{action['priority']}` {action['action']}")
|
||||
lines.extend(["", "## Opportunities", ""])
|
||||
opportunity_summary = report.get("opportunity_summary", {})
|
||||
lines.extend(
|
||||
[
|
||||
f"- count: `{opportunity_summary.get('opportunity_count', 0)}`",
|
||||
f"- top score: `{opportunity_summary.get('top_score', 0)}`",
|
||||
f"- ready for approval review: `{opportunity_summary.get('ready_for_approval_review_count', 0)}`",
|
||||
]
|
||||
)
|
||||
for opportunity in report.get("opportunities", []):
|
||||
lines.extend(
|
||||
[
|
||||
f"### {opportunity.get('title', '')}",
|
||||
"",
|
||||
f"- ID: `{opportunity.get('opportunity_id', '')}`",
|
||||
f"- Action: `{opportunity.get('action_type', '')}`",
|
||||
f"- Decision: `{opportunity.get('decision', '')}`",
|
||||
f"- Score: `{opportunity.get('score', 0)}`",
|
||||
f"- Risk: `{opportunity.get('risk_level', '')}`",
|
||||
f"- Policy: {opportunity.get('policy_reason', '')}",
|
||||
]
|
||||
)
|
||||
lines.extend(["", "## Patterns", ""])
|
||||
patterns = report.get("patterns", [])
|
||||
if not patterns:
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_daily_skillops_report.py to score SkillOps opportunities without writing files."
|
||||
|
||||
ACTION_POLICY: dict[str, dict[str, str]] = {
|
||||
"language_default": {
|
||||
"action_type": "patch_existing_skill",
|
||||
"opportunity_type": "report-experience",
|
||||
"reason": "Repeated language preferences can improve existing report templates after approval.",
|
||||
},
|
||||
"report_ui": {
|
||||
"action_type": "patch_existing_skill",
|
||||
"opportunity_type": "artifact-quality",
|
||||
"reason": "Repeated report layout feedback maps to existing renderer and design doctrine changes.",
|
||||
},
|
||||
"approval_safety": {
|
||||
"action_type": "agents_update",
|
||||
"opportunity_type": "governance",
|
||||
"reason": "Repeated privacy and approval signals should tighten durable operating guidance.",
|
||||
},
|
||||
"delivery_format": {
|
||||
"action_type": "patch_existing_skill",
|
||||
"opportunity_type": "artifact-discoverability",
|
||||
"reason": "Repeated delivery-path requests can improve CLI and README artifact discoverability.",
|
||||
},
|
||||
"evidence_testing": {
|
||||
"action_type": "add_eval",
|
||||
"opportunity_type": "quality-gate",
|
||||
"reason": "Repeated verification and evidence requests should become focused checks.",
|
||||
},
|
||||
}
|
||||
|
||||
RISK_PENALTY = {
|
||||
"low": 0,
|
||||
"medium": 8,
|
||||
"high": 18,
|
||||
}
|
||||
|
||||
|
||||
def _stable_id(pattern_id: str, target_files: list[str], title: str) -> str:
|
||||
digest = hashlib.sha1(f"{pattern_id}:{title}:{','.join(target_files)}".encode("utf-8")).hexdigest()
|
||||
return f"skillops-{digest[:10]}"
|
||||
|
||||
|
||||
def _as_int(value: Any, default: int = 0) -> int:
|
||||
if isinstance(value, bool):
|
||||
return default
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _risk_level(proposal: dict[str, Any]) -> str:
|
||||
risk = str(proposal.get("risk_level") or "medium").lower()
|
||||
return risk if risk in RISK_PENALTY else "medium"
|
||||
|
||||
|
||||
def _target_files(proposal: dict[str, Any]) -> list[str]:
|
||||
files = proposal.get("target_files")
|
||||
if not isinstance(files, list):
|
||||
return []
|
||||
return [str(item) for item in files if str(item).strip()]
|
||||
|
||||
|
||||
def _verification_commands(proposal: dict[str, Any]) -> list[str]:
|
||||
commands = proposal.get("verification_commands")
|
||||
if not isinstance(commands, list):
|
||||
return []
|
||||
return [str(item) for item in commands if str(item).strip()]
|
||||
|
||||
|
||||
def _score(pattern: dict[str, Any], proposal: dict[str, Any]) -> tuple[int, list[str]]:
|
||||
support = _as_int(pattern.get("support_count"), _as_int(proposal.get("support_count"), 0))
|
||||
confidence = pattern.get("confidence", 0.5)
|
||||
try:
|
||||
confidence_score = int(float(confidence) * 25)
|
||||
except (TypeError, ValueError):
|
||||
confidence_score = 12
|
||||
target_files = _target_files(proposal)
|
||||
verification_commands = _verification_commands(proposal)
|
||||
risk = _risk_level(proposal)
|
||||
reasons = [
|
||||
f"support_count={support}",
|
||||
f"confidence={confidence}",
|
||||
f"risk={risk}",
|
||||
]
|
||||
score = 28
|
||||
score += min(30, support * 10)
|
||||
score += max(0, min(25, confidence_score))
|
||||
if target_files:
|
||||
score += 8
|
||||
reasons.append("target_files_present")
|
||||
else:
|
||||
reasons.append("target_files_missing")
|
||||
if verification_commands:
|
||||
score += 7
|
||||
reasons.append("verification_present")
|
||||
else:
|
||||
reasons.append("verification_missing")
|
||||
score -= RISK_PENALTY[risk]
|
||||
if proposal.get("requires_approval") is True:
|
||||
reasons.append("approval_required")
|
||||
return max(0, min(100, score)), reasons
|
||||
|
||||
|
||||
def _decision(score: int, proposal: dict[str, Any]) -> str:
|
||||
if score >= 85:
|
||||
return "ready_for_approval_review"
|
||||
if score >= 70:
|
||||
return "proposal_review"
|
||||
if score >= 50:
|
||||
return "observe_more_evidence"
|
||||
if proposal.get("requires_approval") is True:
|
||||
return "report_only"
|
||||
return "no_action"
|
||||
|
||||
|
||||
def _priority(score: int, risk: str) -> str:
|
||||
if score >= 85 and risk == "low":
|
||||
return "high"
|
||||
if score >= 70:
|
||||
return "medium"
|
||||
return "low"
|
||||
|
||||
|
||||
def build_opportunities(patterns: list[dict[str, Any]], proposals: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
pattern_by_id = {str(item.get("pattern_id", "")): item for item in patterns if isinstance(item, dict)}
|
||||
opportunities: list[dict[str, Any]] = []
|
||||
for proposal in proposals:
|
||||
if not isinstance(proposal, dict):
|
||||
continue
|
||||
pattern_id = str(proposal.get("pattern_id") or "generic")
|
||||
pattern = pattern_by_id.get(pattern_id, {})
|
||||
policy = ACTION_POLICY.get(
|
||||
pattern_id,
|
||||
{
|
||||
"action_type": "report_only",
|
||||
"opportunity_type": "manual-review",
|
||||
"reason": "Unknown pattern family requires manual review before any durable change.",
|
||||
},
|
||||
)
|
||||
target_files = _target_files(proposal)
|
||||
verification_commands = _verification_commands(proposal)
|
||||
risk = _risk_level(proposal)
|
||||
score, score_reasons = _score(pattern, proposal)
|
||||
evidence_refs = proposal.get("evidence_refs")
|
||||
if not isinstance(evidence_refs, list):
|
||||
evidence_refs = []
|
||||
opportunities.append(
|
||||
{
|
||||
"opportunity_id": _stable_id(pattern_id, target_files, str(proposal.get("title") or "")),
|
||||
"proposal_id": proposal.get("proposal_id", ""),
|
||||
"pattern_id": pattern_id,
|
||||
"title": proposal.get("title", ""),
|
||||
"opportunity_type": policy["opportunity_type"],
|
||||
"action_type": policy["action_type"],
|
||||
"decision": _decision(score, proposal),
|
||||
"priority": _priority(score, risk),
|
||||
"score": score,
|
||||
"score_reasons": score_reasons,
|
||||
"policy_reason": policy["reason"],
|
||||
"risk_level": risk,
|
||||
"requires_approval": proposal.get("requires_approval") is True,
|
||||
"write_allowed_without_approval": False,
|
||||
"evidence_count": len(evidence_refs) or _as_int(pattern.get("support_count"), 0),
|
||||
"target_files": target_files,
|
||||
"verification_commands": verification_commands,
|
||||
}
|
||||
)
|
||||
opportunities.sort(key=lambda item: (-int(item["score"]), str(item["opportunity_id"])))
|
||||
return opportunities
|
||||
|
||||
|
||||
def summarize_opportunities(opportunities: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
by_action: dict[str, int] = {}
|
||||
by_decision: dict[str, int] = {}
|
||||
for item in opportunities:
|
||||
by_action[str(item.get("action_type", ""))] = by_action.get(str(item.get("action_type", "")), 0) + 1
|
||||
by_decision[str(item.get("decision", ""))] = by_decision.get(str(item.get("decision", "")), 0) + 1
|
||||
top_score = int(opportunities[0]["score"]) if opportunities else 0
|
||||
return {
|
||||
"opportunity_count": len(opportunities),
|
||||
"top_score": top_score,
|
||||
"ready_for_approval_review_count": by_decision.get("ready_for_approval_review", 0),
|
||||
"proposal_review_count": by_decision.get("proposal_review", 0),
|
||||
"observe_more_evidence_count": by_decision.get("observe_more_evidence", 0),
|
||||
"report_only_count": by_decision.get("report_only", 0),
|
||||
"action_type_counts": by_action,
|
||||
"decision_counts": by_decision,
|
||||
}
|
||||
|
||||
|
||||
def decision_policy_contract() -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"contract": "skillops-opportunity-scoring",
|
||||
"score_range": [0, 100],
|
||||
"score_bands": {
|
||||
"85-100": "ready_for_approval_review",
|
||||
"70-84": "proposal_review",
|
||||
"50-69": "observe_more_evidence",
|
||||
"0-49": "report_only_or_no_action",
|
||||
},
|
||||
"action_types": sorted({item["action_type"] for item in ACTION_POLICY.values()} | {"report_only"}),
|
||||
"writes_source_files": False,
|
||||
"auto_patch_enabled": False,
|
||||
"approval_required_for_writes": True,
|
||||
}
|
||||
@@ -150,6 +150,16 @@ def main() -> None:
|
||||
assert payload["summary"]["public_world_class_ready"] is False, payload
|
||||
assert payload["summary"]["release_lock_ready"] is True, payload
|
||||
assert payload["summary"]["evidence_consistency_ok"] is True, payload
|
||||
assert payload["opportunity_summary"]["opportunity_count"] >= 4, payload
|
||||
assert payload["opportunity_summary"]["top_score"] >= 70, payload
|
||||
assert payload["decision_policy"]["writes_source_files"] is False, payload
|
||||
assert payload["decision_policy"]["auto_patch_enabled"] is False, payload
|
||||
action_types = {item["action_type"] for item in payload["opportunities"]}
|
||||
assert {"patch_existing_skill", "agents_update"} <= action_types, payload["opportunities"]
|
||||
assert action_types <= set(payload["decision_policy"]["action_types"]), payload["opportunities"]
|
||||
assert all(item["write_allowed_without_approval"] is False for item in payload["opportunities"]), payload[
|
||||
"opportunities"
|
||||
]
|
||||
action_keys = {item["key"] for item in payload["actions"]}
|
||||
assert {"review-adaptation-proposals", "resolve-pending-approvals", "close-world-class-evidence"} <= action_keys, payload
|
||||
assert output_json.exists(), output_json
|
||||
@@ -168,6 +178,8 @@ def main() -> None:
|
||||
assert "Daily SkillOps Report" in markdown, markdown
|
||||
assert "does not scan private logs" in markdown, markdown
|
||||
assert "daily_report_counts_as_world_class_evidence: `false`" in markdown, markdown
|
||||
assert "## Opportunities" in markdown, markdown
|
||||
assert "ready for approval review" in markdown, markdown
|
||||
|
||||
cli_dir = TMP / "cli-daily-skillops-demo"
|
||||
cli_dir.mkdir(parents=True)
|
||||
@@ -227,6 +239,7 @@ def main() -> None:
|
||||
assert no_refresh_payload["summary"]["source_supplied"] is True, no_refresh_payload
|
||||
assert no_refresh_payload["summary"]["pattern_count"] >= 4, no_refresh_payload
|
||||
assert no_refresh_payload["summary"]["proposal_count"] >= 4, no_refresh_payload
|
||||
assert no_refresh_payload["opportunity_summary"]["opportunity_count"] >= 4, no_refresh_payload
|
||||
assert not (no_refresh_dir / "reports" / "adaptation_proposals.json").exists(), no_refresh_dir
|
||||
|
||||
print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
|
||||
from skillops_opportunity import build_opportunities, decision_policy_contract, summarize_opportunities
|
||||
|
||||
|
||||
def main() -> None:
|
||||
patterns = [
|
||||
{
|
||||
"pattern_id": "evidence_testing",
|
||||
"support_count": 4,
|
||||
"confidence": 0.95,
|
||||
"evidence": [{"record_id": "a"}, {"record_id": "b"}, {"record_id": "c"}],
|
||||
},
|
||||
{
|
||||
"pattern_id": "approval_safety",
|
||||
"support_count": 2,
|
||||
"confidence": 0.79,
|
||||
"evidence": [{"record_id": "d"}, {"record_id": "e"}],
|
||||
},
|
||||
{
|
||||
"pattern_id": "unknown_manual",
|
||||
"support_count": 1,
|
||||
"confidence": 0.55,
|
||||
"evidence": [{"record_id": "f"}],
|
||||
},
|
||||
]
|
||||
proposals = [
|
||||
{
|
||||
"proposal_id": "adapt-high",
|
||||
"pattern_id": "evidence_testing",
|
||||
"title": "Attach tests and evidence refresh to each upgrade",
|
||||
"risk_level": "medium",
|
||||
"requires_approval": True,
|
||||
"target_files": ["tests/verify_adaptation_safety.py"],
|
||||
"verification_commands": ["python3 tests/verify_adaptation_safety.py"],
|
||||
"evidence_refs": [{"record_id": "a"}, {"record_id": "b"}, {"record_id": "c"}],
|
||||
},
|
||||
{
|
||||
"proposal_id": "adapt-approval",
|
||||
"pattern_id": "approval_safety",
|
||||
"title": "Keep adaptive iteration approval-gated",
|
||||
"risk_level": "low",
|
||||
"requires_approval": True,
|
||||
"target_files": ["AGENTS.md"],
|
||||
"verification_commands": ["python3 tests/verify_adaptation_safety.py"],
|
||||
"evidence_refs": [{"record_id": "d"}, {"record_id": "e"}],
|
||||
},
|
||||
{
|
||||
"proposal_id": "adapt-unknown",
|
||||
"pattern_id": "unknown_manual",
|
||||
"title": "Review repeated preference pattern",
|
||||
"risk_level": "high",
|
||||
"requires_approval": True,
|
||||
"target_files": [],
|
||||
"verification_commands": [],
|
||||
"evidence_refs": [{"record_id": "f"}],
|
||||
},
|
||||
]
|
||||
opportunities = build_opportunities(patterns, proposals)
|
||||
assert len(opportunities) == 3, opportunities
|
||||
assert opportunities[0]["score"] >= opportunities[1]["score"] >= opportunities[2]["score"], opportunities
|
||||
by_pattern = {item["pattern_id"]: item for item in opportunities}
|
||||
assert by_pattern["evidence_testing"]["action_type"] == "add_eval", by_pattern["evidence_testing"]
|
||||
assert by_pattern["evidence_testing"]["decision"] in {"ready_for_approval_review", "proposal_review"}, by_pattern[
|
||||
"evidence_testing"
|
||||
]
|
||||
assert by_pattern["approval_safety"]["action_type"] == "agents_update", by_pattern["approval_safety"]
|
||||
assert by_pattern["unknown_manual"]["action_type"] == "report_only", by_pattern["unknown_manual"]
|
||||
assert by_pattern["unknown_manual"]["score"] < by_pattern["evidence_testing"]["score"], by_pattern
|
||||
assert all(item["write_allowed_without_approval"] is False for item in opportunities), opportunities
|
||||
assert all(0 <= item["score"] <= 100 for item in opportunities), opportunities
|
||||
summary = summarize_opportunities(opportunities)
|
||||
assert summary["opportunity_count"] == 3, summary
|
||||
assert summary["action_type_counts"]["add_eval"] == 1, summary
|
||||
assert summary["action_type_counts"]["agents_update"] == 1, summary
|
||||
assert summary["action_type_counts"]["report_only"] == 1, summary
|
||||
assert summary["top_score"] == opportunities[0]["score"], summary
|
||||
contract = decision_policy_contract()
|
||||
assert contract["writes_source_files"] is False, contract
|
||||
assert contract["auto_patch_enabled"] is False, contract
|
||||
assert contract["approval_required_for_writes"] is True, contract
|
||||
assert contract["score_range"] == [0, 100], contract
|
||||
schema = json.loads((ROOT / "schemas" / "skillops-opportunity.schema.json").read_text(encoding="utf-8"))
|
||||
assert "score" in schema["properties"], schema
|
||||
print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user