wehub/microsoft--skillopt
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:24:16 +08:00
commit 4b92209caa
317 changed files with 67848 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/__init__.py
View File
scripts/eval_only.py
+501
View File
@@ -0,0 +1,501 @@
#!/usr/bin/env python3
"""SkillOpt eval-only: run a single skill on a dataset without training.
Usage
-----
python scripts/eval_only.py \
--config configs/spreadsheetbench/default.yaml \
--skill skillopt/envs/spreadsheetbench/skills/initial.md \
--split_dir /path/to/split \
--out_root outputs/eval_skill0
All YAML keys can be overridden from the CLI, same as train.py.
"""
from __future__ import annotations
import argparse
import datetime
import json
import os
import sys
_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
_PROJECT_ROOT = os.path.dirname(_SCRIPT_DIR)
if _PROJECT_ROOT not in sys.path:
sys.path.insert(0, _PROJECT_ROOT)
from skillopt.model import (
configure_azure_openai,
configure_claude_code_exec,
configure_codex_exec,
configure_qwen_chat,
configure_minimax_chat,
set_reasoning_effort,
set_target_backend,
set_target_deployment,
set_optimizer_backend,
set_optimizer_deployment,
)
from skillopt.model.common import default_model_for_backend, normalize_backend_name
_OPENAI_DEFAULT_MODEL_SENTINELS = {"gpt-5.4", "gpt-5.5"}
from skillopt.utils import compute_score
# ── Reuse registry from train.py ───────────────────────────────────────────
_ENV_REGISTRY: dict[str, type] = {}
def _register_builtins() -> None:
try:
from skillopt.envs.alfworld.adapter import ALFWorldAdapter
_ENV_REGISTRY["alfworld"] = ALFWorldAdapter
except ImportError:
pass
try:
from skillopt.envs.searchqa.adapter import SearchQAAdapter
_ENV_REGISTRY["searchqa"] = SearchQAAdapter
except ImportError:
pass
try:
from skillopt.envs.livemathematicianbench.adapter import LiveMathematicianBenchAdapter
_ENV_REGISTRY["livemathematicianbench"] = LiveMathematicianBenchAdapter
except ImportError:
pass
try:
from skillopt.envs.babyvision.adapter import BabyVisionAdapter
_ENV_REGISTRY["babyvision"] = BabyVisionAdapter
except ImportError:
pass
try:
from skillopt.envs.spreadsheetbench.adapter import SpreadsheetBenchAdapter
_ENV_REGISTRY["spreadsheetbench"] = SpreadsheetBenchAdapter
except ImportError:
pass
try:
from skillopt.envs.mmrb.adapter import MMRBAdapter
_ENV_REGISTRY["mmrb"] = MMRBAdapter
except ImportError:
pass
try:
from skillopt.envs.docvqa.adapter import DocVQAAdapter
_ENV_REGISTRY["docvqa"] = DocVQAAdapter
except ImportError:
pass
try:
from skillopt.envs.mathverse.adapter import MathVerseAdapter
_ENV_REGISTRY["mathverse"] = MathVerseAdapter
except ImportError:
pass
try:
from skillopt.envs.officeqa.adapter import OfficeQAAdapter
_ENV_REGISTRY["officeqa"] = OfficeQAAdapter
except ImportError:
pass
try:
from skillopt.envs.sealqa.adapter import SealQAAdapter
_ENV_REGISTRY["sealqa"] = SealQAAdapter
except ImportError:
pass
try:
from skillopt.envs.swebench.adapter import SWEBenchAdapter
_ENV_REGISTRY["swebench"] = SWEBenchAdapter
except ImportError:
pass
def get_adapter(cfg: dict):
_register_builtins()
env_name = cfg.get("env", "alfworld")
if env_name not in _ENV_REGISTRY:
raise ValueError(
f"Unknown environment '{env_name}'. "
f"Available: {list(_ENV_REGISTRY.keys())}"
)
adapter_cls = _ENV_REGISTRY[env_name]
import inspect
sig = inspect.signature(adapter_cls.__init__)
accepted = set(sig.parameters.keys()) - {"self"}
adapter_kwargs = {k: cfg[k] for k in accepted if k in cfg}
return adapter_cls(**adapter_kwargs)
# ── CLI ────────────────────────────────────────────────────────────────────
_BOOL = lambda x: str(x).lower() in ("true", "1", "yes") # noqa: E731
def parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description="SkillOpt eval-only")
p.add_argument("--config", type=str, required=True)
p.add_argument("--skill", type=str, required=True,
help="Path to skill .md file to evaluate")
p.add_argument("--split", type=str, default="all",
help="Which split to eval: train/valid_seen/valid_unseen/all (default: all)")
p.add_argument("--cfg-options", nargs="+", default=[],
help="Override config: section.key=value")
# Legacy flat overrides
p.add_argument("--env", type=str)
p.add_argument("--backend", type=str,
choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "minimax", "minimax_chat"])
p.add_argument("--optimizer_model", type=str)
p.add_argument("--target_model", type=str)
p.add_argument("--optimizer_backend", type=str)
p.add_argument("--target_backend", type=str)
p.add_argument("--reasoning_effort", type=str,
choices=["", "low", "medium", "high", "xhigh", "max"])
p.add_argument("--azure_endpoint", type=str)
p.add_argument("--azure_api_version", type=str)
p.add_argument("--azure_api_key", type=str)
p.add_argument("--azure_openai_endpoint", type=str)
p.add_argument("--azure_openai_api_version", type=str)
p.add_argument("--azure_openai_api_key", type=str)
p.add_argument("--azure_openai_auth_mode", type=str)
p.add_argument("--azure_openai_ad_scope", type=str)
p.add_argument("--azure_openai_managed_identity_client_id", type=str)
p.add_argument("--optimizer_azure_openai_endpoint", type=str)
p.add_argument("--optimizer_azure_openai_api_version", type=str)
p.add_argument("--optimizer_azure_openai_api_key", type=str)
p.add_argument("--optimizer_azure_openai_auth_mode", type=str)
p.add_argument("--optimizer_azure_openai_ad_scope", type=str)
p.add_argument("--optimizer_azure_openai_managed_identity_client_id", type=str)
p.add_argument("--target_azure_openai_endpoint", type=str)
p.add_argument("--target_azure_openai_api_version", type=str)
p.add_argument("--target_azure_openai_api_key", type=str)
p.add_argument("--target_azure_openai_auth_mode", type=str)
p.add_argument("--target_azure_openai_ad_scope", type=str)
p.add_argument("--target_azure_openai_managed_identity_client_id", type=str)
p.add_argument("--codex_exec_path", type=str)
p.add_argument("--codex_exec_sandbox", type=str)
p.add_argument("--codex_exec_profile", type=str)
p.add_argument("--codex_exec_full_auto", type=_BOOL)
p.add_argument("--codex_exec_reasoning_effort", type=str)
p.add_argument("--codex_exec_use_sdk", type=str)
p.add_argument("--codex_exec_network_access", type=_BOOL)
p.add_argument("--codex_exec_web_search", type=_BOOL)
p.add_argument("--codex_exec_approval_policy", type=str)
p.add_argument("--claude_code_exec_path", type=str)
p.add_argument("--claude_code_exec_profile", type=str)
p.add_argument("--claude_code_exec_use_sdk", type=str)
p.add_argument("--claude_code_exec_effort", type=str)
p.add_argument("--claude_code_exec_max_thinking_tokens", type=int)
p.add_argument("--minimax_base_url", type=str)
p.add_argument("--minimax_api_key", type=str)
p.add_argument("--minimax_model", type=str)
p.add_argument("--minimax_temperature", type=float)
p.add_argument("--minimax_max_tokens", type=int)
p.add_argument("--minimax_enable_thinking", type=_BOOL)
p.add_argument("--out_root", type=str)
p.add_argument("--data_path", type=str)
p.add_argument("--split_mode", type=str,
choices=["ratio", "split_dir"])
p.add_argument("--split_ratio", type=str)
p.add_argument("--split_seed", type=int)
p.add_argument("--split_dir", type=str)
p.add_argument("--split_output_dir", type=str)
p.add_argument("--data_root", type=str)
p.add_argument("--max_turns", type=int)
p.add_argument("--workers", type=int)
p.add_argument("--max_api_workers", type=int)
p.add_argument("--seed", type=int)
p.add_argument("--test_env_num", type=int)
p.add_argument("--mode", type=str,
help="SpreadsheetBench: single/multi/react (default comes from config)")
return p.parse_args()
def main() -> None:
args = parse_args()
from skillopt.config import load_config as _load, flatten_config, is_structured
cfg = _load(args.config, overrides=args.cfg_options)
structured = is_structured(cfg)
# Apply legacy --key value overrides
cli = {k: v for k, v in vars(args).items()
if v is not None and k not in ("config", "skill", "split", "cfg_options")}
if cli:
if structured:
from skillopt.config import apply_overrides
_MAP = {
"backend": "model.backend",
"optimizer_model": "model.optimizer",
"target_model": "model.target",
"optimizer_backend": "model.optimizer_backend",
"target_backend": "model.target_backend",
"reasoning_effort": "model.reasoning_effort",
"azure_endpoint": "model.azure_endpoint",
"azure_api_version": "model.azure_api_version",
"azure_api_key": "model.azure_api_key",
"azure_openai_endpoint": "model.azure_openai_endpoint",
"azure_openai_api_version": "model.azure_openai_api_version",
"azure_openai_api_key": "model.azure_openai_api_key",
"azure_openai_auth_mode": "model.azure_openai_auth_mode",
"azure_openai_ad_scope": "model.azure_openai_ad_scope",
"azure_openai_managed_identity_client_id": "model.azure_openai_managed_identity_client_id",
"optimizer_azure_openai_endpoint": "model.optimizer_azure_openai_endpoint",
"optimizer_azure_openai_api_version": "model.optimizer_azure_openai_api_version",
"optimizer_azure_openai_api_key": "model.optimizer_azure_openai_api_key",
"optimizer_azure_openai_auth_mode": "model.optimizer_azure_openai_auth_mode",
"optimizer_azure_openai_ad_scope": "model.optimizer_azure_openai_ad_scope",
"optimizer_azure_openai_managed_identity_client_id": "model.optimizer_azure_openai_managed_identity_client_id",
"target_azure_openai_endpoint": "model.target_azure_openai_endpoint",
"target_azure_openai_api_version": "model.target_azure_openai_api_version",
"target_azure_openai_api_key": "model.target_azure_openai_api_key",
"target_azure_openai_auth_mode": "model.target_azure_openai_auth_mode",
"target_azure_openai_ad_scope": "model.target_azure_openai_ad_scope",
"target_azure_openai_managed_identity_client_id": "model.target_azure_openai_managed_identity_client_id",
"codex_exec_path": "model.codex_exec_path",
"codex_exec_sandbox": "model.codex_exec_sandbox",
"codex_exec_profile": "model.codex_exec_profile",
"codex_exec_full_auto": "model.codex_exec_full_auto",
"codex_exec_reasoning_effort": "model.codex_exec_reasoning_effort",
"codex_exec_use_sdk": "model.codex_exec_use_sdk",
"codex_exec_network_access": "model.codex_exec_network_access",
"codex_exec_web_search": "model.codex_exec_web_search",
"codex_exec_approval_policy": "model.codex_exec_approval_policy",
"claude_code_exec_path": "model.claude_code_exec_path",
"claude_code_exec_profile": "model.claude_code_exec_profile",
"claude_code_exec_use_sdk": "model.claude_code_exec_use_sdk",
"claude_code_exec_effort": "model.claude_code_exec_effort",
"claude_code_exec_max_thinking_tokens": "model.claude_code_exec_max_thinking_tokens",
"minimax_base_url": "model.minimax_base_url",
"minimax_api_key": "model.minimax_api_key",
"minimax_model": "model.minimax_model",
"minimax_temperature": "model.minimax_temperature",
"minimax_max_tokens": "model.minimax_max_tokens",
"minimax_enable_thinking": "model.minimax_enable_thinking",
"seed": "train.seed",
"test_env_num": "evaluation.test_env_num",
"env": "env.name",
"out_root": "env.out_root",
}
mapped = []
for k, v in cli.items():
dotted = _MAP.get(k)
if dotted:
mapped.append(f"{dotted}={v}")
else:
mapped.append(f"env.{k}={v}")
apply_overrides(cfg, mapped)
else:
cfg.update(cli)
cfg = flatten_config(cfg) if structured else cfg
for new_key, old_key in (
("azure_openai_endpoint", "azure_endpoint"),
("azure_openai_api_version", "azure_api_version"),
("azure_openai_api_key", "azure_api_key"),
):
if cfg.get(new_key) in (None, "") and cfg.get(old_key) not in (None, ""):
cfg[new_key] = cfg[old_key]
explicit_backend = getattr(args, "backend", None)
if explicit_backend is None:
for option in args.cfg_options or []:
key = str(option).split("=", 1)[0].strip()
if key == "model.backend":
explicit_backend = str(option).split("=", 1)[1].strip()
break
backend = normalize_backend_name(cfg.get("model_backend") or cfg.get("target_backend") or "azure_openai")
def _has_model_override(dotted_key: str, legacy_key: str) -> bool:
if getattr(args, legacy_key, None) is not None:
return True
for option in args.cfg_options or []:
key = str(option).split("=", 1)[0].strip()
if key == dotted_key:
return True
return False
if explicit_backend is not None:
backend = normalize_backend_name(explicit_backend)
cfg["model_backend"] = backend
if backend in {"claude", "claude_chat"}:
cfg.setdefault("optimizer_backend", "claude_chat")
cfg.setdefault("target_backend", "claude_chat")
elif backend in {"codex", "codex_exec"}:
cfg.setdefault("optimizer_backend", "openai_chat")
cfg.setdefault("target_backend", "codex_exec")
elif backend == "claude_code_exec":
cfg.setdefault("optimizer_backend", "openai_chat")
cfg.setdefault("target_backend", "claude_code_exec")
elif backend in {"minimax", "minimax_chat"}:
cfg.setdefault("optimizer_backend", "openai_chat")
cfg.setdefault("target_backend", "minimax_chat")
else:
cfg.setdefault("optimizer_backend", "openai_chat")
cfg.setdefault("target_backend", "openai_chat")
else:
cfg.setdefault("optimizer_backend", "openai_chat")
cfg.setdefault("target_backend", "openai_chat")
if cfg.get("optimizer_backend") == "claude_chat":
if (
str(cfg.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
and not _has_model_override("model.optimizer", "optimizer_model")
):
cfg["optimizer_model"] = default_model_for_backend("claude_chat")
if cfg.get("target_backend") == "claude_chat":
if (
str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
and not _has_model_override("model.target", "target_model")
):
cfg["target_model"] = default_model_for_backend("claude_chat")
if cfg.get("target_backend") == "claude_code_exec":
if (
str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
and not _has_model_override("model.target", "target_model")
):
cfg["target_model"] = default_model_for_backend("claude_chat")
if cfg.get("target_backend") == "minimax_chat":
if (
str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
and not _has_model_override("model.target", "target_model")
):
cfg["target_model"] = (
cfg.get("minimax_model")
or default_model_for_backend("minimax_chat")
)
if not cfg.get("out_root"):
env = cfg.get("env", "unknown")
model = cfg.get("target_model", "unknown").replace("/", "-")
ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
cfg["out_root"] = os.path.join("outputs", f"eval_{env}_{model}_{ts}")
cfg["out_root"] = os.path.abspath(cfg["out_root"])
out_root = cfg["out_root"]
os.makedirs(out_root, exist_ok=True)
# Load skill
skill_path = os.path.abspath(args.skill)
with open(skill_path) as f:
skill_content = f.read()
print(f" [skill] {skill_path} ({len(skill_content)} chars)")
# Configure models
configure_azure_openai(
endpoint=(cfg.get("azure_openai_endpoint") or cfg.get("azure_endpoint") or None),
api_version=(cfg.get("azure_openai_api_version") or cfg.get("azure_api_version") or None),
api_key=(cfg.get("azure_openai_api_key") or cfg.get("azure_api_key") or None),
auth_mode=cfg.get("azure_openai_auth_mode") or None,
ad_scope=cfg.get("azure_openai_ad_scope") or None,
managed_identity_client_id=cfg.get("azure_openai_managed_identity_client_id") or None,
optimizer_endpoint=cfg.get("optimizer_azure_openai_endpoint") or None,
optimizer_api_version=cfg.get("optimizer_azure_openai_api_version") or None,
optimizer_api_key=cfg.get("optimizer_azure_openai_api_key") or None,
optimizer_auth_mode=cfg.get("optimizer_azure_openai_auth_mode") or None,
optimizer_ad_scope=cfg.get("optimizer_azure_openai_ad_scope") or None,
optimizer_managed_identity_client_id=(
cfg.get("optimizer_azure_openai_managed_identity_client_id") or None
),
target_endpoint=cfg.get("target_azure_openai_endpoint") or None,
target_api_version=cfg.get("target_azure_openai_api_version") or None,
target_api_key=cfg.get("target_azure_openai_api_key") or None,
target_auth_mode=cfg.get("target_azure_openai_auth_mode") or None,
target_ad_scope=cfg.get("target_azure_openai_ad_scope") or None,
target_managed_identity_client_id=(
cfg.get("target_azure_openai_managed_identity_client_id") or None
),
)
set_optimizer_backend(cfg.get("optimizer_backend", "openai_chat"))
set_target_backend(cfg.get("target_backend", "openai_chat"))
set_optimizer_deployment(cfg.get("optimizer_model", default_model_for_backend(backend)))
set_target_deployment(cfg.get("target_model", default_model_for_backend(backend)))
configure_codex_exec(
path=cfg.get("codex_exec_path", "codex"),
sandbox=cfg.get("codex_exec_sandbox", "workspace-write"),
profile=cfg.get("codex_exec_profile", ""),
full_auto=cfg.get("codex_exec_full_auto", False),
reasoning_effort=cfg.get("codex_exec_reasoning_effort", "none"),
use_sdk=cfg.get("codex_exec_use_sdk", None),
network_access=cfg.get("codex_exec_network_access", False),
web_search=cfg.get("codex_exec_web_search", False),
approval_policy=cfg.get("codex_exec_approval_policy", "never"),
)
configure_claude_code_exec(
path=cfg.get("claude_code_exec_path", "claude"),
profile=cfg.get("claude_code_exec_profile", ""),
use_sdk=cfg.get("claude_code_exec_use_sdk", None),
effort=cfg.get("claude_code_exec_effort", cfg.get("reasoning_effort", "medium")),
max_thinking_tokens=cfg.get("claude_code_exec_max_thinking_tokens", 16384),
)
configure_qwen_chat(
base_url=cfg.get("qwen_chat_base_url") or None,
api_key=cfg.get("qwen_chat_api_key") or None,
temperature=cfg.get("qwen_chat_temperature"),
timeout_seconds=cfg.get("qwen_chat_timeout_seconds"),
max_tokens=cfg.get("qwen_chat_max_tokens"),
enable_thinking=cfg.get("qwen_chat_enable_thinking"),
target_base_url=cfg.get("target_qwen_chat_base_url") or None,
target_api_key=cfg.get("target_qwen_chat_api_key") or None,
target_temperature=cfg.get("target_qwen_chat_temperature"),
target_timeout_seconds=cfg.get("target_qwen_chat_timeout_seconds"),
target_max_tokens=cfg.get("target_qwen_chat_max_tokens"),
target_enable_thinking=cfg.get("target_qwen_chat_enable_thinking"),
)
configure_minimax_chat(
base_url=cfg.get("minimax_base_url") or None,
api_key=cfg.get("minimax_api_key") or None,
temperature=cfg.get("minimax_temperature"),
max_tokens=cfg.get("minimax_max_tokens"),
enable_thinking=cfg.get("minimax_enable_thinking"),
)
minimax_model_cfg = cfg.get("minimax_model")
if minimax_model_cfg and cfg.get("target_backend") == "minimax_chat":
set_target_deployment(str(minimax_model_cfg))
set_reasoning_effort(cfg.get("reasoning_effort", "") or None)
# Build adapter
adapter = get_adapter(cfg)
adapter.setup(cfg)
seed = cfg.get("seed", 42)
split = args.split or "all"
if split == "all":
items = (
adapter.build_eval_env(0, "train", seed)
+ adapter.build_eval_env(0, "valid_seen", seed)
+ adapter.build_eval_env(0, "valid_unseen", seed)
)
else:
env_num = cfg.get("test_env_num", 0)
items = adapter.build_eval_env(env_num, split, seed)
print(f"\n [eval] split={split} items={len(items)}")
print(f" [eval] out_root={out_root}")
print(f"{'='*60}")
# Run rollout
results = adapter.rollout(items, skill_content, out_root)
# Score
hard, soft = compute_score(results)
print(f"\n{'='*60}")
print(f" Results: hard={hard:.4f} soft={soft:.4f} (n={len(results)})")
print(f"{'='*60}")
# Save summary
summary = {
"skill": skill_path,
"split": split,
"n_items": len(results),
"hard": hard,
"soft": soft,
}
with open(os.path.join(out_root, "eval_summary.json"), "w") as f:
json.dump(summary, f, indent=2, ensure_ascii=False)
print(f" Saved to: {out_root}")
if __name__ == "__main__":
main()
scripts/materialize_searchqa.py
+148
View File
@@ -0,0 +1,148 @@
"""Materialize runnable SearchQA splits from the released ID manifest."""
from __future__ import annotations
import argparse
import json
from collections.abc import Iterable, Mapping
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parent.parent
SPLITS = ("train", "val", "test")
REQUIRED_FIELDS = ("question", "context", "answers")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--manifest-dir",
type=Path,
default=PROJECT_ROOT / "data" / "searchqa_id_split",
help="Directory containing train/val/test ID manifests.",
)
parser.add_argument(
"--output-dir",
type=Path,
default=PROJECT_ROOT / "data" / "searchqa_split",
help="Directory to write runnable train/val/test splits.",
)
parser.add_argument(
"--dataset",
default="lucadiliello/searchqa",
help="Hugging Face dataset repository to load.",
)
return parser.parse_args()
def load_manifest_ids(manifest_dir: Path) -> dict[str, list[str]]:
split_ids = {}
for split in SPLITS:
path = manifest_dir / split / "items.json"
with path.open(encoding="utf-8") as file:
items = json.load(file)
split_ids[split] = [str(item["id"]) for item in items]
return split_ids
def _iter_dataset_rows(dataset: Mapping[str, Iterable[dict]]) -> Iterable[dict]:
for source_split in dataset.values():
yield from source_split
def _normalize_row(row: dict) -> dict:
try:
key = str(row["key"])
except KeyError as exc:
raise ValueError("SearchQA source row is missing required field: key") from exc
missing = [field for field in REQUIRED_FIELDS if field not in row]
if missing:
raise ValueError(f"SearchQA source row {key!r} is missing required fields: {', '.join(missing)}")
return {
"id": key,
"question": row["question"],
"context": row["context"],
"answers": row["answers"],
}
def materialize_searchqa_splits(
manifest_dir: Path,
output_dir: Path,
dataset: Mapping[str, Iterable[dict]],
*,
dataset_name: str,
) -> dict[str, int]:
"""Write runnable SearchQA train/val/test splits from a source dataset."""
manifest_dir = manifest_dir.resolve()
output_dir = output_dir.resolve()
split_ids = load_manifest_ids(manifest_dir)
wanted_ids = {item_id for ids in split_ids.values() for item_id in ids}
selected: dict[str, dict] = {}
duplicate_ids: set[str] = set()
for row in _iter_dataset_rows(dataset):
key = str(row.get("key", ""))
if key not in wanted_ids:
continue
if key in selected:
duplicate_ids.add(key)
continue
selected[key] = _normalize_row(row)
if duplicate_ids:
preview = ", ".join(sorted(duplicate_ids)[:5])
raise ValueError(f"SearchQA source dataset contains duplicate manifest IDs. First IDs: {preview}")
missing = sorted(wanted_ids - selected.keys())
if missing:
preview = ", ".join(missing[:5])
raise RuntimeError(f"SearchQA source dataset is missing {len(missing)} manifest IDs. First IDs: {preview}")
counts = {}
for split, ids in split_ids.items():
items = [selected[item_id] for item_id in ids]
split_dir = output_dir / split
split_dir.mkdir(parents=True, exist_ok=True)
with (split_dir / "items.json").open("w", encoding="utf-8") as file:
json.dump(items, file, ensure_ascii=False, indent=2)
counts[split] = len(items)
manifest = {
"source_manifest_dir": str(manifest_dir),
"source_dataset": dataset_name,
"counts": counts,
"item_fields": ["id", *REQUIRED_FIELDS],
}
with (output_dir / "split_manifest.json").open("w", encoding="utf-8") as file:
json.dump(manifest, file, ensure_ascii=False, indent=2)
return counts
def main() -> None:
args = parse_args()
try:
from datasets import load_dataset
except ImportError as exc:
raise SystemExit(
"Missing dependency 'datasets'. Install it with:\n"
" python -m pip install 'skillopt[searchqa]'\n"
"or:\n"
" python -m pip install datasets"
) from exc
print(f"Loading {args.dataset}...")
dataset = load_dataset(args.dataset)
counts = materialize_searchqa_splits(
args.manifest_dir,
args.output_dir,
dataset,
dataset_name=args.dataset,
)
print(f"Wrote SearchQA splits to {args.output_dir.resolve()}: {counts}")
if __name__ == "__main__":
main()
scripts/run_alfworld.sh
Executable
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env bash
# ──────────────────────────────────────────────────────────────────────────────
# SkillOpt — ALFWorld training launch script
#
# Prerequisites:
# pip install -e ".[alfworld]"
# pip install alfworld[full] && alfworld-download
#
# Usage:
# bash scripts/run_alfworld.sh
# bash scripts/run_alfworld.sh --num_epochs 2 --edit_budget 6
# bash scripts/run_alfworld.sh --split_dir /path/to/alfworld_split
# ──────────────────────────────────────────────────────────────────────────────
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")"
export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}"
# ALFWorld data — uses ~/.cache/alfworld by default
export ALFWORLD_DATA="${ALFWORLD_DATA:-${HOME}/.cache/alfworld}"
if [ ! -d "${ALFWORLD_DATA}/json_2.1.1" ]; then
echo "ERROR: ALFWorld data not found at ${ALFWORLD_DATA}/json_2.1.1"
echo ""
echo "To download ALFWorld data, run:"
echo " pip install alfworld[full]"
echo " alfworld-download"
echo ""
echo "Or set ALFWORLD_DATA to the directory containing json_2.1.1/"
exit 1
fi
OPTIMIZER_MODEL="${OPTIMIZER_MODEL:-gpt-5.5}"
TARGET_MODEL="${TARGET_MODEL:-gpt-5.5}"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
DEFAULT_OUT_ROOT="${PROJECT_ROOT}/outputs/skillopt_alfworld_${TARGET_MODEL}_${TIMESTAMP}"
echo "============================================================"
echo " SkillOpt — ALFWorld Training"
echo "============================================================"
echo " Optimizer: ${OPTIMIZER_MODEL}"
echo " Target: ${TARGET_MODEL}"
echo " ALFWORLD_DATA: ${ALFWORLD_DATA}"
echo " Output: ${DEFAULT_OUT_ROOT}"
echo "============================================================"
cd "${PROJECT_ROOT}"
python scripts/train.py \
--config configs/alfworld/default.yaml \
--optimizer_model "${OPTIMIZER_MODEL}" \
--target_model "${TARGET_MODEL}" \
--out_root "${DEFAULT_OUT_ROOT}" \
"$@"
echo ""
echo "Done! Results saved to: ${DEFAULT_OUT_ROOT}"
scripts/run_searchqa.sh
Executable
+40
View File
@@ -0,0 +1,40 @@
#!/usr/bin/env bash
# ──────────────────────────────────────────────────────────────────────────────
# SkillOpt — SearchQA training launch script
#
# Usage:
# bash scripts/run_searchqa.sh
# bash scripts/run_searchqa.sh --num_epochs 2 --edit_budget 6
# bash scripts/run_searchqa.sh --split_dir /path/to/searchqa_split
# ──────────────────────────────────────────────────────────────────────────────
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")"
export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}"
OPTIMIZER_MODEL="${OPTIMIZER_MODEL:-gpt-5.5}"
TARGET_MODEL="${TARGET_MODEL:-gpt-5.5}"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
DEFAULT_OUT_ROOT="${PROJECT_ROOT}/outputs/skillopt_searchqa_${TARGET_MODEL}_${TIMESTAMP}"
echo "============================================================"
echo " SkillOpt — SearchQA Training"
echo "============================================================"
echo " Optimizer: ${OPTIMIZER_MODEL}"
echo " Target: ${TARGET_MODEL}"
echo "============================================================"
cd "${PROJECT_ROOT}"
python scripts/train.py \
--config configs/searchqa/default.yaml \
--optimizer_model "${OPTIMIZER_MODEL}" \
--target_model "${TARGET_MODEL}" \
--out_root "${DEFAULT_OUT_ROOT}" \
"$@"
echo ""
echo "Done! Results saved to: ${DEFAULT_OUT_ROOT}"
scripts/run_spreadsheetbench.sh
Executable
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env bash
# ──────────────────────────────────────────────────────────────────────────────
# SkillOpt — SpreadsheetBench training launch script
#
# Usage:
# bash scripts/run_spreadsheetbench.sh --split_dir /path/to/split --data_root /path/to/data
# bash scripts/run_spreadsheetbench.sh --num_epochs 2 --edit_budget 6
# ──────────────────────────────────────────────────────────────────────────────
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")"
export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}"
OPTIMIZER_MODEL="${OPTIMIZER_MODEL:-gpt-5.5}"
TARGET_MODEL="${TARGET_MODEL:-gpt-5.5}"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
DEFAULT_OUT_ROOT="${PROJECT_ROOT}/outputs/skillopt_spreadsheetbench_${TARGET_MODEL}_${TIMESTAMP}"
echo "============================================================"
echo " SkillOpt — SpreadsheetBench Training"
echo "============================================================"
echo " Optimizer: ${OPTIMIZER_MODEL}"
echo " Target: ${TARGET_MODEL}"
echo "============================================================"
cd "${PROJECT_ROOT}"
python scripts/train.py \
--config configs/spreadsheetbench/default.yaml \
--optimizer_model "${OPTIMIZER_MODEL}" \
--target_model "${TARGET_MODEL}" \
--out_root "${DEFAULT_OUT_ROOT}" \
"$@"
echo ""
echo "Done! Results saved to: ${DEFAULT_OUT_ROOT}"
scripts/train.py
+556
View File
@@ -0,0 +1,556 @@
#!/usr/bin/env python3
"""SkillOpt unified training entry point.
Usage
-----
python scripts/train.py --config configs/alfworld/default.yaml
Any YAML key can be overridden from the command line::
python scripts/train.py --config configs/alfworld/default.yaml \\
--batch_size 40 --num_epochs 2 --seed 123
Run ``python scripts/train.py --help`` for a full list of options.
"""
from __future__ import annotations
import argparse
import datetime
import os
import sys
# Ensure the project root is on sys.path so ``import skillopt`` works
# regardless of where the script is invoked from.
_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
_PROJECT_ROOT = os.path.dirname(_SCRIPT_DIR)
if _PROJECT_ROOT not in sys.path:
sys.path.insert(0, _PROJECT_ROOT)
from skillopt.model.common import default_model_for_backend, normalize_backend_name
_OPENAI_DEFAULT_MODEL_SENTINELS = {"gpt-5.4", "gpt-5.5"}
# ── Environment registry ────────────────────────────────────────────────────
_ENV_REGISTRY: dict[str, type] = {}
def _register_builtins() -> None:
"""Lazy-import built-in adapters so we don't pull heavy deps at CLI parse time."""
try:
from skillopt.envs.alfworld.adapter import ALFWorldAdapter
_ENV_REGISTRY["alfworld"] = ALFWorldAdapter
except ImportError:
pass # ALFWorld deps not installed — skip
try:
from skillopt.envs.searchqa.adapter import SearchQAAdapter
_ENV_REGISTRY["searchqa"] = SearchQAAdapter
except ImportError:
pass
try:
from skillopt.envs.livemathematicianbench.adapter import LiveMathematicianBenchAdapter
_ENV_REGISTRY["livemathematicianbench"] = LiveMathematicianBenchAdapter
except ImportError:
pass
try:
from skillopt.envs.babyvision.adapter import BabyVisionAdapter
_ENV_REGISTRY["babyvision"] = BabyVisionAdapter
except ImportError:
pass
try:
from skillopt.envs.spreadsheetbench.adapter import SpreadsheetBenchAdapter
_ENV_REGISTRY["spreadsheetbench"] = SpreadsheetBenchAdapter
except ImportError:
pass
try:
from skillopt.envs.mmrb.adapter import MMRBAdapter
_ENV_REGISTRY["mmrb"] = MMRBAdapter
except ImportError:
pass
try:
from skillopt.envs.docvqa.adapter import DocVQAAdapter
_ENV_REGISTRY["docvqa"] = DocVQAAdapter
except ImportError:
pass
try:
from skillopt.envs.mathverse.adapter import MathVerseAdapter
_ENV_REGISTRY["mathverse"] = MathVerseAdapter
except ImportError:
pass
try:
from skillopt.envs.officeqa.adapter import OfficeQAAdapter
_ENV_REGISTRY["officeqa"] = OfficeQAAdapter
except ImportError:
pass
try:
from skillopt.envs.sealqa.adapter import SealQAAdapter
_ENV_REGISTRY["sealqa"] = SealQAAdapter
except ImportError:
pass
try:
from skillopt.envs.swebench.adapter import SWEBenchAdapter
_ENV_REGISTRY["swebench"] = SWEBenchAdapter
except ImportError:
pass
def get_adapter(cfg: dict):
"""Instantiate the environment adapter specified in ``cfg["env"]``."""
_register_builtins()
env_name = cfg.get("env", "alfworld")
if env_name not in _ENV_REGISTRY:
raise ValueError(
f"Unknown environment '{env_name}'. "
f"Available: {list(_ENV_REGISTRY.keys())}"
)
adapter_cls = _ENV_REGISTRY[env_name]
# Inspect adapter __init__ signature and only pass accepted kwargs
import inspect
sig = inspect.signature(adapter_cls.__init__)
accepted = set(sig.parameters.keys()) - {"self"}
adapter_kwargs: dict = {}
for key in accepted:
if key in cfg:
adapter_kwargs[key] = cfg[key]
return adapter_cls(**adapter_kwargs)
# ── CLI ──────────────────────────────────────────────────────────────────────
_BOOL = lambda x: x.lower() in ("true", "1", "yes") # noqa: E731
def parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(
description="SkillOpt: Executive Strategy for Self-Evolving Agent Skills",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=__doc__,
)
p.add_argument("--config", type=str, required=True,
help="Path to YAML config file")
p.add_argument("--cfg-options", nargs="+", default=[],
help="Override config: section.key=value (e.g. train.batch_size=40)")
# Legacy flat CLI overrides (still work, prefer --cfg-options for new usage)
p.add_argument("--env", type=str)
p.add_argument("--backend", type=str,
choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "qwen", "qwen_chat", "minimax", "minimax_chat"])
p.add_argument("--optimizer_model", type=str)
p.add_argument("--target_model", type=str)
p.add_argument("--optimizer_backend", type=str)
p.add_argument("--target_backend", type=str)
p.add_argument("--reasoning_effort", type=str,
choices=["", "low", "medium", "high", "xhigh", "max"])
p.add_argument("--rewrite_reasoning_effort", type=str)
p.add_argument("--rewrite_max_completion_tokens", type=int)
p.add_argument("--azure_endpoint", type=str)
p.add_argument("--azure_api_version", type=str)
p.add_argument("--azure_api_key", type=str)
p.add_argument("--azure_openai_endpoint", type=str)
p.add_argument("--azure_openai_api_version", type=str)
p.add_argument("--azure_openai_api_key", type=str)
p.add_argument("--azure_openai_auth_mode", type=str)
p.add_argument("--azure_openai_ad_scope", type=str)
p.add_argument("--azure_openai_managed_identity_client_id", type=str)
p.add_argument("--optimizer_azure_openai_endpoint", type=str)
p.add_argument("--optimizer_azure_openai_api_version", type=str)
p.add_argument("--optimizer_azure_openai_api_key", type=str)
p.add_argument("--optimizer_azure_openai_auth_mode", type=str)
p.add_argument("--optimizer_azure_openai_ad_scope", type=str)
p.add_argument("--optimizer_azure_openai_managed_identity_client_id", type=str)
p.add_argument("--target_azure_openai_endpoint", type=str)
p.add_argument("--target_azure_openai_api_version", type=str)
p.add_argument("--target_azure_openai_api_key", type=str)
p.add_argument("--target_azure_openai_auth_mode", type=str)
p.add_argument("--target_azure_openai_ad_scope", type=str)
p.add_argument("--target_azure_openai_managed_identity_client_id", type=str)
p.add_argument("--qwen_chat_base_url", type=str)
p.add_argument("--qwen_chat_api_key", type=str)
p.add_argument("--qwen_chat_temperature", type=float)
p.add_argument("--qwen_chat_timeout_seconds", type=float)
p.add_argument("--qwen_chat_max_tokens", type=int)
p.add_argument("--qwen_chat_enable_thinking", type=_BOOL)
p.add_argument("--optimizer_qwen_chat_base_url", type=str)
p.add_argument("--optimizer_qwen_chat_api_key", type=str)
p.add_argument("--optimizer_qwen_chat_temperature", type=float)
p.add_argument("--optimizer_qwen_chat_timeout_seconds", type=float)
p.add_argument("--optimizer_qwen_chat_max_tokens", type=int)
p.add_argument("--optimizer_qwen_chat_enable_thinking", type=_BOOL)
p.add_argument("--target_qwen_chat_base_url", type=str)
p.add_argument("--target_qwen_chat_api_key", type=str)
p.add_argument("--target_qwen_chat_temperature", type=float)
p.add_argument("--target_qwen_chat_timeout_seconds", type=float)
p.add_argument("--target_qwen_chat_max_tokens", type=int)
p.add_argument("--target_qwen_chat_enable_thinking", type=_BOOL)
p.add_argument("--minimax_base_url", type=str)
p.add_argument("--minimax_api_key", type=str)
p.add_argument("--minimax_model", type=str)
p.add_argument("--minimax_temperature", type=float)
p.add_argument("--minimax_max_tokens", type=int)
p.add_argument("--minimax_enable_thinking", type=_BOOL)
p.add_argument("--codex_exec_path", type=str)
p.add_argument("--codex_exec_sandbox", type=str)
p.add_argument("--codex_exec_profile", type=str)
p.add_argument("--codex_exec_full_auto", type=_BOOL)
p.add_argument("--codex_exec_reasoning_effort", type=str)
p.add_argument("--codex_exec_use_sdk", type=str)
p.add_argument("--codex_exec_network_access", type=_BOOL)
p.add_argument("--codex_exec_web_search", type=_BOOL)
p.add_argument("--codex_exec_approval_policy", type=str)
p.add_argument("--claude_code_exec_path", type=str)
p.add_argument("--claude_code_exec_profile", type=str)
p.add_argument("--claude_code_exec_use_sdk", type=str)
p.add_argument("--claude_code_exec_effort", type=str)
p.add_argument("--claude_code_exec_max_thinking_tokens", type=int)
p.add_argument("--codex_trace_to_optimizer", type=_BOOL)
p.add_argument("--skill_init", type=str)
p.add_argument("--num_epochs", type=int)
p.add_argument("--train_size", type=int)
p.add_argument("--steps_per_epoch", type=int)
p.add_argument("--batch_size", type=int)
p.add_argument("--accumulation", type=int)
p.add_argument("--seed", type=int)
p.add_argument("--edit_budget", type=int)
p.add_argument("--min_edit_budget", type=int)
p.add_argument("--lr_scheduler", type=str,
choices=["constant", "linear", "cosine", "autonomous"])
p.add_argument("--lr_control_mode", type=str,
choices=["fixed", "autonomous", "none"])
p.add_argument("--merge_batch_size", type=int)
p.add_argument("--max_analyst_rounds", type=int)
p.add_argument("--sel_env_num", type=int)
p.add_argument("--test_env_num", type=int)
p.add_argument("--eval_test", type=_BOOL)
p.add_argument("--use_gate", type=_BOOL)
p.add_argument("--max_steps", type=int)
p.add_argument("--max_api_workers", type=int)
p.add_argument("--analyst_workers", type=int)
p.add_argument("--failure_only", type=_BOOL)
p.add_argument("--minibatch_size", type=int)
p.add_argument("--skill_update_mode", type=str,
choices=[
"patch",
"rewrite_from_suggestions",
"rewrite",
"suggestions",
"full_rewrite",
"full_rewrite_minibatch",
"minibatch_full_rewrite",
])
p.add_argument("--use_slow_update", type=_BOOL)
p.add_argument("--slow_update_samples", type=int)
p.add_argument("--longitudinal_pair_policy", type=str,
choices=["mixed", "changed", "unchanged"])
p.add_argument("--use_meta_skill", type=_BOOL)
p.add_argument("--use_skill_aware_reflection", type=_BOOL)
p.add_argument("--skill_aware_appendix_source", type=str,
choices=["both", "failure_only"])
p.add_argument("--skill_aware_consolidate_threshold", type=int)
p.add_argument("--data_path", type=str)
p.add_argument("--split_mode", type=str,
choices=["ratio", "split_dir"])
p.add_argument("--split_ratio", type=str)
p.add_argument("--split_seed", type=int)
p.add_argument("--split_dir", type=str)
p.add_argument("--split_output_dir", type=str)
p.add_argument("--data_root", type=str)
p.add_argument("--max_turns", type=int)
p.add_argument("--workers", type=int)
p.add_argument("--limit", type=int)
p.add_argument("--shuffle_choices", type=_BOOL)
p.add_argument("--use_theorem", type=_BOOL)
p.add_argument("--use_sketch", type=_BOOL)
p.add_argument("--image_detail", type=str)
p.add_argument("--judge_model", type=str)
p.add_argument("--judge_max_completion_tokens", type=int)
p.add_argument("--judge_retries", type=int)
p.add_argument("--out_root", type=str)
p.add_argument("--mode", type=str)
return p.parse_args()
# ── Flat key → structured path mapping (for legacy CLI → structured config) ──
_LEGACY_TO_STRUCTURED: dict[str, str] = {
"backend": "model.backend",
"optimizer_model": "model.optimizer",
"target_model": "model.target",
"optimizer_backend": "model.optimizer_backend",
"target_backend": "model.target_backend",
"reasoning_effort": "model.reasoning_effort",
"rewrite_reasoning_effort": "model.rewrite_reasoning_effort",
"rewrite_max_completion_tokens": "model.rewrite_max_completion_tokens",
"azure_endpoint": "model.azure_endpoint",
"azure_api_version": "model.azure_api_version",
"azure_api_key": "model.azure_api_key",
"azure_openai_endpoint": "model.azure_openai_endpoint",
"azure_openai_api_version": "model.azure_openai_api_version",
"azure_openai_api_key": "model.azure_openai_api_key",
"azure_openai_auth_mode": "model.azure_openai_auth_mode",
"azure_openai_ad_scope": "model.azure_openai_ad_scope",
"azure_openai_managed_identity_client_id": "model.azure_openai_managed_identity_client_id",
"optimizer_azure_openai_endpoint": "model.optimizer_azure_openai_endpoint",
"optimizer_azure_openai_api_version": "model.optimizer_azure_openai_api_version",
"optimizer_azure_openai_api_key": "model.optimizer_azure_openai_api_key",
"optimizer_azure_openai_auth_mode": "model.optimizer_azure_openai_auth_mode",
"optimizer_azure_openai_ad_scope": "model.optimizer_azure_openai_ad_scope",
"optimizer_azure_openai_managed_identity_client_id": "model.optimizer_azure_openai_managed_identity_client_id",
"target_azure_openai_endpoint": "model.target_azure_openai_endpoint",
"target_azure_openai_api_version": "model.target_azure_openai_api_version",
"target_azure_openai_api_key": "model.target_azure_openai_api_key",
"target_azure_openai_auth_mode": "model.target_azure_openai_auth_mode",
"target_azure_openai_ad_scope": "model.target_azure_openai_ad_scope",
"target_azure_openai_managed_identity_client_id": "model.target_azure_openai_managed_identity_client_id",
"qwen_chat_base_url": "model.qwen_chat_base_url",
"qwen_chat_api_key": "model.qwen_chat_api_key",
"qwen_chat_temperature": "model.qwen_chat_temperature",
"qwen_chat_timeout_seconds": "model.qwen_chat_timeout_seconds",
"qwen_chat_max_tokens": "model.qwen_chat_max_tokens",
"qwen_chat_enable_thinking": "model.qwen_chat_enable_thinking",
"optimizer_qwen_chat_base_url": "model.optimizer_qwen_chat_base_url",
"optimizer_qwen_chat_api_key": "model.optimizer_qwen_chat_api_key",
"optimizer_qwen_chat_temperature": "model.optimizer_qwen_chat_temperature",
"optimizer_qwen_chat_timeout_seconds": "model.optimizer_qwen_chat_timeout_seconds",
"optimizer_qwen_chat_max_tokens": "model.optimizer_qwen_chat_max_tokens",
"optimizer_qwen_chat_enable_thinking": "model.optimizer_qwen_chat_enable_thinking",
"target_qwen_chat_base_url": "model.target_qwen_chat_base_url",
"target_qwen_chat_api_key": "model.target_qwen_chat_api_key",
"target_qwen_chat_temperature": "model.target_qwen_chat_temperature",
"target_qwen_chat_timeout_seconds": "model.target_qwen_chat_timeout_seconds",
"target_qwen_chat_max_tokens": "model.target_qwen_chat_max_tokens",
"target_qwen_chat_enable_thinking": "model.target_qwen_chat_enable_thinking",
"minimax_base_url": "model.minimax_base_url",
"minimax_api_key": "model.minimax_api_key",
"minimax_model": "model.minimax_model",
"minimax_temperature": "model.minimax_temperature",
"minimax_max_tokens": "model.minimax_max_tokens",
"minimax_enable_thinking": "model.minimax_enable_thinking",
"codex_exec_path": "model.codex_exec_path",
"codex_exec_sandbox": "model.codex_exec_sandbox",
"codex_exec_profile": "model.codex_exec_profile",
"codex_exec_full_auto": "model.codex_exec_full_auto",
"codex_exec_reasoning_effort": "model.codex_exec_reasoning_effort",
"codex_exec_use_sdk": "model.codex_exec_use_sdk",
"codex_exec_network_access": "model.codex_exec_network_access",
"codex_exec_web_search": "model.codex_exec_web_search",
"codex_exec_approval_policy": "model.codex_exec_approval_policy",
"claude_code_exec_path": "model.claude_code_exec_path",
"claude_code_exec_profile": "model.claude_code_exec_profile",
"claude_code_exec_use_sdk": "model.claude_code_exec_use_sdk",
"claude_code_exec_effort": "model.claude_code_exec_effort",
"claude_code_exec_max_thinking_tokens": "model.claude_code_exec_max_thinking_tokens",
"codex_trace_to_optimizer": "model.codex_trace_to_optimizer",
"num_epochs": "train.num_epochs",
"train_size": "train.train_size",
"steps_per_epoch": "train.steps_per_epoch",
"batch_size": "train.batch_size",
"accumulation": "train.accumulation",
"seed": "train.seed",
"minibatch_size": "gradient.minibatch_size",
"merge_batch_size": "gradient.merge_batch_size",
"analyst_workers": "gradient.analyst_workers",
"max_analyst_rounds": "gradient.max_analyst_rounds",
"failure_only": "gradient.failure_only",
"edit_budget": "optimizer.learning_rate",
"min_edit_budget": "optimizer.min_learning_rate",
"lr_scheduler": "optimizer.lr_scheduler",
"lr_control_mode": "optimizer.lr_control_mode",
"skill_update_mode": "optimizer.skill_update_mode",
"use_slow_update": "optimizer.use_slow_update",
"slow_update_samples": "optimizer.slow_update_samples",
"longitudinal_pair_policy": "optimizer.longitudinal_pair_policy",
"use_meta_skill": "optimizer.use_meta_skill",
"use_skill_aware_reflection": "optimizer.use_skill_aware_reflection",
"skill_aware_appendix_source": "optimizer.skill_aware_appendix_source",
"skill_aware_consolidate_threshold": "optimizer.skill_aware_consolidate_threshold",
"use_gate": "evaluation.use_gate",
"sel_env_num": "evaluation.sel_env_num",
"test_env_num": "evaluation.test_env_num",
"eval_test": "evaluation.eval_test",
"env": "env.name",
"skill_init": "env.skill_init",
"out_root": "env.out_root",
}
def load_config(args: argparse.Namespace) -> dict:
"""Load config with _base_ inheritance, then apply CLI overrides."""
from skillopt.config import load_config as _load, flatten_config, is_structured
cfg = _load(args.config, overrides=args.cfg_options)
structured = is_structured(cfg)
# Apply legacy --key value overrides
cli = {k: v for k, v in vars(args).items()
if v is not None and k not in ("config", "cfg_options")}
if cli:
if structured:
from skillopt.config import apply_overrides
mapped = []
for k, v in cli.items():
dotted = _LEGACY_TO_STRUCTURED.get(k)
if dotted:
mapped.append(f"{dotted}={v}")
else:
mapped.append(f"env.{k}={v}")
apply_overrides(cfg, mapped)
else:
cfg.update(cli)
# Flatten structured config → flat dict for trainer/adapter
flat = flatten_config(cfg) if structured else cfg
for new_key, old_key in (
("azure_openai_endpoint", "azure_endpoint"),
("azure_openai_api_version", "azure_api_version"),
("azure_openai_api_key", "azure_api_key"),
):
if flat.get(new_key) in (None, "") and flat.get(old_key) not in (None, ""):
flat[new_key] = flat[old_key]
explicit_backend = getattr(args, "backend", None)
if explicit_backend is None:
for option in args.cfg_options or []:
key = str(option).split("=", 1)[0].strip()
if key == "model.backend":
explicit_backend = str(option).split("=", 1)[1].strip()
break
backend = normalize_backend_name(flat.get("model_backend") or flat.get("target_backend") or "azure_openai")
def _has_model_override(dotted_key: str, legacy_key: str) -> bool:
if getattr(args, legacy_key, None) is not None:
return True
for option in args.cfg_options or []:
key = str(option).split("=", 1)[0].strip()
if key == dotted_key:
return True
return False
if explicit_backend is not None:
backend = normalize_backend_name(explicit_backend)
flat["model_backend"] = backend
if backend in {"claude", "claude_chat"}:
flat.setdefault("optimizer_backend", "claude_chat")
flat.setdefault("target_backend", "claude_chat")
elif backend in {"codex", "codex_exec"}:
flat.setdefault("optimizer_backend", "openai_chat")
flat.setdefault("target_backend", "codex_exec")
elif backend == "claude_code_exec":
flat.setdefault("optimizer_backend", "openai_chat")
flat.setdefault("target_backend", "claude_code_exec")
elif backend in {"qwen", "qwen_chat"}:
flat.setdefault("optimizer_backend", "openai_chat")
flat.setdefault("target_backend", "qwen_chat")
elif backend in {"minimax", "minimax_chat"}:
flat.setdefault("optimizer_backend", "openai_chat")
flat.setdefault("target_backend", "minimax_chat")
else:
flat.setdefault("optimizer_backend", "openai_chat")
flat.setdefault("target_backend", "openai_chat")
else:
flat.setdefault("optimizer_backend", "openai_chat")
flat.setdefault("target_backend", "openai_chat")
if flat.get("optimizer_backend") == "claude_chat":
if (
str(flat.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
and not _has_model_override("model.optimizer", "optimizer_model")
):
flat["optimizer_model"] = default_model_for_backend("claude_chat")
if flat.get("optimizer_backend") == "qwen_chat":
if (
str(flat.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
and not _has_model_override("model.optimizer", "optimizer_model")
):
flat["optimizer_model"] = default_model_for_backend("qwen_chat")
if flat.get("target_backend") == "claude_chat":
if (
str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
and not _has_model_override("model.target", "target_model")
):
flat["target_model"] = default_model_for_backend("claude_chat")
if flat.get("target_backend") == "claude_code_exec":
if (
str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
and not _has_model_override("model.target", "target_model")
):
flat["target_model"] = default_model_for_backend("claude_chat")
if flat.get("target_backend") == "qwen_chat":
if (
str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
and not _has_model_override("model.target", "target_model")
):
flat["target_model"] = default_model_for_backend("qwen_chat")
if flat.get("target_backend") == "minimax_chat":
if (
str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS
and not _has_model_override("model.target", "target_model")
):
flat["target_model"] = (
flat.get("minimax_model")
or default_model_for_backend("minimax_chat")
)
# Auto-generate output root
if not flat.get("out_root"):
env = flat.get("env", "unknown")
model = flat.get("optimizer_model", "unknown").replace("/", "-")
ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
flat["out_root"] = os.path.join("outputs", f"skillopt_{env}_{model}_{ts}")
flat["out_root"] = os.path.abspath(flat["out_root"])
return flat
# ── Main ─────────────────────────────────────────────────────────────────────
def main() -> None:
args = parse_args()
cfg = load_config(args)
print(f"\n{'='*60}")
print(f" SkillOpt — Executive Strategy for Self-Evolving Agent Skills")
print(f"{'='*60}")
print(f" env: {cfg.get('env')}")
print(f" optimizer_model: {cfg.get('optimizer_model')}")
print(f" target_model: {cfg.get('target_model')}")
print(f" optimizer_backend:{cfg.get('optimizer_backend', 'openai_chat')}")
print(f" target_backend:{cfg.get('target_backend', 'openai_chat')}")
print(f" reasoning: {cfg.get('reasoning_effort') or 'off'}")
print(f" rewrite_effort: {cfg.get('rewrite_reasoning_effort') or 'off'}")
print(f" epochs: {cfg.get('num_epochs')}")
print(f" train_size: {cfg.get('train_size') or 'from dataset'}")
print(f" steps/epoch: auto")
print(f" batch_size: {cfg.get('batch_size')}")
print(f" edit_budget: {cfg.get('edit_budget')}")
print(f" lr_scheduler: {cfg.get('lr_scheduler', 'constant')}")
print(f" update_mode: {cfg.get('skill_update_mode', 'patch')}")
print(f" min_edit_budget:{cfg.get('min_edit_budget', 2)}")
print(f" minibatch_size: {cfg.get('minibatch_size')}")
print(f" seed: {cfg.get('seed')}")
print(f" meta_skill: {cfg.get('use_meta_skill', False)}")
print(f" skill_aware_reflection: {cfg.get('use_skill_aware_reflection', False)}")
print(f" slow_update: {cfg.get('use_slow_update', False)}")
print(f" out_root: {cfg.get('out_root')}")
print(f"{'='*60}\n")
# Build adapter
adapter = get_adapter(cfg)
# Build trainer and run
from skillopt.engine.trainer import ReflACTTrainer
trainer = ReflACTTrainer(cfg, adapter)
summary = trainer.train()
print(f"\n Output saved to: {cfg['out_root']}")
if summary.get("test_hard") is not None:
print(f" Final test: {summary['test_hard']:.4f}")
if __name__ == "__main__":
main()