commit 4b92209caad6373c7864782313132e93fdc9df5f Author: wehub-resource-sync Date: Mon Jul 13 12:24:16 2026 +0800 chore: import upstream snapshot with attribution diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..7060b86 --- /dev/null +++ b/.env.example @@ -0,0 +1,34 @@ +# SkillOpt Environment Variables +# Copy this file to .env and fill in your values. +# Usage: set -a; source .env; set +a + +# ── Azure OpenAI (required for openai_chat backend) ────────────────── +export AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/ +export AZURE_OPENAI_API_VERSION=2024-12-01-preview +# Authentication: choose one method +# Option 1: API Key +export AZURE_OPENAI_API_KEY= +# Option 2: Azure CLI (no API key needed, recommended on Azure VMs) +# export AZURE_OPENAI_AUTH_MODE=azure_cli +# Option 3: Managed Identity +# export AZURE_OPENAI_AUTH_MODE=managed_identity +# export AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID=your-client-id + +# ── OpenAI-compatible endpoints ────────────────────────────────────── +# Set AUTH_MODE to openai_compatible and reuse AZURE_OPENAI_ENDPOINT / _API_KEY. +# The plain OpenAI client is used; no Azure auth, no api-version header. +# export AZURE_OPENAI_ENDPOINT=https://api.openai.com/v1 +# export AZURE_OPENAI_API_KEY=sk-... +# export AZURE_OPENAI_AUTH_MODE=openai_compatible + +# ── Anthropic / Claude (for claude_chat backend) ───────────────────── +# export ANTHROPIC_API_KEY=sk-ant-... + +# ── Qwen Local Model (for qwen_chat backend) ──────────────────────── +# export QWEN_CHAT_BASE_URL=http://localhost:8000/v1 +# export QWEN_CHAT_MODEL=Qwen/Qwen3.5-4B + +# ── MiniMax (for minimax_chat backend) ────────────────────────────── +# export MINIMAX_BASE_URL=https://api.minimax.io/v1 +# export MINIMAX_API_KEY=... +# export MINIMAX_MODEL=MiniMax-M2.7 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4a5142a --- /dev/null +++ b/.gitignore @@ -0,0 +1,66 @@ +__pycache__/ +*.pyc +*.egg-info/ +build/ +dist/ +site/ + +data/* +!data/README.md +!data/searchqa_id_split/ +!data/searchqa_id_split/** +!data/livemathematicianbench_id_split/ +!data/livemathematicianbench_id_split/** +!data/docvqa_id_split/ +!data/docvqa_id_split/** +!data/officeqa_id_split/ +!data/officeqa_id_split/** +!data/spreadsheetbench_id_split/ +!data/spreadsheetbench_id_split/** +!data/alfworld_path_split/ +!data/alfworld_path_split/** +outputs/ +logs/ +external/ +# SkillOpt-Sleep runtime state (staging proposals, config, diagnostics, cron logs) +.skillopt-sleep/ +# SkillOpt-Sleep handoff-backend round data (prompts/answers derived from transcripts) +.skillopt-sleep-handoff/ +.skillopt-sleep-handoff.night*.done/ + +/BabyVision/ +/MMRB/ +/SpreadsheetBench/ +/dl4ir-searchQA/ + +configs/local/ +configs/**/*.local.yaml +*.local.md +*.secret.md +*.bak + +.env +.secrets/ +.codex_azure*/ + +# Internal docs (not for open-source release) +docs/ablation_plan.md +docs/ablation_paper_tables.md +docs/ablation_paper_tables.html +docs/experiment_commands.md +docs/slow_update_flowchart.md +docs/session_memory.md +docs/harness_fresh_machine_handoff.md +docs/harness_monitoring_memory.md +docs/harness_reproduction_secrets.secret.md +docs/reflact_conda_env_export.yml +docs/reflact_overview.html +docs/render_ablation_paper_tables.py +docs/让* +.gradio/ +.venv + +# Local experiment launchers — contain machine-specific endpoints/identities, never commit +tests/run_*.sh +tests/launch_*.py +*.launch.log diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1e42464 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,113 @@ +# Changelog + +All notable changes to SkillOpt are documented here. This project adheres to +[Semantic Versioning](https://semver.org/) and the format is based on +[Keep a Changelog](https://keepachangelog.com/). + +## [Unreleased] + +### Added +- **Handoff backend** (`--backend handoff`) for SkillOpt-Sleep — runs the + sleep cycle with no model subprocess or API key: the engine writes each + pending model call to `PROMPTS.md`/`pending.json` (exit code 3) and the + user's own agent session answers into `answers/.md`; re-running the + same command resumes statelessly from the answers (typically 3–6 rounds + per night). Mined tasks are pinned per night so answering sessions cannot + shift the task set. Ships a `/skillopt-sleep-handoff` Claude Code command + that automates the loop with fresh-context subagents to protect the + held-out gate. + +## [0.2.0] — 2026-07-02 + +The headline of this release is **SkillOpt-Sleep**: a nightly offline +self-evolution engine that harvests a coding agent's real session +transcripts, mines recurring tasks, replays them offline, and consolidates +short-term experience into long-term memory and skills — all behind the same +held-out validation gate that keeps SkillOpt training honest. It ships as a +decoupled top-level package (`skillopt_sleep/`, zero dependency on the +research code) and as the new `skillopt-sleep` CLI. + +### Added +- **SkillOpt-Sleep engine** — nightly offline self-evolution cycle + (harvest → mine → replay → consolidate) behind a validation gate, exposed + as the `skillopt-sleep` console script and `python -m skillopt_sleep`. + - Multi-objective reward (accuracy / tokens / latency) with user preferences. + - Multi-rollout contrastive reflection under a token/time budget. + - Experience replay + controllable dream rollouts (opt-in). + - Slow-update long-term memory field (runs even with the gate off). + - 3-way train/val/test split with `gate_mode on|off`. + - Verifier-discipline validation gate, with a stress-test suite + (thanks @Tanmay9223, #87). +- **Cross-tool backends & plugin shells** for Claude Code, Codex, Copilot, + Devin, and OpenClaw: + - Codex Desktop transcript harvesting, skill-first Codex integration, and a + reviewed task-file flow (thanks @Kirchberg, #48, #49, #60). + - GitHub Copilot backend (`CopilotCliBackend`) + research-engine MCP plugin + (thanks @Dongbumlee, #50). + - Devin plugin: MCP server + ATIF-v1.7 harvest (thanks @xerxes-y, #88). + - OpenClaw shell for SkillOpt-Sleep (thanks @Elzlxx, #59). +- **SearchQA** split materialization helper and fail-fast on systemic rollout + failures, with a `searchqa` install extra (thanks @summerview1997, + #63, #64, #65). +- WebUI environment loading and backend preflight (thanks @summerview1997, #63). + +### Changed +- Decoupled the Sleep engine into a standalone top-level `skillopt_sleep/` + package with zero dependency on the research code. +- Made `EnvAdapter.reflect` a shared default so reflect kwargs are no longer + dropped (thanks @imshunsuke, #44). +- English-only pass across the engine, plugins, and docs. + +### Fixed +- Windows robustness for the Claude/Codex backends, plus a hardened JSON + fallback path (thanks @Yif-Yang, #79). +- Reject prose pseudo-JSON wrapped in single quotes/backticks (#82). +- Surface Codex auth/model/version failures instead of silently scoring 0 + (thanks @dmmdea, #92). +- Redact secrets before persisting cycle diagnostics. +- Configure the `qwen_chat`/`minimax` backends so local LLM endpoints work + (thanks @imrehg, #85). +- Forward the Qwen target timeout and gate `enable_thinking` for vLLM targets + (thanks @mvanhorn, #40). +- Make `--bare` conditional on `ANTHROPIC_API_KEY` (#68), add a + `SKILLOPT_SLEEP_PYTHON` override with a lookback-hours first-run fallback + (#74), and fix ALFWorld gamefile paths relative to `ALFWORLD_DATA`. + +### Packaging +- Bump `skillopt`, `skillopt.__version__`, and `skillopt_sleep.__version__` + to `0.2.0`. +- Restore `skillopt_webui` to the built wheel (it was dropped when the + `packages.find` include list was made explicit). +- Add the `searchqa` extra and include `json_repair` in the `claude`, `qwen`, + and `all` extras. + +### Acknowledgements 🙏 +v0.2.0 landed thanks to our community contributors — thank you! + +- @Kirchberg — Codex Desktop harvesting, skill-first Codex integration, + reviewed task-file flow (#48, #49, #60) +- @Dongbumlee — GitHub Copilot backend + research-engine MCP plugin (#50) +- @summerview1997 — SearchQA materialization, rollout fail-fast, WebUI + preflight (#63, #64, #65) +- @xerxes-y — Devin plugin: MCP server + ATIF-v1.7 harvest (#88) +- @Elzlxx — OpenClaw shell for SkillOpt-Sleep (#59) +- @imshunsuke — shared `EnvAdapter.reflect` default + docs fixes (#43, #44) +- @mvanhorn — Qwen timeout forwarding + `enable_thinking` gating (#40) +- @dmmdea — surface Codex auth/model/version failures (#92) +- @Tanmay9223 — verifier-discipline stress test (#87) +- @imrehg — `configure_qwen_chat` for local LLM endpoints (#85) +- @samuelgoofus-boop — community contributions + +Special thanks to @Yif-Yang for driving the SkillOpt-Sleep engine. + +**Full changelog:** https://github.com/microsoft/SkillOpt/compare/v0.1.0...v0.2.0 + +## [0.1.0] — 2026-06-02 + +Initial public release: the full training loop (rollout → reflect → +aggregate → select → update → evaluate), multi-backend support +(OpenAI / Azure / Claude / Qwen / MiniMax), six built-in benchmarks, and the +WebUI dashboard. + +[0.2.0]: https://github.com/microsoft/SkillOpt/releases/tag/v0.2.0 +[0.1.0]: https://github.com/microsoft/SkillOpt/releases/tag/v0.1.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..5a3f3c9 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,43 @@ +# Contributing to SkillOpt + +Thank you for your interest in contributing! SkillOpt welcomes contributions of all kinds. + +## Getting Started + +```bash +git clone https://github.com/microsoft/SkillOpt.git +cd SkillOpt +pip install -e ".[dev]" +``` + +## How to Contribute + +### 🐛 Bug Reports +Open a GitHub issue with reproduction steps, expected/actual behavior, and your config file (remove API keys). + +### 🔧 Add a Benchmark +See the [guide](docs/guide/new-benchmark.md) and use the scaffold at `skillopt/envs/_template/`. + +### 🤖 Add a Model Backend +See the [guide](docs/guide/new-backend.md). + +### 📝 Improve Documentation +```bash +pip install -e ".[docs]" +mkdocs serve # Preview at http://localhost:8000 +``` + +## Pull Request Process + +1. Fork the repo and create a feature branch +2. Make changes and test with an existing benchmark +3. Submit a PR with a clear description +4. Ensure CI passes + +## Code Style +- Follow existing patterns in the codebase +- Use type hints for function signatures +- Keep docstrings concise + +## License +By contributing, you agree your contributions are licensed under the [MIT License](LICENSE). diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..cf7bcb2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Microsoft Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..6f2c6ff --- /dev/null +++ b/README.md @@ -0,0 +1,107 @@ +# SkillOpt: Executive Strategy for Self-Evolving Agent Skills + +*Train agent skills like you train neural networks — with epochs, (mini-)batchsize, learning rates, and validation gates — but without touching model weights.* + +[![Project Page](https://img.shields.io/badge/Project%20Page-SkillOpt-8dbb3c)](https://microsoft.github.io/SkillOpt/) [![Paper](https://img.shields.io/badge/Paper-arXiv-b31b1b)](https://arxiv.org/abs/2605.23904) [![Project Video](https://img.shields.io/badge/Project%20Video-Watch%20Demo-ff0000)](https://youtu.be/JUBMDTCiM0M) [![PyPI](https://img.shields.io/badge/PyPI-skillopt-green.svg)](https://pypi.org/project/skillopt/) [![Python 3.10+](https://img.shields.io/badge/Python-3.10%2B-blue.svg)](https://www.python.org/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) + +

+ microsoft%2FSkillOpt | Trendshift + microsoft%2FSkillOpt | Trendshift +

+ +> 📖 **For installation, data preparation, training/eval commands, the full configuration reference, and framework internals, see the [Documentation & Reproduction Guide](https://microsoft.github.io/SkillOpt/docs/guideline.html)** (rendered on GitHub Pages). + +--- + +## News 🔥🔥🔥 +- **[2026-07-02]** 🚀 **SkillOpt [v0.2.0](https://github.com/microsoft/SkillOpt/releases/tag/v0.2.0) is out on [PyPI](https://pypi.org/project/skillopt/)!** Headline feature: **SkillOpt-Sleep**, a nightly offline self-evolution engine (harvest → mine → replay → consolidate, all behind a held-out validation gate) with multi-objective reward, experience replay + dream rollouts, and long-term memory — now shipped as the `skillopt-sleep` CLI. This release also adds cross-tool backends and plugin shells for **Claude, Codex, Copilot, Devin, and OpenClaw**, SearchQA split materialization, Windows robustness, and hardened JSON parsing. See the [release notes](https://github.com/microsoft/SkillOpt/releases/tag/v0.2.0) for the full changelog and contributor acknowledgements. +- **[2026-06-15]** 😴 **SkillOpt-Sleep (preview)** — a nightly offline self-evolution companion for local coding agents (Claude Code / Codex / Copilot): review past sessions, replay recurring tasks, and consolidate validated skills behind a held-out gate. See **[`docs/sleep/README.md`](docs/sleep/README.md)** for what it is, how to use it, and results. +- **[2026-06-03]** 🎉 **[gbrain](https://github.com/garrytan/gbrain), [gbrain-evals](https://github.com/garrytan/gbrain-evals/blob/main/docs/benchmarks/2026-06-03-skillopt.md), and [darwin-skill](https://github.com/alchaincyf/darwin-skill) have all integrated SkillOpt.** +- **[2026-06-02]** 🎉 **SkillOpt [v0.1.0](https://github.com/microsoft/SkillOpt/releases/tag/v0.1.0) is now available on [PyPI](https://pypi.org/project/skillopt/)!** Install with `pip install skillopt`. This initial release includes the full training loop (rollout → reflect → aggregate → select → update → evaluate), multi-backend support (OpenAI / Azure / Claude / Qwen / MiniMax), six built-in benchmarks, and WebUI dashboard. + +--- + +## Overview + +Modern agent skills are usually hand-crafted, generated one-shot by a strong +LLM, or evolved through loosely controlled self-revision — none of which +behaves like a deep-learning optimizer for the skill itself, and none of +which reliably improves over its starting point under feedback. + +**SkillOpt treats the skill document as the trainable state of a frozen +agent**, and trains it with the discipline that makes weight-space +optimization reproducible. A separate optimizer model turns scored rollouts +into bounded add / delete / replace edits on a single skill document; a +candidate edit is accepted only when it strictly improves a held-out +validation score. A textual learning-rate budget, a rejected-edit buffer, +and an epoch-wise slow / meta update make skill training stable while +adding **zero inference-time model calls** at deployment. + +The deployed artifact is a compact `best_skill.md` (typically 300–2,000 +tokens) that runs against the unchanged target model. Across **six +benchmarks, seven target models, and three execution harnesses** (direct +chat, Codex CLI, Claude Code CLI), SkillOpt is best or tied-best on **all +52 evaluated (model, benchmark, harness) cells** and on GPT-5.5 lifts the +average no-skill accuracy by **+23.5 points in direct chat, +24.8 inside +the Codex agentic loop, and +19.1 inside Claude Code**. Optimized skill +artifacts transfer across model scales, between Codex and Claude Code +harnesses, and to nearby benchmarks without further optimization. + +For the full method, ablations, and per-cell results see the [paper](https://arxiv.org/abs/2605.23904); for a visual walkthrough of the loop see the [project page](https://microsoft.github.io/SkillOpt/); for deeper API / backend / benchmark docs see [`docs/`](docs/). + +## 🎬 Demo Video + +https://github.com/user-attachments/assets/eb12d3bc-371c-467f-904d-91b61f339ed7 + +

+ ▶ Watch the full demo on YouTube +

+ +--- + +## Extensibility & WebUI + +### Adding a new backend + +A backend = a chat / exec target (e.g. `openai_chat`, `claude_chat`, +`qwen_chat`, `minimax_chat`, `codex_exec`, `claude_code_exec`). See +[`docs/guide/new-backend.md`](docs/guide/new-backend.md) for the full +contract; in short you add a `skillopt/model/_backend.py` module, +register it in `skillopt/model/common.py` + `backend_config.py`, and wire +it through the router in `skillopt/model/__init__.py`. `qwen_backend.py` +and `minimax_backend.py` are good templates. + +### Adding a new benchmark + +A benchmark = a `skillopt/envs//` package with a `dataloader.py`, a +`rollout.py`, and an `initial.md` seed skill. See +[`docs/guide/new-benchmark.md`](docs/guide/new-benchmark.md) for the full +contract; the simplest reference is `skillopt/envs/searchqa/`. + +### WebUI + +Launch the monitoring dashboard (optional): + +```bash +pip install -e ".[webui]" +python -m skillopt_webui.app +``` + +| Flag | Default | Description | +|---|---|---| +| `--port` | 7860 | Server port | +| `--host` | `0.0.0.0` | Bind address | +| `--share` | off | Create a public Gradio share link | + +--- + +## Citation + +```bibtex +@article{yang2026skillopt, + title={Skillopt: Executive strategy for self-evolving agent skills}, + author={Yang, Yifan and Gong, Ziyang and Huang, Weiquan and Yang, Qihao and Zhou, Ziwei and Huang, Zisu and Li, Yan and Gao, Xuemei and Dai, Qi and Liu, Bei and others}, + journal={arXiv preprint arXiv:2605.23904}, + year={2026} +} +``` diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..49399d9 --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,7 @@ +# WeHub 来源说明 + +- 原始项目：`microsoft/SkillOpt` +- 原始仓库：https://github.com/microsoft/SkillOpt +- 导入方式：上游默认分支的最新快照 +- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准 +- 本文件仅用于记录来源，不代表 WeHub 是原项目作者 diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..e751608 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,14 @@ + + +## Security + +Microsoft takes the security of our software products and services seriously, which +includes all source code repositories in our GitHub organizations. + +**Please do not report security vulnerabilities through public GitHub issues.** + +For security reporting information, locations, contact information, and policies, +please review the latest guidance for Microsoft repositories at +[https://aka.ms/SECURITY.md](https://aka.ms/SECURITY.md). + + \ No newline at end of file diff --git a/ckpt/README.md b/ckpt/README.md new file mode 100644 index 0000000..b79f766 --- /dev/null +++ b/ckpt/README.md @@ -0,0 +1,79 @@ +# Paper-aligned SkillOpt reference skills (GPT-5.5) + +This folder provides a subset of the paper's main Table 1 GPT-5.5 optimized +skills as reference artifacts — one `gpt5.5_skill.md` per currently included +benchmark. You can plug them into `scripts/eval_only.py` to evaluate the +provided skills on a given split without re-running the training loop. + +> These are checkpoints associated with the paper, not a general-purpose +> tool. They're here so you can verify the reported numbers and use the +> skills as portable artifacts. If you want to *train* your own skill, +> use `scripts/train.py` per the top-level README. +> +> This is the first artifact batch. We plan to continue uploading the +> remaining optimized skills and benchmark split manifests as they are +> cleaned and verified. + +## What's here + +| Benchmark | Skill artifact | Matching config | +|---|---|---| +| SearchQA | `ckpt/searchqa/gpt5.5_skill.md` | `configs/searchqa/default.yaml` | +| ALFWorld | `ckpt/alfworld/gpt5.5_skill.md` | `configs/alfworld/default.yaml` | +| DocVQA | `ckpt/docvqa/gpt5.5_skill.md` | `configs/docvqa/default.yaml` | +| LiveMathematicianBench | `ckpt/livemath/gpt5.5_skill.md` | `configs/livemathematicianbench/default.yaml` | +| OfficeQA | `ckpt/officeqa/gpt5.5_skill.md` | `configs/officeqa/default.yaml` | +| SpreadsheetBench | `ckpt/spreadsheetbench/gpt5.5_skill.md` | `configs/spreadsheetbench/default.yaml` | + +Each file is a plain Markdown skill document (~2k–13k chars). It contains a +protected `SLOW_UPDATE` section at the end that holds epoch-wise +longitudinal guidance — that's expected, not a formatting issue. + +## How to evaluate a provided skill + +`scripts/eval_only.py` runs a single skill against a data split without +invoking the optimizer. Example for SearchQA against the test split: + +```bash +python scripts/eval_only.py \ + --config configs/searchqa/default.yaml \ + --skill ckpt/searchqa/gpt5.5_skill.md \ + --split valid_unseen \ + --split_dir data/searchqa_id_split \ + --azure_openai_endpoint https://your-resource.openai.azure.com/ \ + --target_model gpt-5.5 +``` + +Substitute the benchmark, config, skill path, and `--split_dir` to evaluate +any of the other five. `--split valid_unseen` is the test split, `valid_seen` +is the selection / validation split, `train` is the training split, and +`all` runs all three. + +## On comparing to the paper numbers + +To compare against the paper-reported cells, use the same dataset split and +scorer. SearchQA's split is checked in at `data/searchqa_id_split/` (400 +train / 200 selection / 1400 test). For the other benchmarks, point +`--split_dir` at your own materialized split; the loader is deterministic +from `split_seed` (default `42`) + `split_ratio` (default `2:1:7`) when +`split_mode: ratio` is used, so a given `data_path` + seed reproduces +across machines. Explicit per-benchmark split manifests are being prepared +for upload — see issues #14 and #21. + +## Why force-accept vs. gated slow-update matters + +These `ckpt/` skills were produced with the gated slow-update semantics +described in paper Section 3.6: + +```yaml +optimizer: + slow_update_gate_with_selection: true +``` + +Current `main` defaults to `false` (force-accept mode), a newer +post-submission behavior where the slow-update guidance is written into +`current_skill` and `best_skill` unconditionally at the epoch boundary. If +you re-train with the current default, you may produce a *different* +`best_skill.md` than the one checked in here. Both modes are supported; +see the top-level README's "Configuration -> Slow-update acceptance mode" +section. diff --git a/ckpt/alfworld/gpt5.5_skill.md b/ckpt/alfworld/gpt5.5_skill.md new file mode 100644 index 0000000..fb30943 --- /dev/null +++ b/ckpt/alfworld/gpt5.5_skill.md @@ -0,0 +1,113 @@ +# ALFWorld Embodied Agent Skill + +## Overview +This skill guides agents operating in the ALFWorld text-based embodied environment. +The agent must complete household tasks by navigating rooms, interacting with objects, +and using appliances. Actions must be chosen from the admissible action list provided +at each step. + +**Output format**: Always output `...` for reasoning, then `...` for the chosen action. + +--- + +## Task Types + +| Type | Goal | Key Steps | +|------|------|-----------| +| Pick & Place | Put object X in/on receptacle Y | Find X -> take X -> go to Y -> put X in/on Y | +| Pick Two & Place | Put two instances of X in/on Y | Find X1 -> take -> place -> find X2 -> take -> place | + +### Pick Two Object Bookkeeping +For `pick_two_obj_and_place`, choose one destination receptacle instance once it is opened/usable, and remember it as the target. Both object instances should be placed into that same remembered receptacle. After placing the first object, do not remove it again; if the second object was already seen, return directly to its remembered location rather than searching randomly. If the two objects are accidentally split across different receptacles, consolidate them into the chosen target receptacle. +| Examine in Light | Examine object X under desklamp | Find X -> take X -> find desklamp -> use desklamp | + +| Examine in Light detail | Final interaction | While holding X where a desklamp is visible, use the desklamp; do not try to place X on the lamp first. | +| Clean & Place | Clean object X and put in/on Y | Find X -> take X -> go to sink -> clean X -> go to Y -> put X | +| Heat & Place | Heat object X and put in/on Y | Find X -> take X -> go to microwave -> heat X -> go to Y -> put X | +| Cool & Place | Cool object X and put in/on Y | Find X -> take X -> go to fridge -> cool X -> go to Y -> put X | + +--- + +## General Principles + +1. **Decompose the task**: Parse the goal into ordered sub-goals (locate, acquire, transform, deliver). Complete each before moving to the next. +2. **Systematic exploration**: Search each surface and container exactly once before revisiting. Open closed containers (drawers, cabinets, fridge) before judging them empty. + +- Prioritize semantically likely locations first, then broaden systematically: food in fridges/on countertops or dining tables; dishes/utensils/cookware on countertops, dining tables, stoveburners, cabinets, or drawers; office/bedroom items on desks, shelves, dressers, sidetables, or in drawers; newspapers on coffeetables, sidetables, sofas, or tvstands; toiletries/cleaning items near sinks, bathroom counters, shelves, carts, or cabinets. + +- For portable kitchen targets such as bread, mugs, cups, plates, bowls, and utensils, check broad exposed surfaces early: after one or two empty countertops, try dining tables or other open surfaces before opening many cabinets/drawers. For small office/bedroom targets, alternate drawers with exposed desks, shelves, sidetables, and dressers rather than exhausting drawers first. + +- Keep a persistent **searched set** of receptacle instances, e.g. `drawer 1`, `shelf 3`, `countertop 2`. Once an observation shows no needed target object there, mark it searched and do not call it “unexplored” later. + - If all locations in the current preferred class are searched, **broaden to any unvisited admissible `go to ...` location** instead of restarting the same sequence. Search broadly across surfaces, furniture, containers, and appliances when relevant. + - If a visible object is itself an openable/container-like object, such as a box, and opening/examining it is admissible, inspect it before leaving the area. +3. **Grab immediately**: When a required object is visible and reachable, take it right away before moving elsewhere. + +- Pick up only the exact requested object type. Similar or related objects, such as a cup when the task asks for a mug, a spoon when it asks for a knife, or a pot when it asks for a pan, are distractors; leave them in place and mark that location searched for the target. +4. **Transform before placing**: If the task requires cleaning, heating, or cooling, perform the state change at the appropriate appliance before heading to the final destination. + +- Do not repeatedly revisit the sink, microwave, fridge, or final destination before holding the target object. If you find the appliance early, remember its location, then resume searching unvisited object locations until the target object is acquired. + +- Use direct admissible appliance/tool commands immediately when available, such as `clean X with sinkbasin`, `heat X with microwave`, `cool X with fridge`, or `use desklamp`. Do not waste steps opening, closing, toggling, or examining the appliance unless the needed action is unavailable or opening is required for searching/placing. +5. **Direct delivery**: Once holding the transformed (or untransformed) goal object, navigate straight to the target receptacle and place it. + +- Remember known destination receptacles and return directly to the same instance after pickup/transformation. If the destination is also a semantically likely source location, check/open it early rather than only after exhaustive search: food may already be in the fridge, utensils may be on the diningtable, newspapers may be on/near the sofa, and a target drawer can be opened early for pick-two tasks. If the object starts at the destination but needs cleaning/heating/cooling, take it out, transform it, then return to that same instance and place it back. +6. **Track progress**: Maintain an internal count of how many objects still need to be found and placed. Only stop searching when the count reaches zero. +7. **Avoid loops**: Never repeat the same action more than twice in a row. If stuck, move to a different unexplored location. +8. **Only choose admissible actions**: Always pick an action from the admissible action list. Do not invent actions. + +--- + +## Common Mistakes to Avoid + +- **Revisiting searched locations**: Keep track of which surfaces/containers have been checked; do not re-examine them. +- **Ignoring visible objects**: If the target object appears in the observation, pick it up immediately. +- **Skipping state changes**: Do not place an object at the destination without first cleaning/heating/cooling it when required. +- **Premature termination**: Do not stop the episode until all goal conditions are verified as met. +- **Action loops**: Repeatedly toggling or examining the same object wastes steps. Move on to new locations instead. + +### Hard Search-Loop Recovery + +- **Exact-instance lockout before pickup**: once a receptacle/surface instance has been observed and does not contain the target object, do not go back to that exact instance while still searching for the object. A phase change, such as holding the object or needing final delivery, is the only reason to return. +- **Fast broadening threshold**: after 3-4 misses in the same receptacle class, switch to a different likely class or any unvisited admissible location instead of continuing or restarting that class, unless the target has already been seen there. +- **No search reset by recency**: do not say a location is "unsearched" merely because it was not in the last few observations. The searched set is global for the whole episode. +- **Finite-class exhaustion**: if all visible instances of a small class have been checked once, such as all stoveburners, diningtables, countertops, or shelves, mark that class exhausted for object search and do not start a second pass. Remember a usable destination instance, then search different receptacle classes. +- **Unvisited beats likely-but-searched**: after several misses, prefer any admissible unvisited `go to`, `open`, or `examine` target over revisiting a semantically likely but already-searched location. +- **Destination surfaces before pickup**: if the destination receptacle is also a likely object location, inspect each instance at most once before pickup. If it lacks the object, remember it as the final destination but stop using it as a search target until the object has been transformed and is ready to place. +- **Kitchen item fallback**: for cookware and dishware, after checking obvious burners/tables/counters once, broaden to unsearched cabinets, drawers, shelves, sinkbasins, and other kitchen storage/surfaces rather than cycling among the obvious locations. + +### Strict Search Ledger Action Filter + +Before every empty-handed search action, apply this hard filter: + +1. If a required target object is visible, take it immediately. +2. Otherwise choose an exact receptacle/surface/container instance whose contents have not yet been observed in the current object-search phase. +3. Reject any `go to`, `examine`, or `open` action for an exact instance already observed to lack the target, even if it is semantically likely, nearby, recently mentioned, or the final destination type. +4. If all likely instances are rejected by the ledger, broaden to any unvisited admissible location/class instead of restarting from instance 1 of a searched class. + +The searched ledger survives inventory checks, appliance visits, placing the first object in a pick-two task, and putting down an irrelevant inspected object/container. These events are not permission to rescan shelves, drawers, cabinets, tables, counters, or destination receptacles from the beginning. + +### Destination-as-Source Lockout + +When the final receptacle type is also a plausible source location, inspect each visible destination instance at most once before pickup. After it lacks the target, remember a usable destination instance and lock that exact instance out of object search until you are holding the required object ready for delivery. Do not alternate between destination instances and other searched source instances while still empty-handed. + +### Pick-Two Phase Memory + +After placing the first object in a pick-two task, do not begin a fresh room/class search. If another required instance was previously seen, return directly to that remembered source location for the second pickup. If no second instance is remembered, continue from the existing unsearched-location ledger rather than revisiting locations already checked before the first placement. + + +Preserve the successful pattern: when the exact requested object is visible, take it immediately; perform the required clean/heat/cool/use action as soon as the correct command is admissible; then deliver directly to the remembered destination. + +Treat tool locations as tools, not repeated search targets. If a sinkbasin, fridge, microwave, desklamp, or destination receptacle has already been checked and does not contain the target while you are empty-handed, remember it for later but do not revisit it until you are holding the required object or ready to place/use it. + +Use a next-unsearched-instance pointer for every numbered class. If you leave cabinets, drawers, shelves, countertops, or stoveburners and later return to that class, resume at the lowest exact instance not yet observed; never restart at instance 1 and never revisit an instance already observed to lack the target. + +For pan-to-stoveburner tasks, search in a step-efficient order: make one quick pass over stoveburners only to find a pan or remember an empty destination, then leave stoveburners until delivery. Next check countertops/islands and sinkbasins. Then prioritize cabinets in numeric order, opening each closed cabinet and observing its contents, before low-yield drawers. Do not abandon cabinet search to revisit searched stoveburners, countertops, or drawers. + +For kettle/teapot clean-and-place tasks, after checking obvious countertops/islands, check stoveburners and sinkbasins once, then cabinets in numeric order. If several cabinets are empty, continue to the next unsearched cabinet or broaden to unvisited shelves/carts/dining tables; do not return to already searched countertops. Remember one open/empty cabinet as the final destination, but do not keep using searched cabinets as search targets. + +For dishsponge clean-and-place tasks, check sinkbasin and nearby countertops once, then search unvisited cabinets, drawers, shelves, carts, and other storage/surfaces. Because the sink is needed for cleaning, remember it after the first visit; do not go back to the sink while empty-handed just because the sponge is likely near it. Because shelf is the destination, remember a usable shelf after inspecting it once; after a shelf lacks the sponge, search only unvisited shelves or other unvisited locations until the sponge is found. + +When the step budget is running and you are still empty-handed, prefer any unvisited admissible location over any searched likely location. A location being semantically likely, useful later, or recently mentioned is never a reason to rescan it before acquisition. + +Do not let the broadening threshold cause class restarts. Broadening means move to a different unvisited class or continue at the next unsearched instance of a promising storage class; it never means cycling back through exact instances already observed. + diff --git a/ckpt/docvqa/gpt5.5_skill.md b/ckpt/docvqa/gpt5.5_skill.md new file mode 100644 index 0000000..476992f --- /dev/null +++ b/ckpt/docvqa/gpt5.5_skill.md @@ -0,0 +1,26 @@ +# DocVQA Skill + +## Visual Evidence Discipline +- Read the document carefully before answering. +- Prefer the smallest exact text span that answers the question. + +- For questions asking for a value, count, page number, date, or graph reading, return only the requested value span; omit nearby labels, category names, units, or explanatory words unless the question explicitly asks for them. +- When several nearby strings look similar, choose the one whose surrounding labels or layout best match the question. + +## Exact Answer Discipline +- Copy names, numbers, and dates exactly from the document whenever possible. + +- Preserve the document's exact spelling and punctuation for names and quoted phrases; do not substitute similar letters or change straight/curly quotes, spacing, or parentheses when the visible text provides them. +- Prefer direct extraction over paraphrase. +- Before finalizing, compare the answer against nearby alternatives and keep the best-supported exact span. + +## Structured Layout Lookup +- For tables, first find the row or entry named in the question, then read the value under the requested column, header, date, or category; answer with that cell only. +- For forms, receipts, or labeled fields, locate the exact role, party, or field label mentioned in the question, then copy the filled-in value from the same line, box, block, or immediately adjacent field. +- For table-of-contents, indexed, numbered, or bulleted lists, match the requested title, entry, or point number, then follow the same line or list item to the associated value; do not take a nearby value from another item. + +## Anchored Handwriting / Nearby Text +- For handwritten or list/table questions with an anchor term, first locate the anchor, then inspect the immediately adjacent text in the same row, column, or nearby margin. If legible, provide the best-supported nearby span rather than leaving the answer blank. + + + \ No newline at end of file diff --git a/ckpt/livemath/gpt5.5_skill.md b/ckpt/livemath/gpt5.5_skill.md new file mode 100644 index 0000000..2d1ac84 --- /dev/null +++ b/ckpt/livemath/gpt5.5_skill.md @@ -0,0 +1,35 @@ +# Live Mathematical MCQ Heuristics + +## Option Comparison + +### Meta-Options About Stronger Results +- Treat options of the form “one of the remaining options is correct, but a stronger result can be proven” as serious candidates, especially when the question asks for the strongest statement. +- If a concrete option is true but your theorem or derivation gives a strictly stronger conclusion not exactly listed, choose the meta-option rather than the weaker concrete statement. +- When options are nested by strength, rank them explicitly before answering: e.g. finite-time blowup is stronger than merely “not globally bounded”; positive stable growth is stronger than ordinary unboundedness; sharper constants, rates, exceptional-set bounds, endpoint inclusion, or full equivalences are stronger than weaker asymptotic versions. +- Compare all options before committing. The correct choice is often the strongest statement justified by the question, while nearby distractors are weaker, overstrong, or miss an equality case. +- Track exact quantifiers such as "there exists", "for every", "if and only if", and "exactly when". + +## Theorem-Level Precision + +- Do not add converse, realization, or classification claims unless the theorem explicitly proves them. Phrases such as “conversely,” “every such parameter occurs,” “if and only if,” or “exactly all” add strength beyond a one-way implication. +- Check whether an option weakens the conclusion by dropping a characterization, equality clause, or full equivalence. +- Check whether an option overstates the theorem by upgrading regularity, removing scale restrictions, or changing an existential statement into a universal one. + +## Hypotheses + +### Exact Conditions and Thresholds +- For biconditional/equivalence questions, reject conditions that are merely necessary or merely sufficient. A broader condition, such as congruence modulo a divisor instead of modulo the full modulus, is usually weaker and not equivalent unless the domain collapses the extra cases. +- For threshold conditions, verify the exact sign and endpoint: distinguish \(\mu_0\) from \(-\mu_0\), \(<\) from \(\le\), and whether the equality case belongs to the positive, zero, or negative parameter regime. +- When options differ by “for every” vs “for sufficiently large,” local vs global domains, strict vs non-strict inequalities, or dependence of constants, rank them by logical strength and match the sharpest justified version. +- Verify the hypotheses and domain carefully. Distractors often keep the theorem shape but alter the required assumptions. +- Pay close attention to equality cases, extremal conditions, and whether a result applies to the full family or only a restricted subfamily. + +## Final Answer +- Output the final answer as the single option label only. + +## Exact Scope and Quantitative Wording +- Distinguish global conclusions from localized or completed ones. Equivalence after localization, completion, or at each prime/scale is usually weaker than an unqualified equivalence. +- In estimate-heavy options, compare every quantitative detail: exponent, derivative index range, constants and their parameter dependence, log factors, additive terms, one-sided vs two-sided notation, and pointwise vs uniform convergence. + + + \ No newline at end of file diff --git a/ckpt/officeqa/gpt5.5_skill.md b/ckpt/officeqa/gpt5.5_skill.md new file mode 100644 index 0000000..869c1d4 --- /dev/null +++ b/ckpt/officeqa/gpt5.5_skill.md @@ -0,0 +1,50 @@ +# OfficeQA Skill + +## Retrieval Discipline + +- When an external official time-series observation is needed, prefer the source's series/data-download/table page once identified. If exact-date or guessed-value searches return empty results, stop repeating them; broaden to the official series name/code plus `data` or `download` and use the table values. + +- Treat provided/oracle parsed pages as primary evidence: if they contain the relevant table and period, extract directly from them before searching elsewhere; search only for missing continuation pages, missing periods, or an official actual value not present. +- Start by narrowing to the most likely candidate file before reading long passages. +- Prefer targeted search terms that name the exact entity, period, measure, or table concept from the question. +- After a promising match, read only a small surrounding span and verify it matches the requested year, basis, and unit. + +- If the requested date range extends beyond the provided/oracle page, first enumerate the required periods and verify that every period is present in evidence. Do not compute from a partial ledger or fill missing periods from memory; retrieve continuation pages, adjacent issues, or a later issue of the same table that contains the missing dates/revisions. + +## Evidence Discipline +- Extract the exact value from the retrieved text before doing any arithmetic. +- Keep track of each operand's period, unit, and semantic role so nearby proxy values are not mixed in. + +- For Treasury financing narratives, label each amount by transaction role before calculating: offered amount, tenders/subscriptions received, tenders accepted, competitive/noncompetitive accepted, foreign or Government-account exchange tenders, refunding, and **new cash** are not interchangeable. +- When converting currencies or scales, make a direction ledger first: source table unit, source currency, exchange-rate orientation (foreign currency per U.S. dollar means divide by the rate; U.S. dollars per foreign unit means multiply), and requested final unit. + +- For tables, align values by row label and exact column header, not proximity alone; watch for continued or unlabeled columns, footnotes, adjacent amount-versus-percent columns, fiscal-year versus calendar-year sections, and repeated month rows under different year blocks. +- If the question asks for a transformed or derived quantity, compute only after confirming every operand. + +- For derived comparisons, preserve the direction and sign implied by the wording: “change from A to B” means B minus A; “former than latter” means former minus latter; “share accounted for by X” means X divided by the stated total; paired “gap” questions require computing each within-row difference before ranking. +- For statistical, regression, correlation, and growth-rate questions, write a formula ledger before calculating: confirm the exact series/endpoints, ordered vector, elapsed intervals, and requested convention such as continuously compounded rate, CAGR, Pearson correlation, or OLS index/year choice. +- For multi-stage questions where one table determines the period/entity used in another lookup, freeze that derived key with evidence first, then retrieve the second measure only for that exact month/year/reporting date/entity. + +- For inclusive time-series ranges, make a period-by-period ledger covering every requested month/year exactly once, preserving calendar versus fiscal basis, end-of-month or end-of-fiscal-month status, source units, and any specified adjustments. + +- For statistical transforms over time-series windows, confirm endpoint inclusion/exclusion exactly as worded, use consecutive time indices for trend regressions when appropriate, sort values before medians, and for logarithmic growth use ln(final/initial) before converting to the requested percentage format. + +## Final Answer Discipline + +- Before finalizing, enforce the requested unit and format: convert thousands/millions/billions or full nominal dollars as needed, then apply no-comma, fixed-decimal, whole-number, or nearest-tenth/thousandth formatting exactly as asked. +- Return the final answer only after one last consistency check against the retrieved evidence. +- Copy the final answer from a checked value, not from an unverified intermediate guess. + +## Statistical and Time-Series Calculation Checks + +- Before computing any statistic, write the intended formula and denominator convention. If the prompt explicitly says **population standard deviation**, divide by `n`; if it says **sample**, divide by `n-1`; for a z-score comparing one observation against a small set of comparison months/periods and no population convention is stated, estimate dispersion with the **sample** standard deviation of the comparison set. Do not round intermediate operands, weighted averages, logs, exchange-rate conversions, or standard deviations before the final requested rounding. +- For long inclusive ranges, first enumerate the expected count of observations and the first/last period, then verify the ledger has exactly that count. Exclude totals, cumulative-to-date columns, comparable-period columns, estimates, and extra latest-month columns outside the requested calendar or fiscal range. +- When a page contains multiple nearby sections with similar labels, use only the section whose title and row label match the requested measure exactly; do not compute from the first visible table if the requested measure/table title is absent or only partially shown. +- For Treasury security quotations, obey the table's quote basis. If the table states that price decimals are 32nds, convert quotes such as `99.27` as `99 + 27/32`, not as decimal `99.27`. If a task asks for smoothing, averaging, or forecasting in a target currency using period-specific exchange rates, convert each period's observation to the target currency first unless the prompt explicitly says to compute in the source currency and convert only the final result. + +## Stricter Final Formatting + +- Match any requested output template exactly. Unless the prompt explicitly asks for unit words or explanatory text, return only the numeric value or requested list; do not append words such as `million`, `dollars`, `percent`, or `percentage points`. Include symbols/commas only when the prompt requests currency-formatted output or the answer format clearly requires them. + + + diff --git a/ckpt/searchqa/gpt5.5_skill.md b/ckpt/searchqa/gpt5.5_skill.md new file mode 100644 index 0000000..b58d10a --- /dev/null +++ b/ckpt/searchqa/gpt5.5_skill.md @@ -0,0 +1,71 @@ +# Question Answering Skill + +(No learned rules yet. Rules will be added through the reflection process.) + +## Concise Answer Normalization +- Prefer the shortest unambiguous answer that directly satisfies the question. Do not include generic descriptors, legal suffixes, or expanded formal names unless the question specifically asks for the full official name or the descriptor is necessary to identify the entity. + +- If the answer appears inside a longer descriptive phrase, strip words that merely repeat the clue's requested type or modifiers already stated in the clue. For short-answer trivia, return the distinctive core entity or headword rather than role titles, product flavor adjectives, or place/facility designators, even when those words are part of a fuller official phrase, unless the full official name is explicitly requested. +- For place/name-etymology questions asking for “the name” or “the word” that means something, answer the distinctive name/word itself rather than a larger phrase with a generic type label. + +- For natural geographic features, preserve conventional feature designators such as “Lake,” “River,” “Bay,” “Gorge,” “Mount,” or “Island” when they are part of the proper name or match the requested feature type. Do not shorten “Lake Okeechobee,” “Tampa Bay,” or “Olduvai Gorge” to an ambiguous base name merely to be concise. +- For companies, brands, and organizations, answer the common distinctive name when sufficient; omit additions such as “Company,” “Corporation,” “Inc.,” etc. unless explicitly required. + +- Preserve the answer surface form supported by the strongest evidence when exact variants differ: spelling, capitalization, punctuation, and word order can matter. Do not substitute an equivalent official/common variant such as an alternate spelling or inverted institution name if a direct title/snippet/answer field gives the expected form. + +- When copying titles or quoted names, preserve ordinary ASCII punctuation from the evidence, especially straight apostrophes (`'`). Do not replace them with typographic curly quotes/apostrophes unless that exact stylized form is explicitly shown as the supported answer. + +- For nicknames, epithets, saints, and quoted titles, copy the supported surface form exactly, including spacing, capitalization, and conventional abbreviations such as “St.” Do not normalize a stylized or quoted form into a lowercase dictionary word or an expanded spelling when the clue/evidence points to the stylized answer. + +- For person answers in trivia or crossword-style clues, prefer the conventional supported name. Use just a surname, first name, or saint/regnal name only when the clue/source clearly expects that short form; otherwise use the canonical full personal name from the strongest evidence or answer field, especially when a lone given name would be ambiguous. +- Return the grammatical base form expected by the clue. Do not add a plural `s` merely because a crossword source pluralizes a shared name or category; if the clue lists people sharing a first name, answer the singular given name. + +- For common-noun category answers, default to the singular dictionary headword in trivia/crossword-style clues, even if the clue uses plural words like “these,” “those,” “places,” or “items” for grammar. Use a plural only when the term is inherently plural or an answer field/source clearly gives a plural phrase. + +- For common-noun clues about things being replaced, used in place of, or substituted by another system/item, answer the broad headword for the thing replaced unless a narrowing modifier is required by the clue or answer field. Do not add adjectives such as “letter,” “regular,” or “standard” merely because they appear in explanatory context. +- For fill-in-the-blank or definitional clues using words like “this” or “that,” provide a standalone noun phrase. Avoid context-dependent pronouns or possessives from the source text; use a natural article such as “the” when needed (e.g., answer “the highest point,” not “its highest point”). + +## Context-Grounded Evidence Matching +- Start by identifying the most distinctive terms in the question: proper names, dates, titles, quoted phrases, unusual words, roles, relationships, and category descriptors. +- Prioritize passages or document titles where several distinctive clue terms occur together, especially if the wording directly repeats or closely paraphrases the question. +- Treat document titles as useful evidence: the answer is often named in a title while the snippet confirms the clue facts. + +- Do not assume the document title itself is the answer. If the requested type differs from the title entity, use the title as context and extract the matching typed entity from the snippet or clue relationship. + +- For “known as,” “called,” “defined as,” or category/type clues, choose the canonical term explicitly used in the strongest matching title/snippet or scraped answer field rather than inventing a related derivative or near-synonym from the clue wording. When multiple plausible candidates appear, prefer the candidate whose evidence directly states the requested relationship and repeats the most distinctive clue facts. +- Ignore noisy results that only match generic words; prefer evidence that directly connects the clue facts to one specific entity. + +## Clue Interpretation and Answer Type +- For Jeopardy-style wording such as “this man,” “this group,” “this film,” “this country,” “this system,” “he,” or “his wife,” infer the expected answer type before choosing the answer. +- Use that expected type to validate candidates: answer with the concise person, place, title, organization, object, term, or phrase requested by the clue. + +- Treat modifiers attached to the requested type as hard filters, not background flavor: constraints like dates, “largest,” “2-letter-named,” “1978 remake,” “hot dog brand,” “dual throne,” or “on this company’s board” must all fit the candidate before you answer. +- For clues centered on creative works such as books, films, plays, songs, poems, or other media, first determine whether the clue asks for the work itself, its creator, a performer or cast member, a character, a quotation source, or a setting. Verbs such as “wrote,” “directed,” “stars,” “played,” and “set in,” plus pronouns like “he” or “her,” usually determine the target. + +- For fill-in-style clues with placeholders such as “this,” “these,” or “one of these,” substitute each candidate back into the clue and choose the concise answer that makes the full phrase, title, or fact read correctly. +- For terse clues that are just examples or names separated by commas, slashes, or “or,” infer the shared category, class, or synonym that links them, then answer with that concise common term. + +- For crossword-style clues, treat parenthetical numbers or stated letter counts as hard constraints on the answer length, and omit generic labels that would violate them. In dual-definition clues using wording like “X, or what Y does,” choose the single word that satisfies both senses and preserve the required inflected form. +- If the clue references an unavailable image or link with wording like “seen here,” “pictured,” or parenthetical visual hints, rely on the textual clues and context to infer the answer; do not treat the missing image as necessary evidence. +- If multiple snippets support the same entity, use that corroboration to choose the canonical/common form of the answer. + +## Trivia / Jeopardy Snippet Formats +- Retrieved trivia snippets may contain the clue and answer in scraped formats such as `CATEGORY | clue | answer`, `clue. ANSWER`, or labels like `right:`. +- When the question text matches the clue in such a snippet, extract the answer field or adjacent answer name, not the category or the whole clue sentence. + +## Common Clue Traps +- Watch for inverse relationships: if the clue says “His third wife was Jiang Qing,” the requested answer is the husband, not Jiang Qing. + +- More generally, preserve relation direction in clues: “A is evidence of this B,” “A is related to this language,” or “home to these characters” asks for the target of the relationship, not the entity already named in the clue. + +- When a clue says examples, models, breeds, members, or items “include,” “like,” or “such as” named entities, treat those names as evidence for the requested parent class or entity. Answer the encompassing brand, animal, category, place, or term requested by “this,” not one of the examples already given. +- If the question gives the start of a quotation or phrase, answer with the exact missing continuation from the context. + +- For song, poem, nursery-rhyme, or quotation clues, first decide whether the question asks for a missing word or phrase from the quote or for the associated creator, performer, or work; use pronouns and answer-type signals to choose the right target. +- When a clue asks for a constrained form such as a first name, abbreviation, acronym, or lyric word, return that exact form rather than the fuller person, title, or explanation; preserve conventional punctuation or spelling when it is part of the requested form. +- If the clue contains wordplay, quotation marks, or puns, treat them as hints, but answer with the real entity supported by the evidence. + +- If a clue includes a quoted title, quoted narration or lyric, named event, slogan, or other distinctive phrase but asks for an associated “this” entity, treat the quote or name as evidence to identify the requested person, work, place, group, category, source, or term; do not return the quoted anchor unless the clue explicitly asks for it. + + + diff --git a/ckpt/spreadsheetbench/gpt5.5_skill.md b/ckpt/spreadsheetbench/gpt5.5_skill.md new file mode 100644 index 0000000..9584d2d --- /dev/null +++ b/ckpt/spreadsheetbench/gpt5.5_skill.md @@ -0,0 +1,133 @@ +# Spreadsheet Manipulation Skill (xlsx) + +## Overview +This skill guides agents in manipulating Excel (.xlsx) spreadsheets using Python. + +**Primary libraries**: `openpyxl` (structure-preserving read/write), `pandas` (data transformation). +Never use any other third-party libraries. + +--- + +## Common Workflow + +1. **Explore** the input file: list sheets, inspect headers, check dimensions. + +- Inspect actual workbook data beyond the preview, including nearby rows/columns, sample outputs, formulas, labels, headers, and any reference/example sheets such as `Output`, `Manual Result`, or `Desired...` tabs. + +- Treat existing filled cells in the requested output area or adjacent example tables as semantic examples for edge cases and expected formats, but still recompute and write the complete requested target range. + - Scan the used range for complete header groups, not just row 1. Tables may start in later rows/columns, have title rows above them, or have multiple source/result tables on the same sheet; use nearby labels and the requested output range to distinguish sources from destinations. + - Locate tables, fields, and target ranges by header text, nearby labels, and surrounding nonblank structure rather than fixed coordinates. Build header maps from actual cells when useful, e.g. `{str(cell.value).strip(): cell.column}`. +2. **Write `solution.py`** with `INPUT_PATH` and `OUTPUT_PATH` defined at the top. +3. **Execute** `python solution.py` and verify the output file was created. +4. **Confirm** the target cells/range contain the expected values. + +--- + +## Library Selection + +| Use case | Library | +|----------|---------| +| Preserve formulas, formatting, named ranges | `openpyxl` | +| Bulk data transformation, aggregation, sorting | `pandas` → write back with `openpyxl` | +| Simple cell read/write | `openpyxl` | + +**Warning**: `pandas.to_excel()` silently destroys existing formulas and named ranges. +When writing back to a spreadsheet that contains formulas, always use `openpyxl.save()`. + +**Formula evaluation caution**: `openpyxl` can write formulas but does **not** calculate them or update cached results. If the requested output will be checked as cell values, compute the result in Python and write literal values unless the user explicitly requires live formulas. When existing formulas are inputs to your logic, load a second workbook with `data_only=True` to read cached values while saving changes through the normal workbook: + +```python +wb = openpyxl.load_workbook(INPUT_PATH) +wb_values = openpyxl.load_workbook(INPUT_PATH, data_only=True) +ws = wb["Sheet1"] +ws_values = wb_values["Sheet1"] +``` + +Treat wording such as “write/fix a formula,” “SUMIFS/COUNTIFS,” “VBA,” or “macro” as a description of the spreadsheet logic unless the deliverable explicitly requires live formula text, an `.xlsm`, or a preserved VBA project. For normal `.xlsx` outputs, implement the equivalent logic in Python/openpyxl and write the computed final values to the requested cells so verification does not depend on Excel recalculation or macros. + +When the user provides an existing or broken formula, use it as a semantic specification: honor its referenced lookup ranges, criteria ranges, return ranges, aggregation intent, and error-handling behavior, then write the resulting values rather than guessing different source columns or leaving unevaluated formulas. + +--- + +## solution.py Template + +```python +import openpyxl +import pandas as pd + +INPUT_PATH = "..." # set to the actual input path +OUTPUT_PATH = "..." # set to the actual output path + +wb = openpyxl.load_workbook(INPUT_PATH) +ws = wb.active # or wb["SheetName"] + +# --- perform manipulation --- + +wb.save(OUTPUT_PATH) +``` + +--- + +## Output Requirements + +- Save the result to `OUTPUT_PATH`. +- Do not hardcode row counts or column letters — iterate over actual rows in the workbook. +- Preserve sheets and cells not mentioned in the instruction. + +## Matching and Target Range Hygiene + +- Choose the comparison operator from the instruction and examples: use `startswith` for “begins with”, substring search for “contains/search/occurrence”, and exact normalized equality only when a whole-cell match is implied. +- Create small helper functions for comparisons and numeric parsing. Normalize text by trimming, collapsing repeated spaces/NBSPs, and casefolding; when names or labels have punctuation/spacing inconsistencies, consider punctuation-insensitive keys. Parse numeric text after removing commas/currency symbols while preserving signs and decimal points; skip `None`/blank and booleans for numeric tests, and handle placeholders such as `"-"`, `"$"`, `"$0"`, blanks, and numeric zero deliberately. +- Normalize date keys deliberately: handle `datetime`/`date` objects, Excel serial numbers, and date-like strings, then compare at the granularity implied by the task, such as exact date, month, month/year, fiscal period, or year. For workday/date-window logic, compute the range in Python and exclude weekends/holidays as specified. + +- For monthly or period summary grids, canonicalize period labels from all sources: sheet names, title text, row/column headers, text months such as `March`, and actual date cells. Match summaries by normalized period plus the other stated criteria rather than by fixed month offsets or existing formulas. +- For date ranges and rolling windows, infer endpoint inclusivity from wording and examples. Phrases like `X to Y`, `through`, and `up to`, or examples such as `2 to 5` meaning `4 days`, usually require inclusive boundary handling. +- For time extraction or time-threshold logic, parse `datetime`, `time`, Excel serial/fractional times, and time-like strings into real Python `time`/`datetime` values. Write real time values with an Excel `number_format` such as `hh:mm:ss AM/PM`; do not write text substrings when the result should behave as a time. +- For joins, deduplication, grouping, interval lookups, lookup grids, and ordered outputs, build explicit normalized keys, including composite keys when the task refers to multiple fields. Preserve original source order within each group unless sorting is explicitly requested. +- For outputs that depend on other rows or lookup grids, make a first pass to build normalized dictionaries/groups/range structures, then a second pass to write results. Avoid nested full-sheet scans per row; split delimited tokens and ignore empty tokens, and treat error literals such as `#N/A` as meaningful sentinel values when the task refers to them. + +- For lookups, filters, joins, and label/header matching, normalize comparison keys consistently: trim whitespace, skip blanks explicitly, use case-insensitive text matching when appropriate, and treat numeric-looking IDs consistently (`330`, `330.0`, and `"330"`). Keep numeric outputs numeric; use `number_format` for display formatting instead of converting numbers to strings unless text is explicitly required. +- When replacing a generated output area, clear only the instructed target range before writing new results so stale values/formulas do not remain. Preserve formatting, column widths, borders, formulas, and unrelated cells unless the instruction explicitly asks to change them. + +- If the instruction includes formatting changes, apply them exactly after writing values and only to the requested cells/range. Use `openpyxl` styles for fills, alignment, fonts, borders, and number formats; convert hex colors to ARGB when needed, for example `#FFC000` → `FFFFC000`. For “format as text,” set `number_format = '@'` and write string values when the expected cell values are text. + +- When the instruction names a destination range or columns, write derived results directly there. Do not insert rows/columns, relocate the source table, or sort/delete source records unless that structural change is explicitly requested. +- For filtered lists, summaries, and aggregations, first collect all source records/results in memory, preserving the required order, then write from the first output row and clear leftover cells below the new results in the target columns. When adding rows, copy style/alignment/number format from an existing template row when appropriate; when deleting rows, delete from bottom to top to avoid row-index shifts. +- Preserve intended blanks as empty cells (`None`) rather than placeholder text or `0` unless the task specifies otherwise. + +- For numeric aggregation, crosstab, SUMIFS-like, and INDEX/MATCH-style summary outputs, infer missing-match behavior from table semantics and examples: numeric summary grids usually require literal `0` for no matching records, while filtered lists or “show only once” outputs usually require blanks (`None`). +- For blank-sensitive logic such as “if input is blank, output blank,” evaluate the driving input with `data_only=True` when it may itself be a formula, and write `None` for truly blank outputs rather than relying on a new formula returning `""`. + +## Robustness for Simple Fill Tasks + +- Prefer simple, auditable row/column loops over complex workbook XML parsing unless the task truly requires unsupported workbook internals. Before returning, run the script once to catch syntax/indentation errors and verify that representative target rows were actually written. + + +When the user asks for a formula, macro, VBA code, or a fix to an Excel formula, still deliver the completed workbook state: compute the intended results in Python and write literal final values into the requested cells. Do not write formula strings unless the task explicitly says the output must contain live formulas. + +After writing, reload or inspect the saved workbook and verify that every requested/evaluated target cell contains a non-formula literal where a value is expected. If a target cell is still `None` unexpectedly, fix the script before finishing. + +Use existing formulas in the workbook as examples/specifications, not as output. If a cell contains a reference formula such as `=A25` or an INDEX/MATCH/SUMIFS pattern, parse what source cells/ranges/criteria it refers to, compute those results yourself, and overwrite the destination with the referenced or calculated value. + +For blank-sensitive formula tasks, compute the branch explicitly: if the driving source cell is truly blank, write `None`; otherwise write the actual result such as `0`, `1`, a category label, or a lookup value. Never rely on `IF(...,"",...)` formulas to be recalculated later. + +For lookup/category tasks, locate both the input rows and the lookup table by headers and nearby labels. Support exact keys, numeric-looking keys, and interval/range tables; then fill every destination row that has a driving input, not just the first visible example. + +For “every nth row” or OFFSET-style tasks, infer the source column, first source row, and step from the provided examples or formulas, then copy the actual source values into the requested output range as literals. + +For schedule/calendar fill tasks, build a cycle-day-to-periods mapping from the schedule/template area first, then fill the daily rows across all requested class columns based on each row’s cycle day. Preserve repeated/double periods exactly as shown by the template; do not leave formulas in the schedule cells. + +For INDEX/MATCH problems where the first row works but subsequent rows fail, treat row labels, column/year headers, region/type criteria, and expense/category labels as a multi-key lookup. Fill the whole result matrix with values from the source data table, using cached `data_only` values when source cells are formulas. + +For multi-step macro/VBA-style requests, implement every stated operation in the workbook, not just the first deletion/filtering step. Re-read the numbered requirements before saving and verify later computed columns, totals, and derived fields as well as the obvious filtered rows. + +When a target range includes special rows such as `Total`, `Grand Total`, `min`, `max`, constraints, headers, or blank separators, do not apply ordinary row logic blindly to those rows. Compute totals as aggregates when indicated, and leave constraint/header/blank cells untouched unless explicitly requested. + +For residual-balancing tasks, identify data rows separately from min/max constraint rows. Add positive residuals from unit 1 toward unit 5 without exceeding max values; subtract negative residuals from unit 5 toward unit 1 without going below min values; update only the unit cells in actual data rows. + +For time-threshold rows, decide per row whether it is a normal data row or a summary row. Normal rows use the before/after threshold rule; summary rows should aggregate the computed normal-row results if the workbook labels or examples indicate a total. + +Keep scripts simple enough to run cleanly. Avoid unnecessary dynamic code generation and fragile f-strings with regex expressions inside them. Always execute the final `solution.py`; fix any syntax, indentation, or runtime error, then verify representative target cells. + +If workbook cells contain arbitrary sample text that could be sensitive or trigger content filters, do not quote large raw cell contents in your response. Process them locally in Python with neutral variable names and output only the completed script/workbook changes. + diff --git a/configs/_base_/default.yaml b/configs/_base_/default.yaml new file mode 100644 index 0000000..fcfdd71 --- /dev/null +++ b/configs/_base_/default.yaml @@ -0,0 +1,103 @@ +# SkillOpt default configuration — base for all environments. +# Environment configs should inherit via: _base_: default.yaml + +model: + backend: azure_openai + optimizer: gpt-5.5 + target: gpt-5.5 + optimizer_backend: openai_chat + target_backend: openai_chat + reasoning_effort: medium + rewrite_reasoning_effort: "" + rewrite_max_completion_tokens: 64000 + codex_exec_path: codex + codex_exec_sandbox: workspace-write + codex_exec_profile: "" + codex_exec_full_auto: false + codex_exec_reasoning_effort: none + codex_exec_use_sdk: auto + codex_exec_network_access: false + codex_exec_web_search: false + codex_exec_approval_policy: never + claude_code_exec_path: claude + claude_code_exec_profile: "" + claude_code_exec_use_sdk: auto + claude_code_exec_effort: medium + claude_code_exec_max_thinking_tokens: 16384 + codex_trace_to_optimizer: true + azure_openai_endpoint: "" # e.g. "https://your-resource.openai.azure.com/" + azure_openai_api_version: "2024-12-01-preview" + azure_openai_api_key: "" # Fill locally if you do not export AZURE_OPENAI_API_KEY + azure_openai_auth_mode: "" # empty → fall back to AZURE_OPENAI_AUTH_MODE env (default "azure_cli") + azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default" + azure_openai_managed_identity_client_id: "" + optimizer_azure_openai_endpoint: "" # e.g. "https://your-resource.openai.azure.com/" + optimizer_azure_openai_api_version: "2024-12-01-preview" + optimizer_azure_openai_api_key: "" + optimizer_azure_openai_auth_mode: "" # empty → fall back to OPTIMIZER_AZURE_OPENAI_AUTH_MODE env, then shared + optimizer_azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default" + optimizer_azure_openai_managed_identity_client_id: "" + target_azure_openai_endpoint: "" # e.g. "https://your-resource.openai.azure.com/" + target_azure_openai_api_version: "2024-12-01-preview" + target_azure_openai_api_key: "" + target_azure_openai_auth_mode: "" # empty → fall back to TARGET_AZURE_OPENAI_AUTH_MODE env, then shared + target_azure_openai_ad_scope: "https://cognitiveservices.azure.com/.default" + target_azure_openai_managed_identity_client_id: "" + + # MiniMax backend settings (minimax_chat target) + minimax_base_url: "" # https://api.minimax.io/v1 if blank + minimax_api_key: "" + minimax_model: "MiniMax-M2.7" + minimax_temperature: "0.7" + minimax_max_tokens: "8000" + minimax_enable_thinking: "false" + optimizer_minimax_base_url: "" # per-role override + target_minimax_base_url: "" # per-role override + optimizer_minimax_api_key: "" + target_minimax_api_key: "" + +train: + num_epochs: 4 + train_size: 0 # 0 = derive from dataset split when available + batch_size: 40 + accumulation: 1 + seed: 42 + +gradient: + minibatch_size: 8 + merge_batch_size: 8 + analyst_workers: 16 + max_analyst_rounds: 3 + failure_only: false + +optimizer: + learning_rate: 4 # max edits per step (edit_budget) + min_learning_rate: 2 # min edits for decay schedulers + lr_scheduler: cosine # constant / linear / cosine / autonomous + lr_control_mode: fixed # fixed / autonomous / none + skill_update_mode: patch # patch / rewrite_from_suggestions / full_rewrite_minibatch + use_slow_update: true + slow_update_samples: 20 + slow_update_gate_with_selection: false + longitudinal_pair_policy: mixed # mixed / changed / unchanged + use_meta_skill: true + use_skill_aware_reflection: false # EmbodiSkill: split failures into SKILL_DEFECT (edit body) vs EXECUTION_LAPSE (protected appendix) + skill_aware_appendix_source: both # both = success+failure emit appendix notes; failure_only = only EXECUTION_LAPSE (paper-faithful) + skill_aware_consolidate_threshold: 0 # 0 = off; >0 = LLM-consolidate the appendix when its note count exceeds N + +evaluation: + use_gate: true + sel_env_num: 0 + test_env_num: 0 + eval_test: true + +env: + name: "" + skill_init: "" + split_mode: ratio # ratio = build deterministic split from data_path; split_dir = use pre-split train/val/test + split_seed: 42 + split_dir: "" + data_path: "" + split_output_dir: "" + exec_timeout: 120 # per target model/code-agent call timeout in seconds + out_root: "" diff --git a/configs/alfworld/default.yaml b/configs/alfworld/default.yaml new file mode 100644 index 0000000..9504140 --- /dev/null +++ b/configs/alfworld/default.yaml @@ -0,0 +1,29 @@ +_base_: ../_base_/default.yaml + +train: + train_size: 0 + accumulation: 1 + +gradient: + minibatch_size: 8 + merge_batch_size: 8 + +optimizer: + learning_rate: 4 + +evaluation: + sel_env_num: 0 + test_env_num: 0 + +env: + name: alfworld + skill_init: skillopt/envs/alfworld/skills/initial.md + split_mode: split_dir + split_dir: data/alfworld_path_split + data_path: "" + split_output_dir: "" + max_steps: 50 + max_completion_tokens: 16384 + workers: 8 + max_api_workers: 8 + limit: 0 diff --git a/configs/docvqa/default.yaml b/configs/docvqa/default.yaml new file mode 100644 index 0000000..c3e8ce0 --- /dev/null +++ b/configs/docvqa/default.yaml @@ -0,0 +1,28 @@ +_base_: ../_base_/default.yaml + +model: + reasoning_effort: medium + +train: + batch_size: 40 + accumulation: 1 + +gradient: + minibatch_size: 8 + merge_batch_size: 8 + +optimizer: + learning_rate: 4 + +env: + name: docvqa + skill_init: skillopt/envs/docvqa/skills/initial.md + split_mode: split_dir + split_dir: data/docvqa/splits + data_path: "" + split_output_dir: "" + max_turns: 1 + max_completion_tokens: 16384 + workers: 16 + image_detail: auto + limit: 0 diff --git a/configs/features/soft_gate.yaml b/configs/features/soft_gate.yaml new file mode 100644 index 0000000..7b622d3 --- /dev/null +++ b/configs/features/soft_gate.yaml @@ -0,0 +1,47 @@ +# ───────────────────────────────────────────────────────────────────────────── +# Feature: soft / mixed validation-gate metric (community-contributed, PR #25) +# ───────────────────────────────────────────────────────────────────────────── +# +# This is NOT a default SkillOpt setting and was NOT used to produce the +# numbers reported in the paper. It is provided as a reference for users +# who encounter a specific scenario where the default `hard` gate is too +# coarse to drive training. +# +# When to consider this: +# - You are running on a custom environment. +# - Your held-out *selection* split has very few items (e.g. ≤ ~10). +# - Your reward function is continuous / partial-credit (e.g. F1, BLEU, +# soft match) rather than purely binary 0/1. +# +# Symptom this addresses: +# With a small selection split + continuous rewards, candidate skills +# often improve per-item soft scores (e.g. 0.06 → 0.26 on one item) but +# never flip the discrete hard outcome. The default `hard` gate then +# rejects every candidate and training stalls. Switching the gate to +# `soft` or `mixed` lets these partial improvements be accepted. +# +# When NOT to use this: +# - When reproducing the paper. The paper-reported numbers were obtained +# under the default `hard` gate. +# - When your selection split is large (dozens+ items) and / or your +# reward is already binary — `hard` is the more conservative choice +# and matches the design described in the paper. +# +# To use: inherit your env config from this file, e.g. +# _base_: ../features/soft_gate.yaml +# or copy the `evaluation:` block below into your config. +# ───────────────────────────────────────────────────────────────────────────── + +_base_: ../_base_/default.yaml + +evaluation: + # Three options: + # 'hard' — default; exact-match accuracy. Use this to reproduce the paper. + # 'soft' — per-item soft / partial-credit score (recommended for the + # small-split + continuous-reward scenario described above). + # 'mixed' — weighted average: (1 - w) * hard + w * soft, with `w` set by + # `gate_mixed_weight` below. + gate_metric: soft + + # Only used when gate_metric == 'mixed'. Ignored otherwise. + gate_mixed_weight: 0.5 diff --git a/configs/livemathematicianbench/default.yaml b/configs/livemathematicianbench/default.yaml new file mode 100644 index 0000000..19401ab --- /dev/null +++ b/configs/livemathematicianbench/default.yaml @@ -0,0 +1,22 @@ +_base_: ../_base_/default.yaml + +train: + train_size: 0 + batch_size: 40 + accumulation: 1 + +env: + name: livemathematicianbench + skill_init: skillopt/envs/livemathematicianbench/skills/initial.md + split_mode: split_dir + split_dir: data/livemathematicianbench_split + data_path: "" + split_output_dir: "" + max_turns: 1 + max_completion_tokens: 16384 + exec_timeout: 300 + workers: 64 + limit: 0 + shuffle_choices: true + use_theorem: false + use_sketch: false diff --git a/configs/officeqa/default.yaml b/configs/officeqa/default.yaml new file mode 100644 index 0000000..7b72f1a --- /dev/null +++ b/configs/officeqa/default.yaml @@ -0,0 +1,34 @@ +_base_: ../_base_/default.yaml + +model: + reasoning_effort: medium + +train: + batch_size: 40 + accumulation: 1 + +gradient: + minibatch_size: 8 + merge_batch_size: 8 + +optimizer: + learning_rate: 4 + +env: + name: officeqa + skill_init: skillopt/envs/officeqa/skills/initial.md + split_mode: split_dir + split_dir: data/officeqa_split + data_dirs: + - data/officeqa_docs_official + workers: 4 + max_tool_turns: 24 + max_completion_tokens: 16384 + search_mode: offline + max_queries_per_turn: 4 + search_api_url: http://apisix.westus2.cloudapp.azure.com/search_tool/search + search_auth_env: OFFICEQA_CUSTOM_SEARCH_AUTH + search_provider: duckduckgo + search_max_num_results: 4 + search_timeout_seconds: 20 + limit: 0 diff --git a/configs/searchqa/default.yaml b/configs/searchqa/default.yaml new file mode 100644 index 0000000..a1177ab --- /dev/null +++ b/configs/searchqa/default.yaml @@ -0,0 +1,32 @@ +_base_: ../_base_/default.yaml + +model: + reasoning_effort: medium + +train: + train_size: 400 + batch_size: 40 + accumulation: 1 + +gradient: + minibatch_size: 8 + merge_batch_size: 8 + +optimizer: + learning_rate: 4 + +evaluation: + sel_env_num: 0 + test_env_num: 0 + +env: + name: searchqa + skill_init: skillopt/envs/searchqa/skills/initial.md + split_mode: split_dir + split_dir: data/searchqa_split + data_path: "" + split_output_dir: "" + max_turns: 1 + max_completion_tokens: 16384 + workers: 24 + limit: 0 diff --git a/configs/spreadsheetbench/default.yaml b/configs/spreadsheetbench/default.yaml new file mode 100644 index 0000000..e93c3a3 --- /dev/null +++ b/configs/spreadsheetbench/default.yaml @@ -0,0 +1,34 @@ +_base_: ../_base_/default.yaml + +model: + reasoning_effort: medium + +train: + train_size: 80 + batch_size: 40 + accumulation: 1 + +gradient: + minibatch_size: 8 + merge_batch_size: 8 + +optimizer: + learning_rate: 4 + +evaluation: + sel_env_num: 0 + test_env_num: 0 + +env: + name: spreadsheetbench + skill_init: skillopt/envs/spreadsheetbench/skills/initial.md + split_mode: split_dir + split_dir: data/spreadsheetbench_split + data_path: "" + split_output_dir: "" + data_root: data/spreadsheetbench_verified_400 + mode: multi + max_turns: 30 + max_completion_tokens: 16384 + exec_timeout: 600 + workers: 24 diff --git a/data/README.md b/data/README.md new file mode 100644 index 0000000..a31c337 --- /dev/null +++ b/data/README.md @@ -0,0 +1,237 @@ +# Data Manifests + +This directory releases lightweight split manifests for the SkillOpt paper +splits. These manifests are not full runnable benchmark payloads. To evaluate a +benchmark, first materialize the full examples from the raw data source when +needed, then point `--split_dir` at the split directory listed below. + +In this README, "coverage" describes which part of the upstream benchmark the +manifest references. It does not mean the released manifest directory contains +the full runnable examples. + +## Layout + +Every released manifest directory uses the same file layout: + +```text +data/_/ +|-- split_manifest.json +|-- train/items.json +|-- val/items.json +`-- test/items.json +``` + +`split_manifest.json` records source metadata, split counts, and item fields. +Each `items.json` contains only stable IDs or source-path hints. + +## Released Splits + +| Manifest directory | Benchmark | Counts | Coverage | Raw data source | `split_dir` | +|---|---|---:|---|---|---| +| `searchqa_id_split/` | SearchQA | 400 / 200 / 1400 | Official HF dataset IDs | [lucadiliello/searchqa](https://huggingface.co/datasets/lucadiliello/searchqa) | `data/searchqa_split` | +| `livemathematicianbench_id_split/` | LiveMathematicianBench | 35 / 18 / 124 | Four official monthly files | [LiveMathematicianBench/LiveMathematicianBench](https://huggingface.co/datasets/LiveMathematicianBench/LiveMathematicianBench) | `data/livemathematicianbench_split` | +| `docvqa_id_split/` | DocVQA | 107 / 53 / 374 | 10% subset of validation | [lmms-lab/DocVQA](https://huggingface.co/datasets/lmms-lab/DocVQA) | `data/docvqa/splits` | +| `officeqa_id_split/` | OfficeQA | 50 / 24 / 172 | OfficeQA Full | [databricks/officeqa](https://huggingface.co/datasets/databricks/officeqa) | `data/officeqa_split` | +| `spreadsheetbench_id_split/` | SpreadsheetBench | 80 / 40 / 280 | SpreadsheetBench Verified 400 | [KAKA22/SpreadsheetBench](https://huggingface.co/datasets/KAKA22/SpreadsheetBench) | `data/spreadsheetbench_split` | +| `alfworld_path_split/` | ALFWorld | 39 / 18 / 134 | ALFWorld `json_2.1.1` paths | [alfworld/alfworld](https://github.com/alfworld/alfworld) | `data/alfworld_path_split` | + +Counts are ordered as train / val / test. + +## Direct Use + +Only `alfworld_path_split/` can be used directly as `--split_dir` from this +release, because the ALFWorld loader reads `gamefile` and `task_type` from the +split items. + +This does not mean the ALFWorld raw data is included. You still need to +download ALFWorld separately with `alfworld-download` and set `$ALFWORLD_DATA` +to the data root containing `json_2.1.1`. + +The other manifest directories are lookup manifests. They intentionally omit +full example fields such as questions, answers, contexts, images, or task +instructions. Materialize those benchmarks into the `split_dir` paths listed +above before running SkillOpt. + +## Lookup Keys + +The manifests are sufficient to locate the corresponding raw examples after +the raw data has been downloaded or otherwise made available: + +| Benchmark | Manifest lookup key | +|---|---| +| SearchQA | Match `items.json[].id` to the `key` field in `lucadiliello/searchqa`. | +| LiveMathematicianBench | Open `source_file`, then match `no`; the manifest `id` is `:`. | +| DocVQA | Match `questionId` within the official DocVQA `validation` split; `image_path` records the expected local image path. | +| OfficeQA | Match `uid` in `officeqa_full.csv`; `source_files` and `source_docs` identify the supporting document. | +| SpreadsheetBench | Match `id`; `spreadsheet_path` identifies the referenced spreadsheet directory. | +| ALFWorld | Resolve `gamefile` relative to `$ALFWORLD_DATA`. | + +## Manifest Item Examples + +SearchQA: + +```json +{ + "id": "221c83e6630f4e7983da48fa28da1882" +} +``` + +LiveMathematicianBench: + +```json +{ + "id": "202602:22", + "month": "202602", + "no": 22, + "paper_link": "http://arxiv.org/abs/2602.10700v1", + "source_file": "data/202602/qa_202602_final.json" +} +``` + +DocVQA: + +```json +{ + "id": "50877", + "questionId": "50877", + "docId": "14724", + "image_path": "data/docvqa_images/q50877_d14724.png", + "source_split": "validation" +} +``` + +OfficeQA: + +```json +{ + "id": "UID0002", + "uid": "UID0002", + "category": "easy", + "source_files": "treasury_bulletin_1944_01.txt" +} +``` + +SpreadsheetBench: + +```json +{ + "id": "32438", + "spreadsheet_path": "spreadsheet/32438", + "instruction_type": "Cell-Level Manipulation" +} +``` + +ALFWorld: + +```json +{ + "id": "train:0000", + "gamefile": "json_2.1.1/train/.../game.tw-pddl", + "task_type": "look_at_obj_in_light" +} +``` + +## Benchmark Notes + +### SearchQA + +`searchqa_id_split/` is an ID-only manifest. Each released `id` exactly matches +the `key` field in `lucadiliello/searchqa`. + +To materialize the runnable SearchQA split used by +`configs/searchqa/default.yaml`, install the optional dependency and run: + +```bash +python -m pip install 'skillopt[searchqa]' +python scripts/materialize_searchqa.py +``` + +This writes full examples to: + +```text +data/searchqa_split +``` + +Materialized examples must include the fields consumed by the SearchQA +environment, including: + +```text +question +context +answers +``` + +### LiveMathematicianBench + +`livemathematicianbench_id_split/` was generated from these raw files: + +```text +data/202511/qa_202511_final.json +data/202512/qa_202512_final.json +data/202601/qa_202601_final.json +data/202602/qa_202602_final.json +``` + +The manifest stores IDs in the loader format: + +```text +: +``` + +Materialized examples must include: + +```text +question +choices +correct_choice +theorem_type +theorem +sketch +paper_link +``` + +### DocVQA + +`docvqa_id_split/` records `docvqa_validation_10pct`: a 10% subset sampled from +the official DocVQA `validation` split. + +```text +source_split: validation +docvqa_validation_10pct: train=107, val=53, test=374 +``` + +Each manifest item contains question/document IDs plus image location metadata. +Materialized examples must provide `question`, `answer` or `ground_truth`, and +an `image_path` that resolves locally. + +### OfficeQA + +`officeqa_id_split/` records the split over OfficeQA Full +(`officeqa_full.csv`). The official OfficeQA CSVs are gated on Hugging Face, so +materialization requires authorized access. + +Each manifest item contains `uid`, `category`, `source_files`, and +`source_docs` hints. Materialized examples must include `question` and +`ground_truth` or `answer`. + +### SpreadsheetBench + +`spreadsheetbench_id_split/` records the split over SpreadsheetBench Verified +400, from `spreadsheetbench_verified_400.tar.gz`. + +Each manifest item contains task identity metadata such as `id`, +`spreadsheet_path`, and `instruction_type`. Materialization must also place the +referenced spreadsheet directories at: + +```text +data/spreadsheetbench_verified_400 +``` + +### ALFWorld + +`alfworld_path_split/` records `gamefile` paths relative to `$ALFWORLD_DATA`. +The source payload is `json_2.1.1`, which must be downloaded separately with +`alfworld-download`. + +This manifest can be used directly as `--split_dir` after `$ALFWORLD_DATA` +points to the local ALFWorld data root containing `json_2.1.1`. diff --git a/data/alfworld_path_split/split_manifest.json b/data/alfworld_path_split/split_manifest.json new file mode 100644 index 0000000..46352df --- /dev/null +++ b/data/alfworld_path_split/split_manifest.json @@ -0,0 +1,29 @@ +{ + "benchmark": "ALFWorld", + "manifest_type": "path_split", + "source_repo": "alfworld/alfworld", + "source_repo_type": "repository", + "source_url": "https://github.com/alfworld/alfworld", + "source_file": "json_2.1.1", + "source_method": "generated by alfworld-download", + "source_split_files": [ + "split_train.json", + "split_val.json", + "split_test.json" + ], + "counts": { + "train": 39, + "val": 18, + "test": 134 + }, + "item_fields": [ + "id", + "gamefile", + "task_type" + ], + "path_root": "$ALFWORLD_DATA", + "notes": [ + "This is a path manifest, not the ALFWorld game payload.", + "The gamefile field is relative to ALFWORLD_DATA and must be expanded before direct use as split_dir data." + ] +} diff --git a/data/alfworld_path_split/test/items.json b/data/alfworld_path_split/test/items.json new file mode 100644 index 0000000..bbcdb8b --- /dev/null +++ b/data/alfworld_path_split/test/items.json @@ -0,0 +1,672 @@ +[ + { + "id": "test:0000", + "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-AlarmClock-None-DeskLamp-308/trial_T20190908_222917_366542/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "test:0001", + "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-AlarmClock-None-DeskLamp-308/trial_T20190908_222933_607649/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "test:0002", + "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-AlarmClock-None-DeskLamp-308/trial_T20190908_222951_616606/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "test:0003", + "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Book-None-DeskLamp-308/trial_T20190908_020029_636862/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "test:0004", + "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Book-None-DeskLamp-308/trial_T20190908_020048_814402/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "test:0005", + "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Book-None-DeskLamp-308/trial_T20190908_144951_587345/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "test:0006", + "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Bowl-None-DeskLamp-308/trial_T20190907_133919_856963/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "test:0007", + "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Bowl-None-DeskLamp-308/trial_T20190907_133935_066606/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "test:0008", + "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Bowl-None-DeskLamp-308/trial_T20190907_133953_562557/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "test:0009", + "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-CD-None-DeskLamp-308/trial_T20190908_141942_810052/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "test:0010", + "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-CD-None-DeskLamp-308/trial_T20190908_141958_463362/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "test:0011", + "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-CD-None-DeskLamp-308/trial_T20190908_142046_281296/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "test:0012", + "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Mug-None-DeskLamp-308/trial_T20190908_161733_213242/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "test:0013", + "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Mug-None-DeskLamp-308/trial_T20190908_201421_021646/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "test:0014", + "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Mug-None-DeskLamp-308/trial_T20190908_201444_037645/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "test:0015", + "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Pencil-None-DeskLamp-308/trial_T20190908_220545_153480/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "test:0016", + "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Pencil-None-DeskLamp-308/trial_T20190908_220604_010430/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "test:0017", + "gamefile": "json_2.1.1/valid_unseen/look_at_obj_in_light-Pencil-None-DeskLamp-308/trial_T20190908_220656_510400/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "test:0018", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Mug-None-Desk-308/trial_T20190908_125200_737896/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0019", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Mug-None-Desk-308/trial_T20190909_203041_433487/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0020", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Mug-None-Desk-308/trial_T20190909_210238_431966/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0021", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Pencil-None-Shelf-308/trial_T20190908_121952_610012/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0022", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Pencil-None-Shelf-308/trial_T20190908_122024_052056/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0023", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Pencil-None-Shelf-308/trial_T20190908_122154_042763/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0024", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-PepperShaker-None-Drawer-10/trial_T20190906_184021_215264/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0025", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-PepperShaker-None-Drawer-10/trial_T20190918_154326_823501/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0026", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-PepperShaker-None-Drawer-10/trial_T20190918_154424_844749/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0027", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-SaltShaker-None-Cabinet-10/trial_T20190906_191429_743650/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0028", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-SaltShaker-None-Cabinet-10/trial_T20190906_191445_723170/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0029", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-SaltShaker-None-Cabinet-10/trial_T20190906_191501_563086/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0030", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-SaltShaker-None-Drawer-10/trial_T20190909_021613_077537/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0031", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-SaltShaker-None-Drawer-10/trial_T20190909_021650_880235/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0032", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-SaltShaker-None-Drawer-10/trial_T20190909_021728_339782/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0033", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-SoapBottle-None-Toilet-424/trial_T20190907_004321_405868/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0034", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-SoapBottle-None-Toilet-424/trial_T20190907_004351_281384/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0035", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-SoapBottle-None-Toilet-424/trial_T20190907_004404_604165/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0036", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Vase-None-Safe-219/trial_T20190908_205204_244321/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0037", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Vase-None-Safe-219/trial_T20190908_205221_748352/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0038", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Vase-None-Safe-219/trial_T20190908_205246_776817/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0039", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Watch-None-Safe-219/trial_T20190907_074524_006355/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0040", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Watch-None-Safe-219/trial_T20190907_074556_124850/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0041", + "gamefile": "json_2.1.1/valid_unseen/pick_and_place_simple-Watch-None-Safe-219/trial_T20190907_074643_810052/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "test:0042", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Bowl-None-Cabinet-10/trial_T20190909_061130_844814/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0043", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Bowl-None-Cabinet-10/trial_T20190909_061158_110530/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0044", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Bowl-None-Cabinet-10/trial_T20190909_061232_368489/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0045", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Cloth-None-Cabinet-424/trial_T20190908_022321_380927/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0046", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Cloth-None-Cabinet-424/trial_T20190908_022436_073995/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0047", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Cloth-None-CounterTop-424/trial_T20190908_100632_546757/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0048", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Cloth-None-CounterTop-424/trial_T20190908_114340_674467/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0049", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Egg-None-Microwave-10/trial_T20190909_120554_888709/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0050", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Egg-None-Microwave-10/trial_T20190909_120632_691361/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0051", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Egg-None-Microwave-10/trial_T20190909_120712_273910/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0052", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Knife-None-CounterTop-10/trial_T20190909_110347_624008/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0053", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Knife-None-CounterTop-10/trial_T20190909_110445_675754/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0054", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Knife-None-CounterTop-10/trial_T20190909_110531_148235/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0055", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Mug-None-CoffeeMachine-10/trial_T20190907_221208_560499/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0056", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Mug-None-CoffeeMachine-10/trial_T20190907_221300_362511/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0057", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Mug-None-CoffeeMachine-10/trial_T20190907_221355_558505/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0058", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Pan-None-CounterTop-10/trial_T20190908_032434_013084/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0059", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Pan-None-CounterTop-10/trial_T20190908_032518_891433/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0060", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Pan-None-CounterTop-10/trial_T20190908_032543_712058/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0061", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Plate-None-CounterTop-10/trial_T20190908_213356_017769/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0062", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Plate-None-CounterTop-10/trial_T20190908_213420_728917/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0063", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Plate-None-CounterTop-10/trial_T20190908_213533_897289/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0064", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-SoapBar-None-Cabinet-424/trial_T20190908_214926_337906/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0065", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-SoapBar-None-Cabinet-424/trial_T20190908_214946_567644/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0066", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-SoapBar-None-Cabinet-424/trial_T20190908_215019_162873/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0067", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-SoapBar-None-CounterTop-424/trial_T20190907_074045_109439/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0068", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-SoapBar-None-CounterTop-424/trial_T20190907_074106_050405/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0069", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-SoapBar-None-CounterTop-424/trial_T20190907_074124_966890/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0070", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Spatula-None-Drawer-10/trial_T20190907_080730_211959/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0071", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Spatula-None-Drawer-10/trial_T20190907_080800_275989/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0072", + "gamefile": "json_2.1.1/valid_unseen/pick_clean_then_place_in_recep-Spatula-None-Drawer-10/trial_T20190907_080825_222432/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "test:0073", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Bread-None-CounterTop-10/trial_T20190908_091747_866951/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0074", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Bread-None-CounterTop-10/trial_T20190908_091811_414150/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0075", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Bread-None-CounterTop-10/trial_T20190908_091835_825830/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0076", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Lettuce-None-CounterTop-10/trial_T20190909_123133_763972/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0077", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Lettuce-None-CounterTop-10/trial_T20190909_174807_646433/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0078", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Lettuce-None-CounterTop-10/trial_T20190909_174840_771703/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0079", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Mug-None-Cabinet-10/trial_T20190909_121559_082363/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0080", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Mug-None-Cabinet-10/trial_T20190909_121635_622676/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0081", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Mug-None-Cabinet-10/trial_T20190909_121710_650938/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0082", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Mug-None-CoffeeMachine-10/trial_T20190907_183715_299073/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0083", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Mug-None-CoffeeMachine-10/trial_T20190907_183807_477267/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0084", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Mug-None-CoffeeMachine-10/trial_T20190907_183853_958104/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0085", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Pan-None-CounterTop-10/trial_T20190908_114545_244903/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0086", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Pan-None-CounterTop-10/trial_T20190908_114622_738670/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0087", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Pan-None-CounterTop-10/trial_T20190908_114656_768805/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0088", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Potato-None-Microwave-10/trial_T20190907_033157_424297/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0089", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Potato-None-Microwave-10/trial_T20190907_033228_194678/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0090", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Potato-None-Microwave-10/trial_T20190907_033306_962974/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0091", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Tomato-None-Microwave-10/trial_T20190909_102608_318800/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0092", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Tomato-None-Microwave-10/trial_T20190909_102644_926781/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0093", + "gamefile": "json_2.1.1/valid_unseen/pick_cool_then_place_in_recep-Tomato-None-Microwave-10/trial_T20190909_102710_795182/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "test:0094", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Apple-None-Fridge-10/trial_T20190906_182259_116320/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0095", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Apple-None-Fridge-10/trial_T20190906_182353_418140/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0096", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Apple-None-Fridge-10/trial_T20190906_182435_622538/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0097", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Apple-None-GarbageCan-10/trial_T20190908_145050_918567/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0098", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Apple-None-GarbageCan-10/trial_T20190908_145143_820541/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0099", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Apple-None-GarbageCan-10/trial_T20190908_145356_918528/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0100", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Cup-None-Cabinet-10/trial_T20190907_083346_800823/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0101", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Cup-None-Cabinet-10/trial_T20190907_083429_887065/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0102", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Cup-None-Cabinet-10/trial_T20190907_083507_594820/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0103", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Egg-None-GarbageCan-10/trial_T20190908_113432_673307/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0104", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Egg-None-GarbageCan-10/trial_T20190908_113523_123938/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0105", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Egg-None-GarbageCan-10/trial_T20190908_113610_425142/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0106", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Mug-None-Cabinet-10/trial_T20190909_021100_341887/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0107", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Mug-None-Cabinet-10/trial_T20190909_021200_669381/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0108", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Mug-None-Cabinet-10/trial_T20190909_021247_306737/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0109", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Mug-None-CoffeeMachine-10/trial_T20190907_171806_406231/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0110", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Mug-None-CoffeeMachine-10/trial_T20190907_171850_960211/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0111", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Mug-None-CoffeeMachine-10/trial_T20190907_171933_349922/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0112", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Potato-None-GarbageCan-10/trial_T20190907_161745_664033/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0113", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Potato-None-GarbageCan-10/trial_T20190907_161853_945788/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0114", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Tomato-None-GarbageCan-10/trial_T20190908_225046_020282/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0115", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Tomato-None-GarbageCan-10/trial_T20190908_225359_617900/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0116", + "gamefile": "json_2.1.1/valid_unseen/pick_heat_then_place_in_recep-Tomato-None-GarbageCan-10/trial_T20190908_225453_272533/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "test:0117", + "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-CD-None-Safe-308/trial_T20190907_050942_897916/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "test:0118", + "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-CD-None-Safe-308/trial_T20190907_051013_060265/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "test:0119", + "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-CD-None-Safe-308/trial_T20190907_051056_585414/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "test:0120", + "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-KeyChain-None-Safe-219/trial_T20190909_011803_423115/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "test:0121", + "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-KeyChain-None-Safe-219/trial_T20190909_012027_782483/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "test:0122", + "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-PepperShaker-None-Drawer-10/trial_T20190908_010306_215435/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "test:0123", + "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-PepperShaker-None-Drawer-10/trial_T20190912_221016_460197/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "test:0124", + "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-PepperShaker-None-Drawer-10/trial_T20190912_221141_608117/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "test:0125", + "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-Pillow-None-Sofa-219/trial_T20190907_163240_345855/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "test:0126", + "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-Pillow-None-Sofa-219/trial_T20190907_163327_486300/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "test:0127", + "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-Pillow-None-Sofa-219/trial_T20190907_163408_914117/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "test:0128", + "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-SoapBar-None-Cabinet-424/trial_T20190909_081720_491733/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "test:0129", + "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-SoapBar-None-Cabinet-424/trial_T20190909_081746_857594/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "test:0130", + "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-SoapBar-None-GarbageCan-424/trial_T20190909_064053_839817/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "test:0131", + "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-SoapBar-None-GarbageCan-424/trial_T20190909_064221_368939/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "test:0132", + "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-SoapBar-None-GarbageCan-424/trial_T20190909_064309_357168/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "test:0133", + "gamefile": "json_2.1.1/valid_unseen/pick_two_obj_and_place-ToiletPaper-None-Cabinet-424/trial_T20190906_202926_527010/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + } +] diff --git a/data/alfworld_path_split/train/items.json b/data/alfworld_path_split/train/items.json new file mode 100644 index 0000000..0be1082 --- /dev/null +++ b/data/alfworld_path_split/train/items.json @@ -0,0 +1,197 @@ +[ + { + "id": "train:0000", + "gamefile": "json_2.1.1/train/look_at_obj_in_light-AlarmClock-None-DeskLamp-305/trial_T20190908_082736_108723/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "train:0001", + "gamefile": "json_2.1.1/train/look_at_obj_in_light-CD-None-DeskLamp-304/trial_T20190907_185649_782438/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "train:0002", + "gamefile": "json_2.1.1/train/look_at_obj_in_light-CD-None-DeskLamp-320/trial_T20190907_224439_174735/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "train:0003", + "gamefile": "json_2.1.1/train/look_at_obj_in_light-Pillow-None-DeskLamp-316/trial_T20190908_232421_645610/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "train:0004", + "gamefile": "json_2.1.1/train/look_at_obj_in_light-Statue-None-DeskLamp-319/trial_T20190907_035546_167548/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "train:0005", + "gamefile": "json_2.1.1/train/pick_and_place_simple-CellPhone-None-Shelf-313/trial_T20190908_123725_452958/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "train:0006", + "gamefile": "json_2.1.1/train/pick_and_place_simple-Newspaper-None-Sofa-211/trial_T20190906_175004_203092/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "train:0007", + "gamefile": "json_2.1.1/train/pick_and_place_simple-Pencil-None-Desk-302/trial_T20190908_032836_462632/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "train:0008", + "gamefile": "json_2.1.1/train/pick_and_place_simple-SoapBar-None-GarbageCan-416/trial_T20190908_020839_714699/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "train:0009", + "gamefile": "json_2.1.1/train/pick_and_place_simple-Statue-None-CoffeeTable-222/trial_T20190907_131249_788749/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "train:0010", + "gamefile": "json_2.1.1/train/pick_and_place_simple-ToiletPaper-None-ToiletPaperHanger-406/trial_T20190908_122807_136741/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "train:0011", + "gamefile": "json_2.1.1/train/pick_and_place_simple-ToiletPaper-None-ToiletPaperHanger-415/trial_T20190908_050443_333939/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "train:0012", + "gamefile": "json_2.1.1/train/pick_clean_then_place_in_recep-Apple-None-DiningTable-4/trial_T20190908_104413_450768/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "train:0013", + "gamefile": "json_2.1.1/train/pick_clean_then_place_in_recep-DishSponge-None-Shelf-20/trial_T20190907_222429_992578/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "train:0014", + "gamefile": "json_2.1.1/train/pick_clean_then_place_in_recep-DishSponge-None-Shelf-401/trial_T20190908_072225_397518/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "train:0015", + "gamefile": "json_2.1.1/train/pick_clean_then_place_in_recep-Kettle-None-Cabinet-2/trial_T20190909_043103_418752/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "train:0016", + "gamefile": "json_2.1.1/train/pick_clean_then_place_in_recep-Knife-None-Drawer-22/trial_T20190907_224827_746945/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "train:0017", + "gamefile": "json_2.1.1/train/pick_clean_then_place_in_recep-Lettuce-None-DiningTable-20/trial_T20190906_191148_519826/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "train:0018", + "gamefile": "json_2.1.1/train/pick_clean_then_place_in_recep-Lettuce-None-Fridge-13/trial_T20190908_203022_601787/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "train:0019", + "gamefile": "json_2.1.1/train/pick_clean_then_place_in_recep-Plate-None-Fridge-5/trial_T20190909_112954_869911/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "train:0020", + "gamefile": "json_2.1.1/train/pick_clean_then_place_in_recep-Spoon-None-DiningTable-18/trial_T20190909_102159_277894/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "train:0021", + "gamefile": "json_2.1.1/train/pick_cool_then_place_in_recep-Bread-None-CounterTop-1/trial_T20190908_212439_711334/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "train:0022", + "gamefile": "json_2.1.1/train/pick_cool_then_place_in_recep-Bread-None-CounterTop-15/trial_T20190909_085448_256298/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "train:0023", + "gamefile": "json_2.1.1/train/pick_cool_then_place_in_recep-Bread-None-CounterTop-16/trial_T20190908_143948_082471/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "train:0024", + "gamefile": "json_2.1.1/train/pick_cool_then_place_in_recep-Pan-None-StoveBurner-27/trial_T20190906_212619_469871/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "train:0025", + "gamefile": "json_2.1.1/train/pick_cool_then_place_in_recep-Plate-None-DiningTable-17/trial_T20190909_122939_032098/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "train:0026", + "gamefile": "json_2.1.1/train/pick_cool_then_place_in_recep-Pot-None-CounterTop-1/trial_T20190909_124252_504581/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "train:0027", + "gamefile": "json_2.1.1/train/pick_heat_then_place_in_recep-Apple-None-Fridge-20/trial_T20190908_013911_274341/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "train:0028", + "gamefile": "json_2.1.1/train/pick_heat_then_place_in_recep-Egg-None-CounterTop-12/trial_T20190908_215527_416490/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "train:0029", + "gamefile": "json_2.1.1/train/pick_heat_then_place_in_recep-Mug-None-CoffeeMachine-1/trial_T20190907_222924_821086/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "train:0030", + "gamefile": "json_2.1.1/train/pick_heat_then_place_in_recep-Mug-None-CoffeeMachine-28/trial_T20190908_062730_537428/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "train:0031", + "gamefile": "json_2.1.1/train/pick_heat_then_place_in_recep-Plate-None-Cabinet-13/trial_T20190907_062749_759882/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "train:0032", + "gamefile": "json_2.1.1/train/pick_heat_then_place_in_recep-Potato-None-Fridge-2/trial_T20190909_030845_198194/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "train:0033", + "gamefile": "json_2.1.1/train/pick_heat_then_place_in_recep-Tomato-None-CounterTop-26/trial_T20190907_005525_499114/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "train:0034", + "gamefile": "json_2.1.1/train/pick_two_obj_and_place-CD-None-Drawer-319/trial_T20190907_145515_348252/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "train:0035", + "gamefile": "json_2.1.1/train/pick_two_obj_and_place-Candle-None-Drawer-427/trial_T20190909_043917_251333/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "train:0036", + "gamefile": "json_2.1.1/train/pick_two_obj_and_place-KeyChain-None-ArmChair-222/trial_T20190909_100312_677332/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "train:0037", + "gamefile": "json_2.1.1/train/pick_two_obj_and_place-Newspaper-None-Sofa-212/trial_T20190908_112632_208041/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "train:0038", + "gamefile": "json_2.1.1/train/pick_two_obj_and_place-SaltShaker-None-SideTable-21/trial_T20190909_041626_844806/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + } +] diff --git a/data/alfworld_path_split/val/items.json b/data/alfworld_path_split/val/items.json new file mode 100644 index 0000000..e696bd3 --- /dev/null +++ b/data/alfworld_path_split/val/items.json @@ -0,0 +1,92 @@ +[ + { + "id": "val:0000", + "gamefile": "json_2.1.1/valid_seen/look_at_obj_in_light-AlarmClock-None-DeskLamp-323/trial_T20190909_044715_250790/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "val:0001", + "gamefile": "json_2.1.1/valid_seen/look_at_obj_in_light-Bowl-None-DeskLamp-301/trial_T20190909_150719_492274/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "val:0002", + "gamefile": "json_2.1.1/valid_seen/look_at_obj_in_light-Pillow-None-DeskLamp-323/trial_T20190908_053153_077977/game.tw-pddl", + "task_type": "look_at_obj_in_light" + }, + { + "id": "val:0003", + "gamefile": "json_2.1.1/valid_seen/pick_and_place_simple-Mug-None-SideTable-329/trial_T20190909_032318_169393/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "val:0004", + "gamefile": "json_2.1.1/valid_seen/pick_and_place_simple-Mug-None-SideTable-329/trial_T20190909_032340_274147/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "val:0005", + "gamefile": "json_2.1.1/valid_seen/pick_and_place_simple-Pencil-None-Desk-310/trial_T20190909_113054_894334/game.tw-pddl", + "task_type": "pick_and_place_simple" + }, + { + "id": "val:0006", + "gamefile": "json_2.1.1/valid_seen/pick_clean_then_place_in_recep-ButterKnife-None-Drawer-30/trial_T20190908_052007_212776/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "val:0007", + "gamefile": "json_2.1.1/valid_seen/pick_clean_then_place_in_recep-ButterKnife-None-Drawer-8/trial_T20190909_124425_112757/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "val:0008", + "gamefile": "json_2.1.1/valid_seen/pick_clean_then_place_in_recep-SoapBar-None-Cabinet-402/trial_T20190908_055221_984342/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "val:0009", + "gamefile": "json_2.1.1/valid_seen/pick_clean_then_place_in_recep-SoapBar-None-Toilet-410/trial_T20190906_201106_979461/game.tw-pddl", + "task_type": "pick_clean_then_place_in_recep" + }, + { + "id": "val:0010", + "gamefile": "json_2.1.1/valid_seen/pick_cool_then_place_in_recep-Apple-None-Microwave-19/trial_T20190906_210937_878489/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "val:0011", + "gamefile": "json_2.1.1/valid_seen/pick_cool_then_place_in_recep-Plate-None-CounterTop-1/trial_T20190906_205324_559361/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "val:0012", + "gamefile": "json_2.1.1/valid_seen/pick_cool_then_place_in_recep-Tomato-None-Microwave-18/trial_T20190909_012524_159092/game.tw-pddl", + "task_type": "pick_cool_then_place_in_recep" + }, + { + "id": "val:0013", + "gamefile": "json_2.1.1/valid_seen/pick_heat_then_place_in_recep-Apple-None-DiningTable-26/trial_T20190907_060234_011675/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "val:0014", + "gamefile": "json_2.1.1/valid_seen/pick_heat_then_place_in_recep-Tomato-None-Fridge-15/trial_T20190909_020200_054379/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "val:0015", + "gamefile": "json_2.1.1/valid_seen/pick_heat_then_place_in_recep-Tomato-None-Fridge-23/trial_T20190909_082320_103350/game.tw-pddl", + "task_type": "pick_heat_then_place_in_recep" + }, + { + "id": "val:0016", + "gamefile": "json_2.1.1/valid_seen/pick_two_obj_and_place-Book-None-Desk-313/trial_T20190908_125930_920681/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + }, + { + "id": "val:0017", + "gamefile": "json_2.1.1/valid_seen/pick_two_obj_and_place-CreditCard-None-Safe-323/trial_T20190907_001129_214240/game.tw-pddl", + "task_type": "pick_two_obj_and_place" + } +] diff --git a/data/docvqa_id_split/split_manifest.json b/data/docvqa_id_split/split_manifest.json new file mode 100644 index 0000000..48696de --- /dev/null +++ b/data/docvqa_id_split/split_manifest.json @@ -0,0 +1,36 @@ +{ + "benchmark": "DocVQA", + "manifest_type": "id_split", + "source_repo": "lmms-lab/DocVQA", + "source_repo_type": "dataset", + "source_url": "https://huggingface.co/datasets/lmms-lab/DocVQA", + "source_revision": "539088ef8a8ada01ac8e2e6d4e372586748a265e", + "source_config": "DocVQA", + "source_split": "validation", + "source_split_name": "docvqa_validation_10pct", + "split_method": "10% subset sampled from the DocVQA validation split", + "counts": { + "train": 107, + "val": 53, + "test": 374 + }, + "item_fields": [ + "id", + "questionId", + "docId", + "image_path", + "ucsf_document_id", + "ucsf_document_page_no", + "topic", + "source_dataset", + "source_config", + "source_split", + "sample_seed" + ], + "notes": [ + "This is a split manifest, not the full DocVQA payload.", + "Materialize full CSV rows and image files before evaluation.", + "This manifest corresponds to docvqa_validation_10pct.", + "All released train/val/test items originate from a 10% subset of the official DocVQA validation split." + ] +} diff --git a/data/docvqa_id_split/test/items.json b/data/docvqa_id_split/test/items.json new file mode 100644 index 0000000..7c103a9 --- /dev/null +++ b/data/docvqa_id_split/test/items.json @@ -0,0 +1,4864 @@ +[ + { + "id": "63180", + "questionId": "63180", + "docId": "9099", + "image_path": "data/docvqa_images/q63180_d9099.png", + "ucsf_document_id": "jlmf0227", + "ucsf_document_page_no": "11", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "53705", + "questionId": "53705", + "docId": "3630", + "image_path": "data/docvqa_images/q53705_d3630.png", + "ucsf_document_id": "rhhx0023", + "ucsf_document_page_no": "1", + "topic": "table/list|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "54376", + "questionId": "54376", + "docId": "3626", + "image_path": "data/docvqa_images/q54376_d3626.png", + "ucsf_document_id": "glxm0052", + "ucsf_document_page_no": "3", + "topic": "form|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63763", + "questionId": "63763", + "docId": "9581", + "image_path": "data/docvqa_images/q63763_d9581.png", + "ucsf_document_id": "mxmg0227", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47931", + "questionId": "47931", + "docId": "13884", + "image_path": "data/docvqa_images/q47931_d13884.png", + "ucsf_document_id": "qnfm0227", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47535", + "questionId": "47535", + "docId": "13650", + "image_path": "data/docvqa_images/q47535_d13650.png", + "ucsf_document_id": "pmdv0228", + "ucsf_document_page_no": "8", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "37329", + "questionId": "37329", + "docId": "10759", + "image_path": "data/docvqa_images/q37329_d10759.png", + "ucsf_document_id": "tjpg0227", + "ucsf_document_page_no": "9", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "26657", + "questionId": "26657", + "docId": "7470", + "image_path": "data/docvqa_images/q26657_d7470.png", + "ucsf_document_id": "lhmg0227", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "44939", + "questionId": "44939", + "docId": "12948", + "image_path": "data/docvqa_images/q44939_d12948.png", + "ucsf_document_id": "pmyl0226", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "58754", + "questionId": "58754", + "docId": "5696", + "image_path": "data/docvqa_images/q58754_d5696.png", + "ucsf_document_id": "ggmk0079", + "ucsf_document_page_no": "5", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "54624", + "questionId": "54624", + "docId": "1992", + "image_path": "data/docvqa_images/q54624_d1992.png", + "ucsf_document_id": "kkny0225", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "1900", + "questionId": "1900", + "docId": "845", + "image_path": "data/docvqa_images/q1900_d845.png", + "ucsf_document_id": "thcn0226", + "ucsf_document_page_no": "3", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "1978", + "questionId": "1978", + "docId": "909", + "image_path": "data/docvqa_images/q1978_d909.png", + "ucsf_document_id": "jqbn0226", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "21068", + "questionId": "21068", + "docId": "6207", + "image_path": "data/docvqa_images/q21068_d6207.png", + "ucsf_document_id": "txcx0227", + "ucsf_document_page_no": "13", + "topic": "form|table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63787", + "questionId": "63787", + "docId": "9619", + "image_path": "data/docvqa_images/q63787_d9619.png", + "ucsf_document_id": "ngvh0227", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "55256", + "questionId": "55256", + "docId": "14298", + "image_path": "data/docvqa_images/q55256_d14298.png", + "ucsf_document_id": "lynb0228", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "56275", + "questionId": "56275", + "docId": "5060", + "image_path": "data/docvqa_images/q56275_d5060.png", + "ucsf_document_id": "gpnn0081", + "ucsf_document_page_no": "3", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50802", + "questionId": "50802", + "docId": "14747", + "image_path": "data/docvqa_images/q50802_d14747.png", + "ucsf_document_id": "nzfv0228", + "ucsf_document_page_no": "2", + "topic": "handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5558", + "questionId": "5558", + "docId": "1885", + "image_path": "data/docvqa_images/q5558_d1885.png", + "ucsf_document_id": "frjh0225", + "ucsf_document_page_no": "5", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "45724", + "questionId": "45724", + "docId": "13549", + "image_path": "data/docvqa_images/q45724_d13549.png", + "ucsf_document_id": "yscw0217", + "ucsf_document_page_no": "12", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "54909", + "questionId": "54909", + "docId": "4183", + "image_path": "data/docvqa_images/q54909_d4183.png", + "ucsf_document_id": "mhcg0072", + "ucsf_document_page_no": "43", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "43773", + "questionId": "43773", + "docId": "12709", + "image_path": "data/docvqa_images/q43773_d12709.png", + "ucsf_document_id": "jkhn0226", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "56451", + "questionId": "56451", + "docId": "14795", + "image_path": "data/docvqa_images/q56451_d14795.png", + "ucsf_document_id": "gnnp0227", + "ucsf_document_page_no": "6", + "topic": "Yes/No|handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "61985", + "questionId": "61985", + "docId": "8179", + "image_path": "data/docvqa_images/q61985_d8179.png", + "ucsf_document_id": "zpyp0227", + "ucsf_document_page_no": "4", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "7012", + "questionId": "7012", + "docId": "2392", + "image_path": "data/docvqa_images/q7012_d2392.png", + "ucsf_document_id": "gggw0004", + "ucsf_document_page_no": "2", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50508", + "questionId": "50508", + "docId": "245", + "image_path": "data/docvqa_images/q50508_d245.png", + "ucsf_document_id": "nrcj0037", + "ucsf_document_page_no": "8", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47517", + "questionId": "47517", + "docId": "13640", + "image_path": "data/docvqa_images/q47517_d13640.png", + "ucsf_document_id": "pydv0228", + "ucsf_document_page_no": "9", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "65341", + "questionId": "65341", + "docId": "10882", + "image_path": "data/docvqa_images/q65341_d10882.png", + "ucsf_document_id": "kfhd0227", + "ucsf_document_page_no": "22", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "6822", + "questionId": "6822", + "docId": "2361", + "image_path": "data/docvqa_images/q6822_d2361.png", + "ucsf_document_id": "rmpn0000", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "55026", + "questionId": "55026", + "docId": "4162", + "image_path": "data/docvqa_images/q55026_d4162.png", + "ucsf_document_id": "yldg0072", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "53438", + "questionId": "53438", + "docId": "2715", + "image_path": "data/docvqa_images/q53438_d2715.png", + "ucsf_document_id": "yhxn0020", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63868", + "questionId": "63868", + "docId": "9631", + "image_path": "data/docvqa_images/q63868_d9631.png", + "ucsf_document_id": "nlcf0227", + "ucsf_document_page_no": "5", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "16837", + "questionId": "16837", + "docId": "5324", + "image_path": "data/docvqa_images/q16837_d5324.png", + "ucsf_document_id": "rgcw0217", + "ucsf_document_page_no": "7", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57531", + "questionId": "57531", + "docId": "4843", + "image_path": "data/docvqa_images/q57531_d4843.png", + "ucsf_document_id": "snbx0223", + "ucsf_document_page_no": "7", + "topic": "Image/Photo", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50770", + "questionId": "50770", + "docId": "377", + "image_path": "data/docvqa_images/q50770_d377.png", + "ucsf_document_id": "mtyj0226", + "ucsf_document_page_no": "8", + "topic": "figure/diagram", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "60597", + "questionId": "60597", + "docId": "7232", + "image_path": "data/docvqa_images/q60597_d7232.png", + "ucsf_document_id": "symf0227", + "ucsf_document_page_no": "5", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "60403", + "questionId": "60403", + "docId": "7135", + "image_path": "data/docvqa_images/q60403_d7135.png", + "ucsf_document_id": "gkpk0226", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "24116", + "questionId": "24116", + "docId": "6922", + "image_path": "data/docvqa_images/q24116_d6922.png", + "ucsf_document_id": "xjhk0226", + "ucsf_document_page_no": "1", + "topic": "form|table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "65404", + "questionId": "65404", + "docId": "10983", + "image_path": "data/docvqa_images/q65404_d10983.png", + "ucsf_document_id": "msmg0227", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "359", + "questionId": "359", + "docId": "287", + "image_path": "data/docvqa_images/q359_d287.png", + "ucsf_document_id": "rzbj0037", + "ucsf_document_page_no": "8", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "58419", + "questionId": "58419", + "docId": "5926", + "image_path": "data/docvqa_images/q58419_d5926.png", + "ucsf_document_id": "lybx0227", + "ucsf_document_page_no": "23", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64587", + "questionId": "64587", + "docId": "10364", + "image_path": "data/docvqa_images/q64587_d10364.png", + "ucsf_document_id": "lpdl0226", + "ucsf_document_page_no": "13", + "topic": "form|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64328", + "questionId": "64328", + "docId": "10196", + "image_path": "data/docvqa_images/q64328_d10196.png", + "ucsf_document_id": "jjmd0217", + "ucsf_document_page_no": "2", + "topic": "Yes/No|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64219", + "questionId": "64219", + "docId": "10014", + "image_path": "data/docvqa_images/q64219_d10014.png", + "ucsf_document_id": "qjcf0227", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "2143", + "questionId": "2143", + "docId": "1039", + "image_path": "data/docvqa_images/q2143_d1039.png", + "ucsf_document_id": "khnk0226", + "ucsf_document_page_no": "4", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47550", + "questionId": "47550", + "docId": "13691", + "image_path": "data/docvqa_images/q47550_d13691.png", + "ucsf_document_id": "frdv0228", + "ucsf_document_page_no": "4", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "8073", + "questionId": "8073", + "docId": "2823", + "image_path": "data/docvqa_images/q8073_d2823.png", + "ucsf_document_id": "pfcn0020", + "ucsf_document_page_no": "2", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5607", + "questionId": "5607", + "docId": "1888", + "image_path": "data/docvqa_images/q5607_d1888.png", + "ucsf_document_id": "fqwx0225", + "ucsf_document_page_no": "10", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "3072", + "questionId": "3072", + "docId": "1210", + "image_path": "data/docvqa_images/q3072_d1210.png", + "ucsf_document_id": "gxph0227", + "ucsf_document_page_no": "8", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "51369", + "questionId": "51369", + "docId": "794", + "image_path": "data/docvqa_images/q51369_d794.png", + "ucsf_document_id": "nlcn0226", + "ucsf_document_page_no": "4", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "59443", + "questionId": "59443", + "docId": "5992", + "image_path": "data/docvqa_images/q59443_d5992.png", + "ucsf_document_id": "ffhx0227", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64188", + "questionId": "64188", + "docId": "9872", + "image_path": "data/docvqa_images/q64188_d9872.png", + "ucsf_document_id": "ptkg0227", + "ucsf_document_page_no": "32", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50924", + "questionId": "50924", + "docId": "473", + "image_path": "data/docvqa_images/q50924_d473.png", + "ucsf_document_id": "ptjf0226", + "ucsf_document_page_no": "3", + "topic": "layout|others", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "788", + "questionId": "788", + "docId": "408", + "image_path": "data/docvqa_images/q788_d408.png", + "ucsf_document_id": "kfpj0226", + "ucsf_document_page_no": "2", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "45005", + "questionId": "45005", + "docId": "12959", + "image_path": "data/docvqa_images/q45005_d12959.png", + "ucsf_document_id": "qtgl0226", + "ucsf_document_page_no": "2", + "topic": "table/list|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57323", + "questionId": "57323", + "docId": "4722", + "image_path": "data/docvqa_images/q57323_d4722.png", + "ucsf_document_id": "xybx0223", + "ucsf_document_page_no": "32", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "32154", + "questionId": "32154", + "docId": "9013", + "image_path": "data/docvqa_images/q32154_d9013.png", + "ucsf_document_id": "qxmp0227", + "ucsf_document_page_no": "4", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "56363", + "questionId": "56363", + "docId": "14780", + "image_path": "data/docvqa_images/q56363_d14780.png", + "ucsf_document_id": "lyvd0228", + "ucsf_document_page_no": "6", + "topic": "figure/diagram", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5392", + "questionId": "5392", + "docId": "1817", + "image_path": "data/docvqa_images/q5392_d1817.png", + "ucsf_document_id": "xhfl0228", + "ucsf_document_page_no": "7", + "topic": "figure/diagram|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5313", + "questionId": "5313", + "docId": "1791", + "image_path": "data/docvqa_images/q5313_d1791.png", + "ucsf_document_id": "myph0227", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "45837", + "questionId": "45837", + "docId": "13402", + "image_path": "data/docvqa_images/q45837_d13402.png", + "ucsf_document_id": "zqdw0217", + "ucsf_document_page_no": "14", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "56457", + "questionId": "56457", + "docId": "14795", + "image_path": "data/docvqa_images/q56457_d14795.png", + "ucsf_document_id": "gnnp0227", + "ucsf_document_page_no": "6", + "topic": "handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "34135", + "questionId": "34135", + "docId": "9725", + "image_path": "data/docvqa_images/q34135_d9725.png", + "ucsf_document_id": "mswg0227", + "ucsf_document_page_no": "4", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "15008", + "questionId": "15008", + "docId": "5026", + "image_path": "data/docvqa_images/q15008_d5026.png", + "ucsf_document_id": "mnvw0217", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "51601", + "questionId": "51601", + "docId": "1130", + "image_path": "data/docvqa_images/q51601_d1130.png", + "ucsf_document_id": "gnjk0226", + "ucsf_document_page_no": "1", + "topic": "handwritten|form|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64491", + "questionId": "64491", + "docId": "10307", + "image_path": "data/docvqa_images/q64491_d10307.png", + "ucsf_document_id": "lpdl0226", + "ucsf_document_page_no": "15", + "topic": "form|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "56887", + "questionId": "56887", + "docId": "5208", + "image_path": "data/docvqa_images/q56887_d5208.png", + "ucsf_document_id": "jxyn0081", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63246", + "questionId": "63246", + "docId": "9218", + "image_path": "data/docvqa_images/q63246_d9218.png", + "ucsf_document_id": "pqxf0227", + "ucsf_document_page_no": "1", + "topic": "layout|others", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "58558", + "questionId": "58558", + "docId": "4748", + "image_path": "data/docvqa_images/q58558_d4748.png", + "ucsf_document_id": "rnbx0223", + "ucsf_document_page_no": "205", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "43110", + "questionId": "43110", + "docId": "12398", + "image_path": "data/docvqa_images/q43110_d12398.png", + "ucsf_document_id": "fggn0226", + "ucsf_document_page_no": "48", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47554", + "questionId": "47554", + "docId": "13691", + "image_path": "data/docvqa_images/q47554_d13691.png", + "ucsf_document_id": "frdv0228", + "ucsf_document_page_no": "4", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "51224", + "questionId": "51224", + "docId": "768", + "image_path": "data/docvqa_images/q51224_d768.png", + "ucsf_document_id": "ngcn0226", + "ucsf_document_page_no": "2", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "51484", + "questionId": "51484", + "docId": "1735", + "image_path": "data/docvqa_images/q51484_d1735.png", + "ucsf_document_id": "fpxh0227", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64634", + "questionId": "64634", + "docId": "10908", + "image_path": "data/docvqa_images/q64634_d10908.png", + "ucsf_document_id": "yrpf0227", + "ucsf_document_page_no": "3", + "topic": "Image/Photo", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "59802", + "questionId": "59802", + "docId": "6708", + "image_path": "data/docvqa_images/q59802_d6708.png", + "ucsf_document_id": "xkxb0228", + "ucsf_document_page_no": "2", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5821", + "questionId": "5821", + "docId": "1996", + "image_path": "data/docvqa_images/q5821_d1996.png", + "ucsf_document_id": "zylj0226", + "ucsf_document_page_no": "7", + "topic": "handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "45425", + "questionId": "45425", + "docId": "13589", + "image_path": "data/docvqa_images/q45425_d13589.png", + "ucsf_document_id": "zrdw0217", + "ucsf_document_page_no": "1", + "topic": "layout|others", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "56580", + "questionId": "56580", + "docId": "14789", + "image_path": "data/docvqa_images/q56580_d14789.png", + "ucsf_document_id": "kpkp0227", + "ucsf_document_page_no": "14", + "topic": "form|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "52637", + "questionId": "52637", + "docId": "2295", + "image_path": "data/docvqa_images/q52637_d2295.png", + "ucsf_document_id": "fglc0003", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "3071", + "questionId": "3071", + "docId": "1210", + "image_path": "data/docvqa_images/q3071_d1210.png", + "ucsf_document_id": "gxph0227", + "ucsf_document_page_no": "8", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "58835", + "questionId": "58835", + "docId": "5846", + "image_path": "data/docvqa_images/q58835_d5846.png", + "ucsf_document_id": "knlm0227", + "ucsf_document_page_no": "4", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "56812", + "questionId": "56812", + "docId": "14919", + "image_path": "data/docvqa_images/q56812_d14919.png", + "ucsf_document_id": "qlkp0227", + "ucsf_document_page_no": "4", + "topic": "handwritten|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "62508", + "questionId": "62508", + "docId": "7596", + "image_path": "data/docvqa_images/q62508_d7596.png", + "ucsf_document_id": "fybg0227", + "ucsf_document_page_no": "4", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50303", + "questionId": "50303", + "docId": "14571", + "image_path": "data/docvqa_images/q50303_d14571.png", + "ucsf_document_id": "rrdd0228", + "ucsf_document_page_no": "14", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "58426", + "questionId": "58426", + "docId": "5313", + "image_path": "data/docvqa_images/q58426_d5313.png", + "ucsf_document_id": "jmcw0217", + "ucsf_document_page_no": "4", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64255", + "questionId": "64255", + "docId": "10121", + "image_path": "data/docvqa_images/q64255_d10121.png", + "ucsf_document_id": "lpjm0223", + "ucsf_document_page_no": "58", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "1014", + "questionId": "1014", + "docId": "491", + "image_path": "data/docvqa_images/q1014_d491.png", + "ucsf_document_id": "gyjf0226", + "ucsf_document_page_no": "1", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "58878", + "questionId": "58878", + "docId": "6096", + "image_path": "data/docvqa_images/q58878_d6096.png", + "ucsf_document_id": "rnbx0223", + "ucsf_document_page_no": "101", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "56371", + "questionId": "56371", + "docId": "14778", + "image_path": "data/docvqa_images/q56371_d14778.png", + "ucsf_document_id": "skgb0228", + "ucsf_document_page_no": "43", + "topic": "figure/diagram", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "58675", + "questionId": "58675", + "docId": "5550", + "image_path": "data/docvqa_images/q58675_d5550.png", + "ucsf_document_id": "fxcv0079", + "ucsf_document_page_no": "4", + "topic": "figure/diagram", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "7028", + "questionId": "7028", + "docId": "2396", + "image_path": "data/docvqa_images/q7028_d2396.png", + "ucsf_document_id": "myjf0004", + "ucsf_document_page_no": "2", + "topic": "handwritten|free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "59022", + "questionId": "59022", + "docId": "6223", + "image_path": "data/docvqa_images/q59022_d6223.png", + "ucsf_document_id": "kmhx0227", + "ucsf_document_page_no": "4", + "topic": "table/list|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47003", + "questionId": "47003", + "docId": "13613", + "image_path": "data/docvqa_images/q47003_d13613.png", + "ucsf_document_id": "pyyc0227", + "ucsf_document_page_no": "52", + "topic": "figure/diagram", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "17140", + "questionId": "17140", + "docId": "5400", + "image_path": "data/docvqa_images/q17140_d5400.png", + "ucsf_document_id": "zkww0217", + "ucsf_document_page_no": "2", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64166", + "questionId": "64166", + "docId": "9866", + "image_path": "data/docvqa_images/q64166_d9866.png", + "ucsf_document_id": "yllg0227", + "ucsf_document_page_no": "2", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5858", + "questionId": "5858", + "docId": "2006", + "image_path": "data/docvqa_images/q5858_d2006.png", + "ucsf_document_id": "lzkh0228", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "2175", + "questionId": "2175", + "docId": "1039", + "image_path": "data/docvqa_images/q2175_d1039.png", + "ucsf_document_id": "khnk0226", + "ucsf_document_page_no": "4", + "topic": "figure/diagram", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57313", + "questionId": "57313", + "docId": "4712", + "image_path": "data/docvqa_images/q57313_d4712.png", + "ucsf_document_id": "mtgj0223", + "ucsf_document_page_no": "17", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "43626", + "questionId": "43626", + "docId": "12512", + "image_path": "data/docvqa_images/q43626_d12512.png", + "ucsf_document_id": "lngn0226", + "ucsf_document_page_no": "5", + "topic": "figure/diagram|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "55032", + "questionId": "55032", + "docId": "4163", + "image_path": "data/docvqa_images/q55032_d4163.png", + "ucsf_document_id": "tqcg0072", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57213", + "questionId": "57213", + "docId": "4812", + "image_path": "data/docvqa_images/q57213_d4812.png", + "ucsf_document_id": "snbx0223", + "ucsf_document_page_no": "6", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "3079", + "questionId": "3079", + "docId": "1210", + "image_path": "data/docvqa_images/q3079_d1210.png", + "ucsf_document_id": "gxph0227", + "ucsf_document_page_no": "8", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "55926", + "questionId": "55926", + "docId": "4244", + "image_path": "data/docvqa_images/q55926_d4244.png", + "ucsf_document_id": "jybx0223", + "ucsf_document_page_no": "11", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "32157", + "questionId": "32157", + "docId": "9013", + "image_path": "data/docvqa_images/q32157_d9013.png", + "ucsf_document_id": "qxmp0227", + "ucsf_document_page_no": "4", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "28064", + "questionId": "28064", + "docId": "7867", + "image_path": "data/docvqa_images/q28064_d7867.png", + "ucsf_document_id": "zznp0227", + "ucsf_document_page_no": "107", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "15041", + "questionId": "15041", + "docId": "5023", + "image_path": "data/docvqa_images/q15041_d5023.png", + "ucsf_document_id": "yxvw0217", + "ucsf_document_page_no": "7", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50515", + "questionId": "50515", + "docId": "219", + "image_path": "data/docvqa_images/q50515_d219.png", + "ucsf_document_id": "ppwl0228", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47073", + "questionId": "47073", + "docId": "13939", + "image_path": "data/docvqa_images/q47073_d13939.png", + "ucsf_document_id": "pnfm0227", + "ucsf_document_page_no": "7", + "topic": "handwritten|table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "58454", + "questionId": "58454", + "docId": "5315", + "image_path": "data/docvqa_images/q58454_d5315.png", + "ucsf_document_id": "mlbw0217", + "ucsf_document_page_no": "6", + "topic": "table/list|others", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "45740", + "questionId": "45740", + "docId": "13581", + "image_path": "data/docvqa_images/q45740_d13581.png", + "ucsf_document_id": "yrvw0217", + "ucsf_document_page_no": "55", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "62129", + "questionId": "62129", + "docId": "8322", + "image_path": "data/docvqa_images/q62129_d8322.png", + "ucsf_document_id": "ljgf0227", + "ucsf_document_page_no": "5", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47537", + "questionId": "47537", + "docId": "13650", + "image_path": "data/docvqa_images/q47537_d13650.png", + "ucsf_document_id": "pmdv0228", + "ucsf_document_page_no": "8", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5552", + "questionId": "5552", + "docId": "1875", + "image_path": "data/docvqa_images/q5552_d1875.png", + "ucsf_document_id": "mnfl0228", + "ucsf_document_page_no": "2", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63876", + "questionId": "63876", + "docId": "9654", + "image_path": "data/docvqa_images/q63876_d9654.png", + "ucsf_document_id": "mswg0227", + "ucsf_document_page_no": "8", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "46435", + "questionId": "46435", + "docId": "13048", + "image_path": "data/docvqa_images/q46435_d13048.png", + "ucsf_document_id": "tzjl0226", + "ucsf_document_page_no": "4", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50968", + "questionId": "50968", + "docId": "549", + "image_path": "data/docvqa_images/q50968_d549.png", + "ucsf_document_id": "qtjf0226", + "ucsf_document_page_no": "2", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "32878", + "questionId": "32878", + "docId": "9253", + "image_path": "data/docvqa_images/q32878_d9253.png", + "ucsf_document_id": "hnhd0227", + "ucsf_document_page_no": "8", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "7025", + "questionId": "7025", + "docId": "2396", + "image_path": "data/docvqa_images/q7025_d2396.png", + "ucsf_document_id": "myjf0004", + "ucsf_document_page_no": "2", + "topic": "handwritten|free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47452", + "questionId": "47452", + "docId": "13639", + "image_path": "data/docvqa_images/q47452_d13639.png", + "ucsf_document_id": "skdv0228", + "ucsf_document_page_no": "5", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63158", + "questionId": "63158", + "docId": "9088", + "image_path": "data/docvqa_images/q63158_d9088.png", + "ucsf_document_id": "lnyc0227", + "ucsf_document_page_no": "3", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "38039", + "questionId": "38039", + "docId": "10946", + "image_path": "data/docvqa_images/q38039_d10946.png", + "ucsf_document_id": "kzng0227", + "ucsf_document_page_no": "48", + "topic": "table/list|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "46421", + "questionId": "46421", + "docId": "13048", + "image_path": "data/docvqa_images/q46421_d13048.png", + "ucsf_document_id": "tzjl0226", + "ucsf_document_page_no": "4", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "62392", + "questionId": "62392", + "docId": "7462", + "image_path": "data/docvqa_images/q62392_d7462.png", + "ucsf_document_id": "jfgg0227", + "ucsf_document_page_no": "3", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "49121", + "questionId": "49121", + "docId": "14218", + "image_path": "data/docvqa_images/q49121_d14218.png", + "ucsf_document_id": "qtyp0227", + "ucsf_document_page_no": "2", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63560", + "questionId": "63560", + "docId": "9304", + "image_path": "data/docvqa_images/q63560_d9304.png", + "ucsf_document_id": "jjvg0227", + "ucsf_document_page_no": "4", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "32896", + "questionId": "32896", + "docId": "9419", + "image_path": "data/docvqa_images/q32896_d9419.png", + "ucsf_document_id": "sxvg0227", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "39079", + "questionId": "39079", + "docId": "11190", + "image_path": "data/docvqa_images/q39079_d11190.png", + "ucsf_document_id": "qqvf0227", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "4438", + "questionId": "4438", + "docId": "1971", + "image_path": "data/docvqa_images/q4438_d1971.png", + "ucsf_document_id": "rxxk0225", + "ucsf_document_page_no": "9", + "topic": "figure/diagram|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "15094", + "questionId": "15094", + "docId": "4768", + "image_path": "data/docvqa_images/q15094_d4768.png", + "ucsf_document_id": "snbx0223", + "ucsf_document_page_no": "211", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "38032", + "questionId": "38032", + "docId": "10946", + "image_path": "data/docvqa_images/q38032_d10946.png", + "ucsf_document_id": "kzng0227", + "ucsf_document_page_no": "48", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "62828", + "questionId": "62828", + "docId": "8866", + "image_path": "data/docvqa_images/q62828_d8866.png", + "ucsf_document_id": "qxhc0228", + "ucsf_document_page_no": "6", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "46295", + "questionId": "46295", + "docId": "13358", + "image_path": "data/docvqa_images/q46295_d13358.png", + "ucsf_document_id": "yscw0217", + "ucsf_document_page_no": "61", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "56435", + "questionId": "56435", + "docId": "14800", + "image_path": "data/docvqa_images/q56435_d14800.png", + "ucsf_document_id": "jrcy0227", + "ucsf_document_page_no": "15", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "53576", + "questionId": "53576", + "docId": "2766", + "image_path": "data/docvqa_images/q53576_d2766.png", + "ucsf_document_id": "hsfn0020", + "ucsf_document_page_no": "2", + "topic": "free_text|table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "37654", + "questionId": "37654", + "docId": "10833", + "image_path": "data/docvqa_images/q37654_d10833.png", + "ucsf_document_id": "yjvg0227", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "46506", + "questionId": "46506", + "docId": "12445", + "image_path": "data/docvqa_images/q46506_d12445.png", + "ucsf_document_id": "gggn0226", + "ucsf_document_page_no": "50", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "25502", + "questionId": "25502", + "docId": "7245", + "image_path": "data/docvqa_images/q25502_d7245.png", + "ucsf_document_id": "pzbd0227", + "ucsf_document_page_no": "2", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "49055", + "questionId": "49055", + "docId": "14189", + "image_path": "data/docvqa_images/q49055_d14189.png", + "ucsf_document_id": "qtyp0227", + "ucsf_document_page_no": "9", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "62284", + "questionId": "62284", + "docId": "8429", + "image_path": "data/docvqa_images/q62284_d8429.png", + "ucsf_document_id": "nhkw0227", + "ucsf_document_page_no": "3", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "54490", + "questionId": "54490", + "docId": "3645", + "image_path": "data/docvqa_images/q54490_d3645.png", + "ucsf_document_id": "tqgk0023", + "ucsf_document_page_no": "13", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63636", + "questionId": "63636", + "docId": "9346", + "image_path": "data/docvqa_images/q63636_d9346.png", + "ucsf_document_id": "fncf0227", + "ucsf_document_page_no": "2", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57242", + "questionId": "57242", + "docId": "4719", + "image_path": "data/docvqa_images/q57242_d4719.png", + "ucsf_document_id": "snbx0223", + "ucsf_document_page_no": "221", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "49326", + "questionId": "49326", + "docId": "14304", + "image_path": "data/docvqa_images/q49326_d14304.png", + "ucsf_document_id": "qqvv0228", + "ucsf_document_page_no": "2", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "65215", + "questionId": "65215", + "docId": "10856", + "image_path": "data/docvqa_images/q65215_d10856.png", + "ucsf_document_id": "kjgf0227", + "ucsf_document_page_no": "4", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64742", + "questionId": "64742", + "docId": "10501", + "image_path": "data/docvqa_images/q64742_d10501.png", + "ucsf_document_id": "zybd0227", + "ucsf_document_page_no": "4", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "37663", + "questionId": "37663", + "docId": "10835", + "image_path": "data/docvqa_images/q37663_d10835.png", + "ucsf_document_id": "jqxf0227", + "ucsf_document_page_no": "28", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64706", + "questionId": "64706", + "docId": "10475", + "image_path": "data/docvqa_images/q64706_d10475.png", + "ucsf_document_id": "njnf0227", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "237", + "questionId": "237", + "docId": "230", + "image_path": "data/docvqa_images/q237_d230.png", + "ucsf_document_id": "ljxj0037", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "6399", + "questionId": "6399", + "docId": "2242", + "image_path": "data/docvqa_images/q6399_d2242.png", + "ucsf_document_id": "jkcn0000", + "ucsf_document_page_no": "2", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64935", + "questionId": "64935", + "docId": "10811", + "image_path": "data/docvqa_images/q64935_d10811.png", + "ucsf_document_id": "ylwg0227", + "ucsf_document_page_no": "15", + "topic": "figure/diagram|free_text|others", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "43224", + "questionId": "43224", + "docId": "12426", + "image_path": "data/docvqa_images/q43224_d12426.png", + "ucsf_document_id": "hmxn0226", + "ucsf_document_page_no": "3", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "31079", + "questionId": "31079", + "docId": "8697", + "image_path": "data/docvqa_images/q31079_d8697.png", + "ucsf_document_id": "yhxd0227", + "ucsf_document_page_no": "3", + "topic": "layout|others", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57543", + "questionId": "57543", + "docId": "4850", + "image_path": "data/docvqa_images/q57543_d4850.png", + "ucsf_document_id": "snbx0223", + "ucsf_document_page_no": "10", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5413", + "questionId": "5413", + "docId": "1840", + "image_path": "data/docvqa_images/q5413_d1840.png", + "ucsf_document_id": "flfl0228", + "ucsf_document_page_no": "1", + "topic": "table/list|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "6357", + "questionId": "6357", + "docId": "2225", + "image_path": "data/docvqa_images/q6357_d2225.png", + "ucsf_document_id": "gmhp0000", + "ucsf_document_page_no": "2", + "topic": "handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "53813", + "questionId": "53813", + "docId": "3200", + "image_path": "data/docvqa_images/q53813_d3200.png", + "ucsf_document_id": "kmfh0023", + "ucsf_document_page_no": "2", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "51664", + "questionId": "51664", + "docId": "1203", + "image_path": "data/docvqa_images/q51664_d1203.png", + "ucsf_document_id": "hnjh0227", + "ucsf_document_page_no": "3", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50295", + "questionId": "50295", + "docId": "14571", + "image_path": "data/docvqa_images/q50295_d14571.png", + "ucsf_document_id": "rrdd0228", + "ucsf_document_page_no": "14", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "16473", + "questionId": "16473", + "docId": "5189", + "image_path": "data/docvqa_images/q16473_d5189.png", + "ucsf_document_id": "hsyn0081", + "ucsf_document_page_no": "16", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "43719", + "questionId": "43719", + "docId": "12536", + "image_path": "data/docvqa_images/q43719_d12536.png", + "ucsf_document_id": "qjgn0226", + "ucsf_document_page_no": "74", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "58694", + "questionId": "58694", + "docId": "5545", + "image_path": "data/docvqa_images/q58694_d5545.png", + "ucsf_document_id": "hhwh0078", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "59655", + "questionId": "59655", + "docId": "6579", + "image_path": "data/docvqa_images/q59655_d6579.png", + "ucsf_document_id": "mzbx0227", + "ucsf_document_page_no": "2", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "58310", + "questionId": "58310", + "docId": "4981", + "image_path": "data/docvqa_images/q58310_d4981.png", + "ucsf_document_id": "fqvw0217", + "ucsf_document_page_no": "39", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5650", + "questionId": "5650", + "docId": "1909", + "image_path": "data/docvqa_images/q5650_d1909.png", + "ucsf_document_id": "ltlj0226", + "ucsf_document_page_no": "7", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47993", + "questionId": "47993", + "docId": "14096", + "image_path": "data/docvqa_images/q47993_d14096.png", + "ucsf_document_id": "lkcv0228", + "ucsf_document_page_no": "17", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64636", + "questionId": "64636", + "docId": "10908", + "image_path": "data/docvqa_images/q64636_d10908.png", + "ucsf_document_id": "yrpf0227", + "ucsf_document_page_no": "3", + "topic": "Image/Photo", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "44854", + "questionId": "44854", + "docId": "12889", + "image_path": "data/docvqa_images/q44854_d12889.png", + "ucsf_document_id": "rmwn0226", + "ucsf_document_page_no": "95", + "topic": "figure/diagram|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "55464", + "questionId": "55464", + "docId": "4331", + "image_path": "data/docvqa_images/q55464_d4331.png", + "ucsf_document_id": "gsgj0223", + "ucsf_document_page_no": "68", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "58289", + "questionId": "58289", + "docId": "4973", + "image_path": "data/docvqa_images/q58289_d4973.png", + "ucsf_document_id": "npvw0217", + "ucsf_document_page_no": "5", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "49126", + "questionId": "49126", + "docId": "14218", + "image_path": "data/docvqa_images/q49126_d14218.png", + "ucsf_document_id": "qtyp0227", + "ucsf_document_page_no": "2", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "1163", + "questionId": "1163", + "docId": "532", + "image_path": "data/docvqa_images/q1163_d532.png", + "ucsf_document_id": "hmjf0226", + "ucsf_document_page_no": "9", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "51615", + "questionId": "51615", + "docId": "1168", + "image_path": "data/docvqa_images/q51615_d1168.png", + "ucsf_document_id": "fzyh0227", + "ucsf_document_page_no": "7", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64903", + "questionId": "64903", + "docId": "10574", + "image_path": "data/docvqa_images/q64903_d10574.png", + "ucsf_document_id": "lmmg0227", + "ucsf_document_page_no": "1", + "topic": "form|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "62524", + "questionId": "62524", + "docId": "8587", + "image_path": "data/docvqa_images/q62524_d8587.png", + "ucsf_document_id": "xhwg0227", + "ucsf_document_page_no": "9", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "22511", + "questionId": "22511", + "docId": "6531", + "image_path": "data/docvqa_images/q22511_d6531.png", + "ucsf_document_id": "xfbc0228", + "ucsf_document_page_no": "2", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "46283", + "questionId": "46283", + "docId": "13358", + "image_path": "data/docvqa_images/q46283_d13358.png", + "ucsf_document_id": "yscw0217", + "ucsf_document_page_no": "61", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "56576", + "questionId": "56576", + "docId": "14792", + "image_path": "data/docvqa_images/q56576_d14792.png", + "ucsf_document_id": "jjfb0228", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47525", + "questionId": "47525", + "docId": "13650", + "image_path": "data/docvqa_images/q47525_d13650.png", + "ucsf_document_id": "pmdv0228", + "ucsf_document_page_no": "8", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "52374", + "questionId": "52374", + "docId": "2251", + "image_path": "data/docvqa_images/q52374_d2251.png", + "ucsf_document_id": "fqny0000", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57786", + "questionId": "57786", + "docId": "4837", + "image_path": "data/docvqa_images/q57786_d4837.png", + "ucsf_document_id": "tnbx0223", + "ucsf_document_page_no": "12", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "3838", + "questionId": "3838", + "docId": "1432", + "image_path": "data/docvqa_images/q3838_d1432.png", + "ucsf_document_id": "fshk0226", + "ucsf_document_page_no": "7", + "topic": "layout|others", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63709", + "questionId": "63709", + "docId": "9387", + "image_path": "data/docvqa_images/q63709_d9387.png", + "ucsf_document_id": "gpcg0227", + "ucsf_document_page_no": "3", + "topic": "layout|others", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "49129", + "questionId": "49129", + "docId": "14218", + "image_path": "data/docvqa_images/q49129_d14218.png", + "ucsf_document_id": "qtyp0227", + "ucsf_document_page_no": "2", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "55197", + "questionId": "55197", + "docId": "4259", + "image_path": "data/docvqa_images/q55197_d4259.png", + "ucsf_document_id": "klvj0223", + "ucsf_document_page_no": "15", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "56345", + "questionId": "56345", + "docId": "5137", + "image_path": "data/docvqa_images/q56345_d5137.png", + "ucsf_document_id": "hsyn0081", + "ucsf_document_page_no": "54", + "topic": "figure/diagram", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "6776", + "questionId": "6776", + "docId": "2359", + "image_path": "data/docvqa_images/q6776_d2359.png", + "ucsf_document_id": "hnhp0000", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "7019", + "questionId": "7019", + "docId": "2395", + "image_path": "data/docvqa_images/q7019_d2395.png", + "ucsf_document_id": "prbw0004", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "4953", + "questionId": "4953", + "docId": "1969", + "image_path": "data/docvqa_images/q4953_d1969.png", + "ucsf_document_id": "qymj0226", + "ucsf_document_page_no": "1", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "42037", + "questionId": "42037", + "docId": "11940", + "image_path": "data/docvqa_images/q42037_d11940.png", + "ucsf_document_id": "tfgn0226", + "ucsf_document_page_no": "63", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47915", + "questionId": "47915", + "docId": "13885", + "image_path": "data/docvqa_images/q47915_d13885.png", + "ucsf_document_id": "qznm0227", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47860", + "questionId": "47860", + "docId": "13856", + "image_path": "data/docvqa_images/q47860_d13856.png", + "ucsf_document_id": "nznm0227", + "ucsf_document_page_no": "96", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64612", + "questionId": "64612", + "docId": "10376", + "image_path": "data/docvqa_images/q64612_d10376.png", + "ucsf_document_id": "hslf0227", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "6806", + "questionId": "6806", + "docId": "2366", + "image_path": "data/docvqa_images/q6806_d2366.png", + "ucsf_document_id": "xlvf0001", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47821", + "questionId": "47821", + "docId": "14057", + "image_path": "data/docvqa_images/q47821_d14057.png", + "ucsf_document_id": "sphv0228", + "ucsf_document_page_no": "21", + "topic": "figure/diagram", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "53466", + "questionId": "53466", + "docId": "2749", + "image_path": "data/docvqa_images/q53466_d2749.png", + "ucsf_document_id": "fpjn0020", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "49406", + "questionId": "49406", + "docId": "14325", + "image_path": "data/docvqa_images/q49406_d14325.png", + "ucsf_document_id": "txpp0227", + "ucsf_document_page_no": "9", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "41799", + "questionId": "41799", + "docId": "11850", + "image_path": "data/docvqa_images/q41799_d11850.png", + "ucsf_document_id": "qjgn0226", + "ucsf_document_page_no": "194", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63730", + "questionId": "63730", + "docId": "9561", + "image_path": "data/docvqa_images/q63730_d9561.png", + "ucsf_document_id": "nlcf0227", + "ucsf_document_page_no": "3", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64381", + "questionId": "64381", + "docId": "10159", + "image_path": "data/docvqa_images/q64381_d10159.png", + "ucsf_document_id": "gxyd0217", + "ucsf_document_page_no": "10", + "topic": "Yes/No|form|handwritten|table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "4968", + "questionId": "4968", + "docId": "1977", + "image_path": "data/docvqa_images/q4968_d1977.png", + "ucsf_document_id": "rlmj0226", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "457", + "questionId": "457", + "docId": "306", + "image_path": "data/docvqa_images/q457_d306.png", + "ucsf_document_id": "mxxj0037", + "ucsf_document_page_no": "2", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "52459", + "questionId": "52459", + "docId": "2050", + "image_path": "data/docvqa_images/q52459_d2050.png", + "ucsf_document_id": "rxxk0225", + "ucsf_document_page_no": "12", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "18611", + "questionId": "18611", + "docId": "5691", + "image_path": "data/docvqa_images/q18611_d5691.png", + "ucsf_document_id": "fgbd0079", + "ucsf_document_page_no": "4", + "topic": "handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "12550", + "questionId": "12550", + "docId": "4024", + "image_path": "data/docvqa_images/q12550_d4024.png", + "ucsf_document_id": "zxlf0065", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "49366", + "questionId": "49366", + "docId": "14314", + "image_path": "data/docvqa_images/q49366_d14314.png", + "ucsf_document_id": "hqgb0228", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "61509", + "questionId": "61509", + "docId": "7751", + "image_path": "data/docvqa_images/q61509_d7751.png", + "ucsf_document_id": "krcy0227", + "ucsf_document_page_no": "27", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "2191", + "questionId": "2191", + "docId": "1400", + "image_path": "data/docvqa_images/q2191_d1400.png", + "ucsf_document_id": "ggjh0227", + "ucsf_document_page_no": "2", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "55340", + "questionId": "55340", + "docId": "14414", + "image_path": "data/docvqa_images/q55340_d14414.png", + "ucsf_document_id": "gnhm0227", + "ucsf_document_page_no": "7", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "18961", + "questionId": "18961", + "docId": "5773", + "image_path": "data/docvqa_images/q18961_d5773.png", + "ucsf_document_id": "npbb0079", + "ucsf_document_page_no": "10", + "topic": "handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "51548", + "questionId": "51548", + "docId": "1311", + "image_path": "data/docvqa_images/q51548_d1311.png", + "ucsf_document_id": "jtlh0227", + "ucsf_document_page_no": "10", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "51248", + "questionId": "51248", + "docId": "836", + "image_path": "data/docvqa_images/q51248_d836.png", + "ucsf_document_id": "pybn0226", + "ucsf_document_page_no": "1", + "topic": "free_text|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5591", + "questionId": "5591", + "docId": "1897", + "image_path": "data/docvqa_images/q5591_d1897.png", + "ucsf_document_id": "gpfl0225", + "ucsf_document_page_no": "7", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "61125", + "questionId": "61125", + "docId": "7604", + "image_path": "data/docvqa_images/q61125_d7604.png", + "ucsf_document_id": "ngmw0227", + "ucsf_document_page_no": "1", + "topic": "layout|others", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "58464", + "questionId": "58464", + "docId": "5315", + "image_path": "data/docvqa_images/q58464_d5315.png", + "ucsf_document_id": "mlbw0217", + "ucsf_document_page_no": "6", + "topic": "table/list|others", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "65203", + "questionId": "65203", + "docId": "10847", + "image_path": "data/docvqa_images/q65203_d10847.png", + "ucsf_document_id": "npvg0227", + "ucsf_document_page_no": "5", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63554", + "questionId": "63554", + "docId": "9304", + "image_path": "data/docvqa_images/q63554_d9304.png", + "ucsf_document_id": "jjvg0227", + "ucsf_document_page_no": "4", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "59850", + "questionId": "59850", + "docId": "6743", + "image_path": "data/docvqa_images/q59850_d6743.png", + "ucsf_document_id": "xngv0228", + "ucsf_document_page_no": "3", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63026", + "questionId": "63026", + "docId": "13532", + "image_path": "data/docvqa_images/q63026_d13532.png", + "ucsf_document_id": "mybw0217", + "ucsf_document_page_no": "5", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50815", + "questionId": "50815", + "docId": "14743", + "image_path": "data/docvqa_images/q50815_d14743.png", + "ucsf_document_id": "mskw0228", + "ucsf_document_page_no": "1", + "topic": "form|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "45151", + "questionId": "45151", + "docId": "13041", + "image_path": "data/docvqa_images/q45151_d13041.png", + "ucsf_document_id": "yjjl0226", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63250", + "questionId": "63250", + "docId": "9218", + "image_path": "data/docvqa_images/q63250_d9218.png", + "ucsf_document_id": "pqxf0227", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63771", + "questionId": "63771", + "docId": "9597", + "image_path": "data/docvqa_images/q63771_d9597.png", + "ucsf_document_id": "nldg0227", + "ucsf_document_page_no": "14", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5439", + "questionId": "5439", + "docId": "1838", + "image_path": "data/docvqa_images/q5439_d1838.png", + "ucsf_document_id": "gmgl0228", + "ucsf_document_page_no": "5", + "topic": "free_text|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "38912", + "questionId": "38912", + "docId": "11156", + "image_path": "data/docvqa_images/q38912_d11156.png", + "ucsf_document_id": "qsnc0227", + "ucsf_document_page_no": "72", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "52155", + "questionId": "52155", + "docId": "1796", + "image_path": "data/docvqa_images/q52155_d1796.png", + "ucsf_document_id": "pgfl0228", + "ucsf_document_page_no": "6", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "61164", + "questionId": "61164", + "docId": "7408", + "image_path": "data/docvqa_images/q61164_d7408.png", + "ucsf_document_id": "jldg0227", + "ucsf_document_page_no": "7", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "1748", + "questionId": "1748", + "docId": "704", + "image_path": "data/docvqa_images/q1748_d704.png", + "ucsf_document_id": "jzbn0226", + "ucsf_document_page_no": "14", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "45111", + "questionId": "45111", + "docId": "13037", + "image_path": "data/docvqa_images/q45111_d13037.png", + "ucsf_document_id": "yyml0226", + "ucsf_document_page_no": "3", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "60447", + "questionId": "60447", + "docId": "7167", + "image_path": "data/docvqa_images/q60447_d7167.png", + "ucsf_document_id": "mfyk0226", + "ucsf_document_page_no": "8", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "21707", + "questionId": "21707", + "docId": "6870", + "image_path": "data/docvqa_images/q21707_d6870.png", + "ucsf_document_id": "ffjw0228", + "ucsf_document_page_no": "11", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63710", + "questionId": "63710", + "docId": "9387", + "image_path": "data/docvqa_images/q63710_d9387.png", + "ucsf_document_id": "gpcg0227", + "ucsf_document_page_no": "3", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "45801", + "questionId": "45801", + "docId": "13560", + "image_path": "data/docvqa_images/q45801_d13560.png", + "ucsf_document_id": "ryvw0217", + "ucsf_document_page_no": "1", + "topic": "free_text|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "58697", + "questionId": "58697", + "docId": "5694", + "image_path": "data/docvqa_images/q58697_d5694.png", + "ucsf_document_id": "fgbd0079", + "ucsf_document_page_no": "7", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "56857", + "questionId": "56857", + "docId": "14906", + "image_path": "data/docvqa_images/q56857_d14906.png", + "ucsf_document_id": "jrcy0227", + "ucsf_document_page_no": "98", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5216", + "questionId": "5216", + "docId": "1768", + "image_path": "data/docvqa_images/q5216_d1768.png", + "ucsf_document_id": "nxkh0227", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "59607", + "questionId": "59607", + "docId": "6561", + "image_path": "data/docvqa_images/q59607_d6561.png", + "ucsf_document_id": "tzgv0228", + "ucsf_document_page_no": "7", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "54586", + "questionId": "54586", + "docId": "3706", + "image_path": "data/docvqa_images/q54586_d3706.png", + "ucsf_document_id": "lfng0023", + "ucsf_document_page_no": "3", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "7615", + "questionId": "7615", + "docId": "2668", + "image_path": "data/docvqa_images/q7615_d2668.png", + "ucsf_document_id": "flxn0020", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "62917", + "questionId": "62917", + "docId": "8966", + "image_path": "data/docvqa_images/q62917_d8966.png", + "ucsf_document_id": "qycc0228", + "ucsf_document_page_no": "4", + "topic": "Image/Photo", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "56441", + "questionId": "56441", + "docId": "14798", + "image_path": "data/docvqa_images/q56441_d14798.png", + "ucsf_document_id": "jnmw0228", + "ucsf_document_page_no": "1", + "topic": "form|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "25734", + "questionId": "25734", + "docId": "7326", + "image_path": "data/docvqa_images/q25734_d7326.png", + "ucsf_document_id": "jzhd0227", + "ucsf_document_page_no": "40", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57532", + "questionId": "57532", + "docId": "4843", + "image_path": "data/docvqa_images/q57532_d4843.png", + "ucsf_document_id": "snbx0223", + "ucsf_document_page_no": "7", + "topic": "Image/Photo", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57454", + "questionId": "57454", + "docId": "4826", + "image_path": "data/docvqa_images/q57454_d4826.png", + "ucsf_document_id": "snbx0223", + "ucsf_document_page_no": "19", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "41690", + "questionId": "41690", + "docId": "11831", + "image_path": "data/docvqa_images/q41690_d11831.png", + "ucsf_document_id": "kmwn0226", + "ucsf_document_page_no": "18", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "55027", + "questionId": "55027", + "docId": "4162", + "image_path": "data/docvqa_images/q55027_d4162.png", + "ucsf_document_id": "yldg0072", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "16429", + "questionId": "16429", + "docId": "5178", + "image_path": "data/docvqa_images/q16429_d5178.png", + "ucsf_document_id": "xjpn0081", + "ucsf_document_page_no": "4", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5827", + "questionId": "5827", + "docId": "1981", + "image_path": "data/docvqa_images/q5827_d1981.png", + "ucsf_document_id": "rlmj0226", + "ucsf_document_page_no": "7", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "1963", + "questionId": "1963", + "docId": "901", + "image_path": "data/docvqa_images/q1963_d901.png", + "ucsf_document_id": "slcn0226", + "ucsf_document_page_no": "3", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57402", + "questionId": "57402", + "docId": "4780", + "image_path": "data/docvqa_images/q57402_d4780.png", + "ucsf_document_id": "snbx0223", + "ucsf_document_page_no": "42", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64673", + "questionId": "64673", + "docId": "10446", + "image_path": "data/docvqa_images/q64673_d10446.png", + "ucsf_document_id": "hslf0227", + "ucsf_document_page_no": "9", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "36593", + "questionId": "36593", + "docId": "10553", + "image_path": "data/docvqa_images/q36593_d10553.png", + "ucsf_document_id": "hqvd0227", + "ucsf_document_page_no": "19", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "39089", + "questionId": "39089", + "docId": "11190", + "image_path": "data/docvqa_images/q39089_d11190.png", + "ucsf_document_id": "qqvf0227", + "ucsf_document_page_no": "1", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "18779", + "questionId": "18779", + "docId": "5732", + "image_path": "data/docvqa_images/q18779_d5732.png", + "ucsf_document_id": "gtph0079", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64387", + "questionId": "64387", + "docId": "10159", + "image_path": "data/docvqa_images/q64387_d10159.png", + "ucsf_document_id": "gxyd0217", + "ucsf_document_page_no": "10", + "topic": "form|handwritten|table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "43154", + "questionId": "43154", + "docId": "12417", + "image_path": "data/docvqa_images/q43154_d12417.png", + "ucsf_document_id": "kmxn0226", + "ucsf_document_page_no": "6", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "39069", + "questionId": "39069", + "docId": "11190", + "image_path": "data/docvqa_images/q39069_d11190.png", + "ucsf_document_id": "qqvf0227", + "ucsf_document_page_no": "1", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "7013", + "questionId": "7013", + "docId": "2392", + "image_path": "data/docvqa_images/q7013_d2392.png", + "ucsf_document_id": "gggw0004", + "ucsf_document_page_no": "2", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "42332", + "questionId": "42332", + "docId": "12065", + "image_path": "data/docvqa_images/q42332_d12065.png", + "ucsf_document_id": "krgn0226", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "1903", + "questionId": "1903", + "docId": "859", + "image_path": "data/docvqa_images/q1903_d859.png", + "ucsf_document_id": "jsbn0226", + "ucsf_document_page_no": "6", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "59928", + "questionId": "59928", + "docId": "6821", + "image_path": "data/docvqa_images/q59928_d6821.png", + "ucsf_document_id": "gmgv0228", + "ucsf_document_page_no": "1", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "51791", + "questionId": "51791", + "docId": "1361", + "image_path": "data/docvqa_images/q51791_d1361.png", + "ucsf_document_id": "slkk0226", + "ucsf_document_page_no": "1", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "16339", + "questionId": "16339", + "docId": "4878", + "image_path": "data/docvqa_images/q16339_d4878.png", + "ucsf_document_id": "snbx0223", + "ucsf_document_page_no": "228", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "15036", + "questionId": "15036", + "docId": "5024", + "image_path": "data/docvqa_images/q15036_d5024.png", + "ucsf_document_id": "sxvw0217", + "ucsf_document_page_no": "2", + "topic": "free_text|table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "58443", + "questionId": "58443", + "docId": "13850", + "image_path": "data/docvqa_images/q58443_d13850.png", + "ucsf_document_id": "fxwv0228", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "62859", + "questionId": "62859", + "docId": "8879", + "image_path": "data/docvqa_images/q62859_d8879.png", + "ucsf_document_id": "zmkp0227", + "ucsf_document_page_no": "7", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "16424", + "questionId": "16424", + "docId": "5177", + "image_path": "data/docvqa_images/q16424_d5177.png", + "ucsf_document_id": "hsyn0081", + "ucsf_document_page_no": "31", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "60048", + "questionId": "60048", + "docId": "6973", + "image_path": "data/docvqa_images/q60048_d6973.png", + "ucsf_document_id": "phwk0226", + "ucsf_document_page_no": "38", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50865", + "questionId": "50865", + "docId": "14731", + "image_path": "data/docvqa_images/q50865_d14731.png", + "ucsf_document_id": "rfgb0228", + "ucsf_document_page_no": "13", + "topic": "figure/diagram", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "52213", + "questionId": "52213", + "docId": "1773", + "image_path": "data/docvqa_images/q52213_d1773.png", + "ucsf_document_id": "lphk0226", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "1982", + "questionId": "1982", + "docId": "913", + "image_path": "data/docvqa_images/q1982_d913.png", + "ucsf_document_id": "jqbn0226", + "ucsf_document_page_no": "33", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "60872", + "questionId": "60872", + "docId": "7330", + "image_path": "data/docvqa_images/q60872_d7330.png", + "ucsf_document_id": "jzhd0227", + "ucsf_document_page_no": "67", + "topic": "table/list|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57312", + "questionId": "57312", + "docId": "4712", + "image_path": "data/docvqa_images/q57312_d4712.png", + "ucsf_document_id": "mtgj0223", + "ucsf_document_page_no": "17", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "56268", + "questionId": "56268", + "docId": "4448", + "image_path": "data/docvqa_images/q56268_d4448.png", + "ucsf_document_id": "jybx0223", + "ucsf_document_page_no": "84", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "55237", + "questionId": "55237", + "docId": "4356", + "image_path": "data/docvqa_images/q55237_d4356.png", + "ucsf_document_id": "hsgj0223", + "ucsf_document_page_no": "96", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50418", + "questionId": "50418", + "docId": "14601", + "image_path": "data/docvqa_images/q50418_d14601.png", + "ucsf_document_id": "qffw0228", + "ucsf_document_page_no": "23", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "62997", + "questionId": "62997", + "docId": "8920", + "image_path": "data/docvqa_images/q62997_d8920.png", + "ucsf_document_id": "xkdv0228", + "ucsf_document_page_no": "13", + "topic": "handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "25385", + "questionId": "25385", + "docId": "7224", + "image_path": "data/docvqa_images/q25385_d7224.png", + "ucsf_document_id": "rycg0227", + "ucsf_document_page_no": "7", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "44978", + "questionId": "44978", + "docId": "12966", + "image_path": "data/docvqa_images/q44978_d12966.png", + "ucsf_document_id": "qhll0226", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "3846", + "questionId": "3846", + "docId": "1424", + "image_path": "data/docvqa_images/q3846_d1424.png", + "ucsf_document_id": "nmmk0226", + "ucsf_document_page_no": "2", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50311", + "questionId": "50311", + "docId": "14575", + "image_path": "data/docvqa_images/q50311_d14575.png", + "ucsf_document_id": "knbd0228", + "ucsf_document_page_no": "1", + "topic": "handwritten|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "49325", + "questionId": "49325", + "docId": "14304", + "image_path": "data/docvqa_images/q49325_d14304.png", + "ucsf_document_id": "qqvv0228", + "ucsf_document_page_no": "2", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47064", + "questionId": "47064", + "docId": "13937", + "image_path": "data/docvqa_images/q47064_d13937.png", + "ucsf_document_id": "nznm0227", + "ucsf_document_page_no": "122", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47559", + "questionId": "47559", + "docId": "13691", + "image_path": "data/docvqa_images/q47559_d13691.png", + "ucsf_document_id": "frdv0228", + "ucsf_document_page_no": "4", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "40624", + "questionId": "40624", + "docId": "11562", + "image_path": "data/docvqa_images/q40624_d11562.png", + "ucsf_document_id": "xlpf0227", + "ucsf_document_page_no": "1", + "topic": "form|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5776", + "questionId": "5776", + "docId": "1940", + "image_path": "data/docvqa_images/q5776_d1940.png", + "ucsf_document_id": "pzyw0224", + "ucsf_document_page_no": "10", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "611", + "questionId": "611", + "docId": "361", + "image_path": "data/docvqa_images/q611_d361.png", + "ucsf_document_id": "yhpj0226", + "ucsf_document_page_no": "2", + "topic": "free_text|table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "9845", + "questionId": "9845", + "docId": "1867", + "image_path": "data/docvqa_images/q9845_d1867.png", + "ucsf_document_id": "ktfl0228", + "ucsf_document_page_no": "2", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "15323", + "questionId": "15323", + "docId": "4866", + "image_path": "data/docvqa_images/q15323_d4866.png", + "ucsf_document_id": "snbx0223", + "ucsf_document_page_no": "3", + "topic": "free_text|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50840", + "questionId": "50840", + "docId": "14741", + "image_path": "data/docvqa_images/q50840_d14741.png", + "ucsf_document_id": "ysmc0228", + "ucsf_document_page_no": "6", + "topic": "figure/diagram", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63569", + "questionId": "63569", + "docId": "9304", + "image_path": "data/docvqa_images/q63569_d9304.png", + "ucsf_document_id": "jjvg0227", + "ucsf_document_page_no": "4", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47044", + "questionId": "47044", + "docId": "13647", + "image_path": "data/docvqa_images/q47044_d13647.png", + "ucsf_document_id": "xndv0228", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "55160", + "questionId": "55160", + "docId": "4256", + "image_path": "data/docvqa_images/q55160_d4256.png", + "ucsf_document_id": "jkvj0223", + "ucsf_document_page_no": "21", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "40598", + "questionId": "40598", + "docId": "11562", + "image_path": "data/docvqa_images/q40598_d11562.png", + "ucsf_document_id": "xlpf0227", + "ucsf_document_page_no": "1", + "topic": "handwritten|layout|others", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "22313", + "questionId": "22313", + "docId": "6486", + "image_path": "data/docvqa_images/q22313_d6486.png", + "ucsf_document_id": "jzbx0227", + "ucsf_document_page_no": "4", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "55459", + "questionId": "55459", + "docId": "4331", + "image_path": "data/docvqa_images/q55459_d4331.png", + "ucsf_document_id": "gsgj0223", + "ucsf_document_page_no": "68", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "16463", + "questionId": "16463", + "docId": "5181", + "image_path": "data/docvqa_images/q16463_d5181.png", + "ucsf_document_id": "psyn0081", + "ucsf_document_page_no": "30", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "1024", + "questionId": "1024", + "docId": "494", + "image_path": "data/docvqa_images/q1024_d494.png", + "ucsf_document_id": "psjf0226", + "ucsf_document_page_no": "3", + "topic": "free_text|table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "45842", + "questionId": "45842", + "docId": "13402", + "image_path": "data/docvqa_images/q45842_d13402.png", + "ucsf_document_id": "zqdw0217", + "ucsf_document_page_no": "14", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "37223", + "questionId": "37223", + "docId": "10703", + "image_path": "data/docvqa_images/q37223_d10703.png", + "ucsf_document_id": "rnjg0227", + "ucsf_document_page_no": "1", + "topic": "form|layout|others", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "59357", + "questionId": "59357", + "docId": "6161", + "image_path": "data/docvqa_images/q59357_d6161.png", + "ucsf_document_id": "qxpn0081", + "ucsf_document_page_no": "9", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "2282", + "questionId": "2282", + "docId": "1009", + "image_path": "data/docvqa_images/q2282_d1009.png", + "ucsf_document_id": "tjpv0228", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "61353", + "questionId": "61353", + "docId": "7714", + "image_path": "data/docvqa_images/q61353_d7714.png", + "ucsf_document_id": "krcy0227", + "ucsf_document_page_no": "38", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "39946", + "questionId": "39946", + "docId": "11396", + "image_path": "data/docvqa_images/q39946_d11396.png", + "ucsf_document_id": "mslw0227", + "ucsf_document_page_no": "49", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "56556", + "questionId": "56556", + "docId": "5197", + "image_path": "data/docvqa_images/q56556_d5197.png", + "ucsf_document_id": "nynn0081", + "ucsf_document_page_no": "8", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "46226", + "questionId": "46226", + "docId": "13351", + "image_path": "data/docvqa_images/q46226_d13351.png", + "ucsf_document_id": "rmdw0217", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63892", + "questionId": "63892", + "docId": "9653", + "image_path": "data/docvqa_images/q63892_d9653.png", + "ucsf_document_id": "lgpg0227", + "ucsf_document_page_no": "11", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "59461", + "questionId": "59461", + "docId": "6263", + "image_path": "data/docvqa_images/q59461_d6263.png", + "ucsf_document_id": "kmcj0227", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "52300", + "questionId": "52300", + "docId": "2432", + "image_path": "data/docvqa_images/q52300_d2432.png", + "ucsf_document_id": "mfnf0004", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "869", + "questionId": "869", + "docId": "427", + "image_path": "data/docvqa_images/q869_d427.png", + "ucsf_document_id": "nhpj0226", + "ucsf_document_page_no": "1", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "4745", + "questionId": "4745", + "docId": "1985", + "image_path": "data/docvqa_images/q4745_d1985.png", + "ucsf_document_id": "spwx0225", + "ucsf_document_page_no": "9", + "topic": "handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "21132", + "questionId": "21132", + "docId": "6299", + "image_path": "data/docvqa_images/q21132_d6299.png", + "ucsf_document_id": "hkhx0227", + "ucsf_document_page_no": "2", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63657", + "questionId": "63657", + "docId": "9360", + "image_path": "data/docvqa_images/q63657_d9360.png", + "ucsf_document_id": "gkvh0227", + "ucsf_document_page_no": "1", + "topic": "layout|others", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64278", + "questionId": "64278", + "docId": "10132", + "image_path": "data/docvqa_images/q64278_d10132.png", + "ucsf_document_id": "lpjm0223", + "ucsf_document_page_no": "56", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57170", + "questionId": "57170", + "docId": "6369", + "image_path": "data/docvqa_images/q57170_d6369.png", + "ucsf_document_id": "zmwm0227", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "1191", + "questionId": "1191", + "docId": "538", + "image_path": "data/docvqa_images/q1191_d538.png", + "ucsf_document_id": "rtjf0226", + "ucsf_document_page_no": "1", + "topic": "form|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "61578", + "questionId": "61578", + "docId": "8000", + "image_path": "data/docvqa_images/q61578_d8000.png", + "ucsf_document_id": "fqyf0227", + "ucsf_document_page_no": "15", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64831", + "questionId": "64831", + "docId": "9160", + "image_path": "data/docvqa_images/q64831_d9160.png", + "ucsf_document_id": "qnyg0227", + "ucsf_document_page_no": "5", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64940", + "questionId": "64940", + "docId": "10713", + "image_path": "data/docvqa_images/q64940_d10713.png", + "ucsf_document_id": "xhfg0227", + "ucsf_document_page_no": "23", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "61605", + "questionId": "61605", + "docId": "7827", + "image_path": "data/docvqa_images/q61605_d7827.png", + "ucsf_document_id": "hlhv0228", + "ucsf_document_page_no": "3", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "16832", + "questionId": "16832", + "docId": "5324", + "image_path": "data/docvqa_images/q16832_d5324.png", + "ucsf_document_id": "rgcw0217", + "ucsf_document_page_no": "7", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47712", + "questionId": "47712", + "docId": "13832", + "image_path": "data/docvqa_images/q47712_d13832.png", + "ucsf_document_id": "fmnm0227", + "ucsf_document_page_no": "6", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "56471", + "questionId": "56471", + "docId": "14791", + "image_path": "data/docvqa_images/q56471_d14791.png", + "ucsf_document_id": "knpp0227", + "ucsf_document_page_no": "4", + "topic": "handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "265", + "questionId": "265", + "docId": "244", + "image_path": "data/docvqa_images/q265_d244.png", + "ucsf_document_id": "lycj0037", + "ucsf_document_page_no": "8", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64319", + "questionId": "64319", + "docId": "10189", + "image_path": "data/docvqa_images/q64319_d10189.png", + "ucsf_document_id": "gxyd0217", + "ucsf_document_page_no": "8", + "topic": "handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63779", + "questionId": "63779", + "docId": "9603", + "image_path": "data/docvqa_images/q63779_d9603.png", + "ucsf_document_id": "qnwd0227", + "ucsf_document_page_no": "25", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "45103", + "questionId": "45103", + "docId": "12994", + "image_path": "data/docvqa_images/q45103_d12994.png", + "ucsf_document_id": "nsnl0226", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "6025", + "questionId": "6025", + "docId": "2149", + "image_path": "data/docvqa_images/q6025_d2149.png", + "ucsf_document_id": "pgxg0224", + "ucsf_document_page_no": "5", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64285", + "questionId": "64285", + "docId": "10137", + "image_path": "data/docvqa_images/q64285_d10137.png", + "ucsf_document_id": "lpjm0223", + "ucsf_document_page_no": "59", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50506", + "questionId": "50506", + "docId": "245", + "image_path": "data/docvqa_images/q50506_d245.png", + "ucsf_document_id": "nrcj0037", + "ucsf_document_page_no": "8", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "55366", + "questionId": "55366", + "docId": "4297", + "image_path": "data/docvqa_images/q55366_d4297.png", + "ucsf_document_id": "fmvj0223", + "ucsf_document_page_no": "2", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "4978", + "questionId": "4978", + "docId": "1982", + "image_path": "data/docvqa_images/q4978_d1982.png", + "ucsf_document_id": "lxcj0224", + "ucsf_document_page_no": "1", + "topic": "form|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "59350", + "questionId": "59350", + "docId": "6175", + "image_path": "data/docvqa_images/q59350_d6175.png", + "ucsf_document_id": "srwn0081", + "ucsf_document_page_no": "5", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64169", + "questionId": "64169", + "docId": "9866", + "image_path": "data/docvqa_images/q64169_d9866.png", + "ucsf_document_id": "yllg0227", + "ucsf_document_page_no": "2", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "44830", + "questionId": "44830", + "docId": "12882", + "image_path": "data/docvqa_images/q44830_d12882.png", + "ucsf_document_id": "rmwn0226", + "ucsf_document_page_no": "81", + "topic": "figure/diagram|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63178", + "questionId": "63178", + "docId": "9099", + "image_path": "data/docvqa_images/q63178_d9099.png", + "ucsf_document_id": "jlmf0227", + "ucsf_document_page_no": "11", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "56075", + "questionId": "56075", + "docId": "5156", + "image_path": "data/docvqa_images/q56075_d5156.png", + "ucsf_document_id": "lnmm0081", + "ucsf_document_page_no": "6", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "58782", + "questionId": "58782", + "docId": "5785", + "image_path": "data/docvqa_images/q58782_d5785.png", + "ucsf_document_id": "khmk0079", + "ucsf_document_page_no": "2", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "61228", + "questionId": "61228", + "docId": "7471", + "image_path": "data/docvqa_images/q61228_d7471.png", + "ucsf_document_id": "ggpf0227", + "ucsf_document_page_no": "27", + "topic": "figure/diagram|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "52274", + "questionId": "52274", + "docId": "1804", + "image_path": "data/docvqa_images/q52274_d1804.png", + "ucsf_document_id": "ypgl0228", + "ucsf_document_page_no": "3", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "26594", + "questionId": "26594", + "docId": "7525", + "image_path": "data/docvqa_images/q26594_d7525.png", + "ucsf_document_id": "nfxd0227", + "ucsf_document_page_no": "3", + "topic": "handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "40587", + "questionId": "40587", + "docId": "11562", + "image_path": "data/docvqa_images/q40587_d11562.png", + "ucsf_document_id": "xlpf0227", + "ucsf_document_page_no": "1", + "topic": "form|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64936", + "questionId": "64936", + "docId": "10811", + "image_path": "data/docvqa_images/q64936_d10811.png", + "ucsf_document_id": "ylwg0227", + "ucsf_document_page_no": "15", + "topic": "figure/diagram|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50873", + "questionId": "50873", + "docId": "14727", + "image_path": "data/docvqa_images/q50873_d14727.png", + "ucsf_document_id": "lsww0228", + "ucsf_document_page_no": "3", + "topic": "handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57564", + "questionId": "57564", + "docId": "4851", + "image_path": "data/docvqa_images/q57564_d4851.png", + "ucsf_document_id": "snbx0223", + "ucsf_document_page_no": "21", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57798", + "questionId": "57798", + "docId": "4856", + "image_path": "data/docvqa_images/q57798_d4856.png", + "ucsf_document_id": "tnbx0223", + "ucsf_document_page_no": "130", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "54958", + "questionId": "54958", + "docId": "14307", + "image_path": "data/docvqa_images/q54958_d14307.png", + "ucsf_document_id": "sspp0227", + "ucsf_document_page_no": "30", + "topic": "figure/diagram", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5675", + "questionId": "5675", + "docId": "1911", + "image_path": "data/docvqa_images/q5675_d1911.png", + "ucsf_document_id": "npwx0225", + "ucsf_document_page_no": "10", + "topic": "handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "49586", + "questionId": "49586", + "docId": "14386", + "image_path": "data/docvqa_images/q49586_d14386.png", + "ucsf_document_id": "fygb0228", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "3116", + "questionId": "3116", + "docId": "1216", + "image_path": "data/docvqa_images/q3116_d1216.png", + "ucsf_document_id": "sxyv0228", + "ucsf_document_page_no": "2", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "55095", + "questionId": "55095", + "docId": "4225", + "image_path": "data/docvqa_images/q55095_d4225.png", + "ucsf_document_id": "nfdg0072", + "ucsf_document_page_no": "12", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "43484", + "questionId": "43484", + "docId": "12473", + "image_path": "data/docvqa_images/q43484_d12473.png", + "ucsf_document_id": "lpgn0226", + "ucsf_document_page_no": "23", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "25714", + "questionId": "25714", + "docId": "7319", + "image_path": "data/docvqa_images/q25714_d7319.png", + "ucsf_document_id": "hsbd0227", + "ucsf_document_page_no": "11", + "topic": "layout|others", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "1261", + "questionId": "1261", + "docId": "555", + "image_path": "data/docvqa_images/q1261_d555.png", + "ucsf_document_id": "jpjf0226", + "ucsf_document_page_no": "1", + "topic": "free_text|table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "16826", + "questionId": "16826", + "docId": "5324", + "image_path": "data/docvqa_images/q16826_d5324.png", + "ucsf_document_id": "rgcw0217", + "ucsf_document_page_no": "7", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64610", + "questionId": "64610", + "docId": "10376", + "image_path": "data/docvqa_images/q64610_d10376.png", + "ucsf_document_id": "hslf0227", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5824", + "questionId": "5824", + "docId": "1942", + "image_path": "data/docvqa_images/q5824_d1942.png", + "ucsf_document_id": "zqdj0224", + "ucsf_document_page_no": "11", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "25788", + "questionId": "25788", + "docId": "7345", + "image_path": "data/docvqa_images/q25788_d7345.png", + "ucsf_document_id": "mldg0227", + "ucsf_document_page_no": "2", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "58431", + "questionId": "58431", + "docId": "5926", + "image_path": "data/docvqa_images/q58431_d5926.png", + "ucsf_document_id": "lybx0227", + "ucsf_document_page_no": "23", + "topic": "figure/diagram|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "6775", + "questionId": "6775", + "docId": "2359", + "image_path": "data/docvqa_images/q6775_d2359.png", + "ucsf_document_id": "hnhp0000", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63859", + "questionId": "63859", + "docId": "9647", + "image_path": "data/docvqa_images/q63859_d9647.png", + "ucsf_document_id": "kzng0227", + "ucsf_document_page_no": "2", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "44130", + "questionId": "44130", + "docId": "12648", + "image_path": "data/docvqa_images/q44130_d12648.png", + "ucsf_document_id": "rmwn0226", + "ucsf_document_page_no": "79", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "43265", + "questionId": "43265", + "docId": "12426", + "image_path": "data/docvqa_images/q43265_d12426.png", + "ucsf_document_id": "hmxn0226", + "ucsf_document_page_no": "3", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "283", + "questionId": "283", + "docId": "256", + "image_path": "data/docvqa_images/q283_d256.png", + "ucsf_document_id": "nhxj0037", + "ucsf_document_page_no": "3", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "42222", + "questionId": "42222", + "docId": "11955", + "image_path": "data/docvqa_images/q42222_d11955.png", + "ucsf_document_id": "kfgn0226", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "65152", + "questionId": "65152", + "docId": "10772", + "image_path": "data/docvqa_images/q65152_d10772.png", + "ucsf_document_id": "pxlg0227", + "ucsf_document_page_no": "2", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "16609", + "questionId": "16609", + "docId": "5258", + "image_path": "data/docvqa_images/q16609_d5258.png", + "ucsf_document_id": "rpcw0217", + "ucsf_document_page_no": "2", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57300", + "questionId": "57300", + "docId": "6278", + "image_path": "data/docvqa_images/q57300_d6278.png", + "ucsf_document_id": "zmcj0227", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "45625", + "questionId": "45625", + "docId": "12717", + "image_path": "data/docvqa_images/q45625_d12717.png", + "ucsf_document_id": "glxn0226", + "ucsf_document_page_no": "4", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "2218", + "questionId": "2218", + "docId": "1260", + "image_path": "data/docvqa_images/q2218_d1260.png", + "ucsf_document_id": "mymk0226", + "ucsf_document_page_no": "9", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64617", + "questionId": "64617", + "docId": "10406", + "image_path": "data/docvqa_images/q64617_d10406.png", + "ucsf_document_id": "hslf0227", + "ucsf_document_page_no": "5", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47454", + "questionId": "47454", + "docId": "13639", + "image_path": "data/docvqa_images/q47454_d13639.png", + "ucsf_document_id": "skdv0228", + "ucsf_document_page_no": "5", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "61764", + "questionId": "61764", + "docId": "8056", + "image_path": "data/docvqa_images/q61764_d8056.png", + "ucsf_document_id": "hrfw0227", + "ucsf_document_page_no": "12", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "65115", + "questionId": "65115", + "docId": "10712", + "image_path": "data/docvqa_images/q65115_d10712.png", + "ucsf_document_id": "lmmf0227", + "ucsf_document_page_no": "3", + "topic": "table/list|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63537", + "questionId": "63537", + "docId": "9293", + "image_path": "data/docvqa_images/q63537_d9293.png", + "ucsf_document_id": "hgwd0227", + "ucsf_document_page_no": "34", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + } +] diff --git a/data/docvqa_id_split/train/items.json b/data/docvqa_id_split/train/items.json new file mode 100644 index 0000000..b2066e7 --- /dev/null +++ b/data/docvqa_id_split/train/items.json @@ -0,0 +1,1393 @@ +[ + { + "id": "50877", + "questionId": "50877", + "docId": "14724", + "image_path": "data/docvqa_images/q50877_d14724.png", + "ucsf_document_id": "ghlw0228", + "ucsf_document_page_no": "2", + "topic": "handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "62132", + "questionId": "62132", + "docId": "8327", + "image_path": "data/docvqa_images/q62132_d8327.png", + "ucsf_document_id": "jqbg0227", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "60996", + "questionId": "60996", + "docId": "7599", + "image_path": "data/docvqa_images/q60996_d7599.png", + "ucsf_document_id": "kgbg0227", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "46117", + "questionId": "46117", + "docId": "13503", + "image_path": "data/docvqa_images/q46117_d13503.png", + "ucsf_document_id": "xmww0217", + "ucsf_document_page_no": "17", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "37456", + "questionId": "37456", + "docId": "10794", + "image_path": "data/docvqa_images/q37456_d10794.png", + "ucsf_document_id": "ypbd0227", + "ucsf_document_page_no": "5", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "51768", + "questionId": "51768", + "docId": "14553", + "image_path": "data/docvqa_images/q51768_d14553.png", + "ucsf_document_id": "fhwc0228", + "ucsf_document_page_no": "12", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5646", + "questionId": "5646", + "docId": "1902", + "image_path": "data/docvqa_images/q5646_d1902.png", + "ucsf_document_id": "qzlj0226", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57087", + "questionId": "57087", + "docId": "4624", + "image_path": "data/docvqa_images/q57087_d4624.png", + "ucsf_document_id": "mnbx0223", + "ucsf_document_page_no": "74", + "topic": "table/list|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "33923", + "questionId": "33923", + "docId": "9616", + "image_path": "data/docvqa_images/q33923_d9616.png", + "ucsf_document_id": "nldg0227", + "ucsf_document_page_no": "13", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "60961", + "questionId": "60961", + "docId": "7572", + "image_path": "data/docvqa_images/q60961_d7572.png", + "ucsf_document_id": "mzlw0227", + "ucsf_document_page_no": "1", + "topic": "layout|others", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "32871", + "questionId": "32871", + "docId": "9252", + "image_path": "data/docvqa_images/q32871_d9252.png", + "ucsf_document_id": "hldg0227", + "ucsf_document_page_no": "7", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "6869", + "questionId": "6869", + "docId": "2314", + "image_path": "data/docvqa_images/q6869_d2314.png", + "ucsf_document_id": "rpmy0000", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "6010", + "questionId": "6010", + "docId": "2143", + "image_path": "data/docvqa_images/q6010_d2143.png", + "ucsf_document_id": "fllg0224", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "37650", + "questionId": "37650", + "docId": "10833", + "image_path": "data/docvqa_images/q37650_d10833.png", + "ucsf_document_id": "yjvg0227", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5474", + "questionId": "5474", + "docId": "1853", + "image_path": "data/docvqa_images/q5474_d1853.png", + "ucsf_document_id": "rkgl0228", + "ucsf_document_page_no": "3", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5612", + "questionId": "5612", + "docId": "1898", + "image_path": "data/docvqa_images/q5612_d1898.png", + "ucsf_document_id": "nhgh0228", + "ucsf_document_page_no": "6", + "topic": "free_text|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63700", + "questionId": "63700", + "docId": "9552", + "image_path": "data/docvqa_images/q63700_d9552.png", + "ucsf_document_id": "kjhf0227", + "ucsf_document_page_no": "3", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "61551", + "questionId": "61551", + "docId": "7805", + "image_path": "data/docvqa_images/q61551_d7805.png", + "ucsf_document_id": "zxkp0227", + "ucsf_document_page_no": "4", + "topic": "handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5778", + "questionId": "5778", + "docId": "1940", + "image_path": "data/docvqa_images/q5778_d1940.png", + "ucsf_document_id": "pzyw0224", + "ucsf_document_page_no": "10", + "topic": "form|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57801", + "questionId": "57801", + "docId": "4856", + "image_path": "data/docvqa_images/q57801_d4856.png", + "ucsf_document_id": "tnbx0223", + "ucsf_document_page_no": "130", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "40714", + "questionId": "40714", + "docId": "11589", + "image_path": "data/docvqa_images/q40714_d11589.png", + "ucsf_document_id": "hthg0227", + "ucsf_document_page_no": "8", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "51244", + "questionId": "51244", + "docId": "836", + "image_path": "data/docvqa_images/q51244_d836.png", + "ucsf_document_id": "pybn0226", + "ucsf_document_page_no": "1", + "topic": "form|table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47071", + "questionId": "47071", + "docId": "13937", + "image_path": "data/docvqa_images/q47071_d13937.png", + "ucsf_document_id": "nznm0227", + "ucsf_document_page_no": "122", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "32546", + "questionId": "32546", + "docId": "9150", + "image_path": "data/docvqa_images/q32546_d9150.png", + "ucsf_document_id": "ztvg0227", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "51661", + "questionId": "51661", + "docId": "1203", + "image_path": "data/docvqa_images/q51661_d1203.png", + "ucsf_document_id": "hnjh0227", + "ucsf_document_page_no": "3", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "44841", + "questionId": "44841", + "docId": "12889", + "image_path": "data/docvqa_images/q44841_d12889.png", + "ucsf_document_id": "rmwn0226", + "ucsf_document_page_no": "95", + "topic": "figure/diagram", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5199", + "questionId": "5199", + "docId": "1768", + "image_path": "data/docvqa_images/q5199_d1768.png", + "ucsf_document_id": "nxkh0227", + "ucsf_document_page_no": "1", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "6351", + "questionId": "6351", + "docId": "2225", + "image_path": "data/docvqa_images/q6351_d2225.png", + "ucsf_document_id": "gmhp0000", + "ucsf_document_page_no": "2", + "topic": "free_text|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "61548", + "questionId": "61548", + "docId": "8776", + "image_path": "data/docvqa_images/q61548_d8776.png", + "ucsf_document_id": "yslf0227", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "56301", + "questionId": "56301", + "docId": "5034", + "image_path": "data/docvqa_images/q56301_d5034.png", + "ucsf_document_id": "rrxm0081", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "60467", + "questionId": "60467", + "docId": "13534", + "image_path": "data/docvqa_images/q60467_d13534.png", + "ucsf_document_id": "rrcw0217", + "ucsf_document_page_no": "2", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "6787", + "questionId": "6787", + "docId": "2363", + "image_path": "data/docvqa_images/q6787_d2363.png", + "ucsf_document_id": "jygp0000", + "ucsf_document_page_no": "3", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63638", + "questionId": "63638", + "docId": "9346", + "image_path": "data/docvqa_images/q63638_d9346.png", + "ucsf_document_id": "fncf0227", + "ucsf_document_page_no": "2", + "topic": "free_text|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "271", + "questionId": "271", + "docId": "248", + "image_path": "data/docvqa_images/q271_d248.png", + "ucsf_document_id": "kscl0037", + "ucsf_document_page_no": "3", + "topic": "handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "19245", + "questionId": "19245", + "docId": "508", + "image_path": "data/docvqa_images/q19245_d508.png", + "ucsf_document_id": "gyjf0226", + "ucsf_document_page_no": "4", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "59796", + "questionId": "59796", + "docId": "6701", + "image_path": "data/docvqa_images/q59796_d6701.png", + "ucsf_document_id": "njnp0227", + "ucsf_document_page_no": "15", + "topic": "free_text|others", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "62826", + "questionId": "62826", + "docId": "8866", + "image_path": "data/docvqa_images/q62826_d8866.png", + "ucsf_document_id": "qxhc0228", + "ucsf_document_page_no": "6", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "61642", + "questionId": "61642", + "docId": "7887", + "image_path": "data/docvqa_images/q61642_d7887.png", + "ucsf_document_id": "gtjc0228", + "ucsf_document_page_no": "4", + "topic": "figure/diagram", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57254", + "questionId": "57254", + "docId": "4746", + "image_path": "data/docvqa_images/q57254_d4746.png", + "ucsf_document_id": "snbx0223", + "ucsf_document_page_no": "36", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "60171", + "questionId": "60171", + "docId": "7071", + "image_path": "data/docvqa_images/q60171_d7071.png", + "ucsf_document_id": "lnbl0226", + "ucsf_document_page_no": "2", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "62478", + "questionId": "62478", + "docId": "8573", + "image_path": "data/docvqa_images/q62478_d8573.png", + "ucsf_document_id": "qfvg0227", + "ucsf_document_page_no": "2", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "18831", + "questionId": "18831", + "docId": "5749", + "image_path": "data/docvqa_images/q18831_d5749.png", + "ucsf_document_id": "jhfd0079", + "ucsf_document_page_no": "9", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "1182", + "questionId": "1182", + "docId": "536", + "image_path": "data/docvqa_images/q1182_d536.png", + "ucsf_document_id": "gmjf0226", + "ucsf_document_page_no": "4", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "62646", + "questionId": "62646", + "docId": "8670", + "image_path": "data/docvqa_images/q62646_d8670.png", + "ucsf_document_id": "ztwd0227", + "ucsf_document_page_no": "2", + "topic": "layout|others", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57391", + "questionId": "57391", + "docId": "4772", + "image_path": "data/docvqa_images/q57391_d4772.png", + "ucsf_document_id": "snbx0223", + "ucsf_document_page_no": "14", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "17001", + "questionId": "17001", + "docId": "5370", + "image_path": "data/docvqa_images/q17001_d5370.png", + "ucsf_document_id": "tfcw0217", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "45726", + "questionId": "45726", + "docId": "13221", + "image_path": "data/docvqa_images/q45726_d13221.png", + "ucsf_document_id": "yscw0217", + "ucsf_document_page_no": "129", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "59600", + "questionId": "59600", + "docId": "6561", + "image_path": "data/docvqa_images/q59600_d6561.png", + "ucsf_document_id": "tzgv0228", + "ucsf_document_page_no": "7", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "62840", + "questionId": "62840", + "docId": "8870", + "image_path": "data/docvqa_images/q62840_d8870.png", + "ucsf_document_id": "kjlp0227", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "55449", + "questionId": "55449", + "docId": "4331", + "image_path": "data/docvqa_images/q55449_d4331.png", + "ucsf_document_id": "gsgj0223", + "ucsf_document_page_no": "68", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57969", + "questionId": "57969", + "docId": "4920", + "image_path": "data/docvqa_images/q57969_d4920.png", + "ucsf_document_id": "lkvw0217", + "ucsf_document_page_no": "2", + "topic": "free_text|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "55949", + "questionId": "55949", + "docId": "5121", + "image_path": "data/docvqa_images/q55949_d5121.png", + "ucsf_document_id": "hsyn0081", + "ucsf_document_page_no": "17", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "32892", + "questionId": "32892", + "docId": "9257", + "image_path": "data/docvqa_images/q32892_d9257.png", + "ucsf_document_id": "kqch0227", + "ucsf_document_page_no": "3", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "7008", + "questionId": "7008", + "docId": "2392", + "image_path": "data/docvqa_images/q7008_d2392.png", + "ucsf_document_id": "gggw0004", + "ucsf_document_page_no": "2", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "56464", + "questionId": "56464", + "docId": "14794", + "image_path": "data/docvqa_images/q56464_d14794.png", + "ucsf_document_id": "gnnp0227", + "ucsf_document_page_no": "4", + "topic": "Yes/No|handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "63109", + "questionId": "63109", + "docId": "9048", + "image_path": "data/docvqa_images/q63109_d9048.png", + "ucsf_document_id": "xlwc0228", + "ucsf_document_page_no": "3", + "topic": "handwritten|table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50307", + "questionId": "50307", + "docId": "14571", + "image_path": "data/docvqa_images/q50307_d14571.png", + "ucsf_document_id": "rrdd0228", + "ucsf_document_page_no": "14", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64599", + "questionId": "64599", + "docId": "10376", + "image_path": "data/docvqa_images/q64599_d10376.png", + "ucsf_document_id": "hslf0227", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "60863", + "questionId": "60863", + "docId": "7328", + "image_path": "data/docvqa_images/q60863_d7328.png", + "ucsf_document_id": "jzhd0227", + "ucsf_document_page_no": "16", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "12577", + "questionId": "12577", + "docId": "4035", + "image_path": "data/docvqa_images/q12577_d4035.png", + "ucsf_document_id": "ykmg0065", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "58750", + "questionId": "58750", + "docId": "6153", + "image_path": "data/docvqa_images/q58750_d6153.png", + "ucsf_document_id": "rnbx0223", + "ucsf_document_page_no": "33", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5663", + "questionId": "5663", + "docId": "1902", + "image_path": "data/docvqa_images/q5663_d1902.png", + "ucsf_document_id": "qzlj0226", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "51085", + "questionId": "51085", + "docId": "673", + "image_path": "data/docvqa_images/q51085_d673.png", + "ucsf_document_id": "prbn0226", + "ucsf_document_page_no": "16", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "60811", + "questionId": "60811", + "docId": "7299", + "image_path": "data/docvqa_images/q60811_d7299.png", + "ucsf_document_id": "phvd0227", + "ucsf_document_page_no": "10", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64494", + "questionId": "64494", + "docId": "10311", + "image_path": "data/docvqa_images/q64494_d10311.png", + "ucsf_document_id": "lpdl0226", + "ucsf_document_page_no": "35", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "297", + "questionId": "297", + "docId": "258", + "image_path": "data/docvqa_images/q297_d258.png", + "ucsf_document_id": "rzbj0037", + "ucsf_document_page_no": "7", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57145", + "questionId": "57145", + "docId": "4692", + "image_path": "data/docvqa_images/q57145_d4692.png", + "ucsf_document_id": "mtgj0223", + "ucsf_document_page_no": "19", + "topic": "table/list|others", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "377", + "questionId": "377", + "docId": "272", + "image_path": "data/docvqa_images/q377_d272.png", + "ucsf_document_id": "hjxj0037", + "ucsf_document_page_no": "2", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "53696", + "questionId": "53696", + "docId": "3630", + "image_path": "data/docvqa_images/q53696_d3630.png", + "ucsf_document_id": "rhhx0023", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "18842", + "questionId": "18842", + "docId": "5755", + "image_path": "data/docvqa_images/q18842_d5755.png", + "ucsf_document_id": "npbb0079", + "ucsf_document_page_no": "11", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5181", + "questionId": "5181", + "docId": "1763", + "image_path": "data/docvqa_images/q5181_d1763.png", + "ucsf_document_id": "grlh0227", + "ucsf_document_page_no": "5", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "52153", + "questionId": "52153", + "docId": "1341", + "image_path": "data/docvqa_images/q52153_d1341.png", + "ucsf_document_id": "jxmk0226", + "ucsf_document_page_no": "3", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57526", + "questionId": "57526", + "docId": "4847", + "image_path": "data/docvqa_images/q57526_d4847.png", + "ucsf_document_id": "snbx0223", + "ucsf_document_page_no": "15", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "45117", + "questionId": "45117", + "docId": "13037", + "image_path": "data/docvqa_images/q45117_d13037.png", + "ucsf_document_id": "yyml0226", + "ucsf_document_page_no": "3", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "59304", + "questionId": "59304", + "docId": "6128", + "image_path": "data/docvqa_images/q59304_d6128.png", + "ucsf_document_id": "rnbx0223", + "ucsf_document_page_no": "53", + "topic": "layout|Image/Photo", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5388", + "questionId": "5388", + "docId": "1817", + "image_path": "data/docvqa_images/q5388_d1817.png", + "ucsf_document_id": "xhfl0228", + "ucsf_document_page_no": "7", + "topic": "figure/diagram|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64583", + "questionId": "64583", + "docId": "10364", + "image_path": "data/docvqa_images/q64583_d10364.png", + "ucsf_document_id": "lpdl0226", + "ucsf_document_page_no": "13", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57900", + "questionId": "57900", + "docId": "4898", + "image_path": "data/docvqa_images/q57900_d4898.png", + "ucsf_document_id": "txvw0217", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "596", + "questionId": "596", + "docId": "357", + "image_path": "data/docvqa_images/q596_d357.png", + "ucsf_document_id": "mtyj0226", + "ucsf_document_page_no": "15", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64324", + "questionId": "64324", + "docId": "10189", + "image_path": "data/docvqa_images/q64324_d10189.png", + "ucsf_document_id": "gxyd0217", + "ucsf_document_page_no": "8", + "topic": "handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "32881", + "questionId": "32881", + "docId": "9253", + "image_path": "data/docvqa_images/q32881_d9253.png", + "ucsf_document_id": "hnhd0227", + "ucsf_document_page_no": "8", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "9381", + "questionId": "9381", + "docId": "3115", + "image_path": "data/docvqa_images/q9381_d3115.png", + "ucsf_document_id": "plxw0023", + "ucsf_document_page_no": "1", + "topic": "handwritten|table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "59539", + "questionId": "59539", + "docId": "6256", + "image_path": "data/docvqa_images/q59539_d6256.png", + "ucsf_document_id": "xhcc0228", + "ucsf_document_page_no": "3", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "49279", + "questionId": "49279", + "docId": "14184", + "image_path": "data/docvqa_images/q49279_d14184.png", + "ucsf_document_id": "flpp0227", + "ucsf_document_page_no": "16", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "53468", + "questionId": "53468", + "docId": "2749", + "image_path": "data/docvqa_images/q53468_d2749.png", + "ucsf_document_id": "fpjn0020", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57141", + "questionId": "57141", + "docId": "4692", + "image_path": "data/docvqa_images/q57141_d4692.png", + "ucsf_document_id": "mtgj0223", + "ucsf_document_page_no": "19", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "51134", + "questionId": "51134", + "docId": "700", + "image_path": "data/docvqa_images/q51134_d700.png", + "ucsf_document_id": "kzbn0226", + "ucsf_document_page_no": "18", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "3042", + "questionId": "3042", + "docId": "1204", + "image_path": "data/docvqa_images/q3042_d1204.png", + "ucsf_document_id": "xfjv0228", + "ucsf_document_page_no": "3", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "62404", + "questionId": "62404", + "docId": "8554", + "image_path": "data/docvqa_images/q62404_d8554.png", + "ucsf_document_id": "pgjw0227", + "ucsf_document_page_no": "5", + "topic": "others", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "8122", + "questionId": "8122", + "docId": "2834", + "image_path": "data/docvqa_images/q8122_d2834.png", + "ucsf_document_id": "zxjw0023", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47974", + "questionId": "47974", + "docId": "14084", + "image_path": "data/docvqa_images/q47974_d14084.png", + "ucsf_document_id": "fphv0228", + "ucsf_document_page_no": "8", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "1183", + "questionId": "1183", + "docId": "536", + "image_path": "data/docvqa_images/q1183_d536.png", + "ucsf_document_id": "gmjf0226", + "ucsf_document_page_no": "4", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "6003", + "questionId": "6003", + "docId": "2143", + "image_path": "data/docvqa_images/q6003_d2143.png", + "ucsf_document_id": "fllg0224", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "42304", + "questionId": "42304", + "docId": "12048", + "image_path": "data/docvqa_images/q42304_d12048.png", + "ucsf_document_id": "fkxn0226", + "ucsf_document_page_no": "14", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "61503", + "questionId": "61503", + "docId": "7751", + "image_path": "data/docvqa_images/q61503_d7751.png", + "ucsf_document_id": "krcy0227", + "ucsf_document_page_no": "27", + "topic": "Image/Photo", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "17150", + "questionId": "17150", + "docId": "5403", + "image_path": "data/docvqa_images/q17150_d5403.png", + "ucsf_document_id": "kfdw0217", + "ucsf_document_page_no": "1", + "topic": "others", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "62995", + "questionId": "62995", + "docId": "8920", + "image_path": "data/docvqa_images/q62995_d8920.png", + "ucsf_document_id": "xkdv0228", + "ucsf_document_page_no": "13", + "topic": "handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "46299", + "questionId": "46299", + "docId": "13360", + "image_path": "data/docvqa_images/q46299_d13360.png", + "ucsf_document_id": "ysbw0217", + "ucsf_document_page_no": "3", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "43600", + "questionId": "43600", + "docId": "12508", + "image_path": "data/docvqa_images/q43600_d12508.png", + "ucsf_document_id": "gmwn0226", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "62001", + "questionId": "62001", + "docId": "8197", + "image_path": "data/docvqa_images/q62001_d8197.png", + "ucsf_document_id": "fgkw0228", + "ucsf_document_page_no": "4", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "9367", + "questionId": "9367", + "docId": "3115", + "image_path": "data/docvqa_images/q9367_d3115.png", + "ucsf_document_id": "plxw0023", + "ucsf_document_page_no": "1", + "topic": "handwritten|form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50817", + "questionId": "50817", + "docId": "14743", + "image_path": "data/docvqa_images/q50817_d14743.png", + "ucsf_document_id": "mskw0228", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "3847", + "questionId": "3847", + "docId": "1424", + "image_path": "data/docvqa_images/q3847_d1424.png", + "ucsf_document_id": "nmmk0226", + "ucsf_document_page_no": "2", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50362", + "questionId": "50362", + "docId": "14589", + "image_path": "data/docvqa_images/q50362_d14589.png", + "ucsf_document_id": "gscv0228", + "ucsf_document_page_no": "6", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64152", + "questionId": "64152", + "docId": "9857", + "image_path": "data/docvqa_images/q64152_d9857.png", + "ucsf_document_id": "xglg0227", + "ucsf_document_page_no": "10", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "59693", + "questionId": "59693", + "docId": "6703", + "image_path": "data/docvqa_images/q59693_d6703.png", + "ucsf_document_id": "qtxb0228", + "ucsf_document_page_no": "1", + "topic": "form|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "59502", + "questionId": "59502", + "docId": "6255", + "image_path": "data/docvqa_images/q59502_d6255.png", + "ucsf_document_id": "rpvm0227", + "ucsf_document_page_no": "23", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + } +] diff --git a/data/docvqa_id_split/val/items.json b/data/docvqa_id_split/val/items.json new file mode 100644 index 0000000..89b4acc --- /dev/null +++ b/data/docvqa_id_split/val/items.json @@ -0,0 +1,691 @@ +[ + { + "id": "62409", + "questionId": "62409", + "docId": "8554", + "image_path": "data/docvqa_images/q62409_d8554.png", + "ucsf_document_id": "pgjw0227", + "ucsf_document_page_no": "5", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50961", + "questionId": "50961", + "docId": "549", + "image_path": "data/docvqa_images/q50961_d549.png", + "ucsf_document_id": "qtjf0226", + "ucsf_document_page_no": "2", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "46461", + "questionId": "46461", + "docId": "13361", + "image_path": "data/docvqa_images/q46461_d13361.png", + "ucsf_document_id": "ysbw0217", + "ucsf_document_page_no": "5", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "3041", + "questionId": "3041", + "docId": "1204", + "image_path": "data/docvqa_images/q3041_d1204.png", + "ucsf_document_id": "xfjv0228", + "ucsf_document_page_no": "3", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "41716", + "questionId": "41716", + "docId": "11835", + "image_path": "data/docvqa_images/q41716_d11835.png", + "ucsf_document_id": "qjgn0226", + "ucsf_document_page_no": "131", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "61123", + "questionId": "61123", + "docId": "7374", + "image_path": "data/docvqa_images/q61123_d7374.png", + "ucsf_document_id": "mldg0227", + "ucsf_document_page_no": "5", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "43068", + "questionId": "43068", + "docId": "12393", + "image_path": "data/docvqa_images/q43068_d12393.png", + "ucsf_document_id": "rmwn0226", + "ucsf_document_page_no": "52", + "topic": "figure/diagram", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "51221", + "questionId": "51221", + "docId": "764", + "image_path": "data/docvqa_images/q51221_d764.png", + "ucsf_document_id": "kzbn0226", + "ucsf_document_page_no": "14", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "6397", + "questionId": "6397", + "docId": "2242", + "image_path": "data/docvqa_images/q6397_d2242.png", + "ucsf_document_id": "jkcn0000", + "ucsf_document_page_no": "2", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57428", + "questionId": "57428", + "docId": "4779", + "image_path": "data/docvqa_images/q57428_d4779.png", + "ucsf_document_id": "rnbx0223", + "ucsf_document_page_no": "208", + "topic": "Image/Photo", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "3135", + "questionId": "3135", + "docId": "1221", + "image_path": "data/docvqa_images/q3135_d1221.png", + "ucsf_document_id": "ngph0227", + "ucsf_document_page_no": "5", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "18819", + "questionId": "18819", + "docId": "5749", + "image_path": "data/docvqa_images/q18819_d5749.png", + "ucsf_document_id": "jhfd0079", + "ucsf_document_page_no": "9", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "15382", + "questionId": "15382", + "docId": "4890", + "image_path": "data/docvqa_images/q15382_d4890.png", + "ucsf_document_id": "kjvw0217", + "ucsf_document_page_no": "3", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5772", + "questionId": "5772", + "docId": "1940", + "image_path": "data/docvqa_images/q5772_d1940.png", + "ucsf_document_id": "pzyw0224", + "ucsf_document_page_no": "10", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "49077", + "questionId": "49077", + "docId": "14179", + "image_path": "data/docvqa_images/q49077_d14179.png", + "ucsf_document_id": "nrxb0228", + "ucsf_document_page_no": "3", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "58519", + "questionId": "58519", + "docId": "5347", + "image_path": "data/docvqa_images/q58519_d5347.png", + "ucsf_document_id": "sjbw0217", + "ucsf_document_page_no": "11", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50720", + "questionId": "50720", + "docId": "281", + "image_path": "data/docvqa_images/q50720_d281.png", + "ucsf_document_id": "nrcj0037", + "ucsf_document_page_no": "7", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "56785", + "questionId": "56785", + "docId": "14289", + "image_path": "data/docvqa_images/q56785_d14289.png", + "ucsf_document_id": "xkbv0228", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "59653", + "questionId": "59653", + "docId": "6579", + "image_path": "data/docvqa_images/q59653_d6579.png", + "ucsf_document_id": "mzbx0227", + "ucsf_document_page_no": "2", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "61791", + "questionId": "61791", + "docId": "8072", + "image_path": "data/docvqa_images/q61791_d8072.png", + "ucsf_document_id": "hfmf0227", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "37229", + "questionId": "37229", + "docId": "10742", + "image_path": "data/docvqa_images/q37229_d10742.png", + "ucsf_document_id": "nkcd0227", + "ucsf_document_page_no": "2", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "60407", + "questionId": "60407", + "docId": "7135", + "image_path": "data/docvqa_images/q60407_d7135.png", + "ucsf_document_id": "gkpk0226", + "ucsf_document_page_no": "1", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64420", + "questionId": "64420", + "docId": "10230", + "image_path": "data/docvqa_images/q64420_d10230.png", + "ucsf_document_id": "jnjm0223", + "ucsf_document_page_no": "107", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47365", + "questionId": "47365", + "docId": "13813", + "image_path": "data/docvqa_images/q47365_d13813.png", + "ucsf_document_id": "nxym0227", + "ucsf_document_page_no": "28", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "47458", + "questionId": "47458", + "docId": "13639", + "image_path": "data/docvqa_images/q47458_d13639.png", + "ucsf_document_id": "skdv0228", + "ucsf_document_page_no": "5", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "7621", + "questionId": "7621", + "docId": "2668", + "image_path": "data/docvqa_images/q7621_d2668.png", + "ucsf_document_id": "flxn0020", + "ucsf_document_page_no": "1", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "53575", + "questionId": "53575", + "docId": "2766", + "image_path": "data/docvqa_images/q53575_d2766.png", + "ucsf_document_id": "hsfn0020", + "ucsf_document_page_no": "2", + "topic": "free_text|table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "60913", + "questionId": "60913", + "docId": "7349", + "image_path": "data/docvqa_images/q60913_d7349.png", + "ucsf_document_id": "jzhd0227", + "ucsf_document_page_no": "61", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "60454", + "questionId": "60454", + "docId": "7163", + "image_path": "data/docvqa_images/q60454_d7163.png", + "ucsf_document_id": "jgyk0226", + "ucsf_document_page_no": "1", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57978", + "questionId": "57978", + "docId": "4920", + "image_path": "data/docvqa_images/q57978_d4920.png", + "ucsf_document_id": "lkvw0217", + "ucsf_document_page_no": "2", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64547", + "questionId": "64547", + "docId": "10361", + "image_path": "data/docvqa_images/q64547_d10361.png", + "ucsf_document_id": "lpdl0226", + "ucsf_document_page_no": "32", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "59481", + "questionId": "59481", + "docId": "6243", + "image_path": "data/docvqa_images/q59481_d6243.png", + "ucsf_document_id": "psgv0228", + "ucsf_document_page_no": "5", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "61472", + "questionId": "61472", + "docId": "7757", + "image_path": "data/docvqa_images/q61472_d7757.png", + "ucsf_document_id": "ymkp0227", + "ucsf_document_page_no": "13", + "topic": "handwritten|table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5673", + "questionId": "5673", + "docId": "1908", + "image_path": "data/docvqa_images/q5673_d1908.png", + "ucsf_document_id": "lldj0224", + "ucsf_document_page_no": "2", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "49109", + "questionId": "49109", + "docId": "13644", + "image_path": "data/docvqa_images/q49109_d13644.png", + "ucsf_document_id": "mzdv0228", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "46123", + "questionId": "46123", + "docId": "13503", + "image_path": "data/docvqa_images/q46123_d13503.png", + "ucsf_document_id": "xmww0217", + "ucsf_document_page_no": "17", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "48158", + "questionId": "48158", + "docId": "13976", + "image_path": "data/docvqa_images/q48158_d13976.png", + "ucsf_document_id": "zqhm0227", + "ucsf_document_page_no": "1", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "1955", + "questionId": "1955", + "docId": "892", + "image_path": "data/docvqa_images/q1955_d892.png", + "ucsf_document_id": "jsbn0226", + "ucsf_document_page_no": "2", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "8127", + "questionId": "8127", + "docId": "2754", + "image_path": "data/docvqa_images/q8127_d2754.png", + "ucsf_document_id": "xtvn0020", + "ucsf_document_page_no": "2", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57431", + "questionId": "57431", + "docId": "4779", + "image_path": "data/docvqa_images/q57431_d4779.png", + "ucsf_document_id": "rnbx0223", + "ucsf_document_page_no": "208", + "topic": "Image/Photo", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64306", + "questionId": "64306", + "docId": "10149", + "image_path": "data/docvqa_images/q64306_d10149.png", + "ucsf_document_id": "lpjm0223", + "ucsf_document_page_no": "23", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "64887", + "questionId": "64887", + "docId": "9754", + "image_path": "data/docvqa_images/q64887_d9754.png", + "ucsf_document_id": "szpg0227", + "ucsf_document_page_no": "9", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "58680", + "questionId": "58680", + "docId": "5545", + "image_path": "data/docvqa_images/q58680_d5545.png", + "ucsf_document_id": "hhwh0078", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "5287", + "questionId": "5287", + "docId": "1785", + "image_path": "data/docvqa_images/q5287_d1785.png", + "ucsf_document_id": "mtnh0227", + "ucsf_document_page_no": "10", + "topic": "form", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "55471", + "questionId": "55471", + "docId": "4340", + "image_path": "data/docvqa_images/q55471_d4340.png", + "ucsf_document_id": "fsgj0223", + "ucsf_document_page_no": "96", + "topic": "free_text", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "53095", + "questionId": "53095", + "docId": "296", + "image_path": "data/docvqa_images/q53095_d296.png", + "ucsf_document_id": "qhxj0037", + "ucsf_document_page_no": "3", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "53726", + "questionId": "53726", + "docId": "2008", + "image_path": "data/docvqa_images/q53726_d2008.png", + "ucsf_document_id": "hhnf0094", + "ucsf_document_page_no": "5", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "57321", + "questionId": "57321", + "docId": "4722", + "image_path": "data/docvqa_images/q57321_d4722.png", + "ucsf_document_id": "xybx0223", + "ucsf_document_page_no": "32", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "26659", + "questionId": "26659", + "docId": "7470", + "image_path": "data/docvqa_images/q26659_d7470.png", + "ucsf_document_id": "lhmg0227", + "ucsf_document_page_no": "1", + "topic": "layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "38920", + "questionId": "38920", + "docId": "11157", + "image_path": "data/docvqa_images/q38920_d11157.png", + "ucsf_document_id": "klnf0227", + "ucsf_document_page_no": "1", + "topic": "table/list|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "50837", + "questionId": "50837", + "docId": "14742", + "image_path": "data/docvqa_images/q50837_d14742.png", + "ucsf_document_id": "ysmc0228", + "ucsf_document_page_no": "4", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "59615", + "questionId": "59615", + "docId": "6569", + "image_path": "data/docvqa_images/q59615_d6569.png", + "ucsf_document_id": "hnnp0227", + "ucsf_document_page_no": "45", + "topic": "handwritten|table/list|layout", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + }, + { + "id": "58687", + "questionId": "58687", + "docId": "5545", + "image_path": "data/docvqa_images/q58687_d5545.png", + "ucsf_document_id": "hhwh0078", + "ucsf_document_page_no": "1", + "topic": "table/list", + "source_dataset": "lmms-lab/DocVQA", + "source_config": "DocVQA", + "source_split": "validation", + "sample_seed": "full_validation_5349" + } +] diff --git a/data/livemathematicianbench_id_split/split_manifest.json b/data/livemathematicianbench_id_split/split_manifest.json new file mode 100644 index 0000000..9af68a0 --- /dev/null +++ b/data/livemathematicianbench_id_split/split_manifest.json @@ -0,0 +1,34 @@ +{ + "benchmark": "LiveMathematicianBench", + "manifest_type": "id_split", + "source_repo": "LiveMathematicianBench/LiveMathematicianBench", + "source_repo_type": "dataset", + "source_url": "https://huggingface.co/datasets/LiveMathematicianBench/LiveMathematicianBench", + "source_revision": "b72450f6ce96c26158d64d945a5d31ef7727be41", + "source_files": [ + "data/202511/qa_202511_final.json", + "data/202512/qa_202512_final.json", + "data/202601/qa_202601_final.json", + "data/202602/qa_202602_final.json" + ], + "split_mode": "ratio", + "split_ratio": "2:1:7", + "split_seed": 42, + "counts": { + "train": 35, + "val": 18, + "test": 124 + }, + "item_fields": [ + "id", + "month", + "no", + "paper_link", + "source_file" + ], + "id_format": ":", + "notes": [ + "This is an ID split manifest, not the full LiveMathematicianBench payload.", + "Materialize full split items from the official LiveMathematicianBench raw qa_*_final.json files before evaluation." + ] +} diff --git a/data/livemathematicianbench_id_split/test/items.json b/data/livemathematicianbench_id_split/test/items.json new file mode 100644 index 0000000..83006f3 --- /dev/null +++ b/data/livemathematicianbench_id_split/test/items.json @@ -0,0 +1,870 @@ +[ + { + "id": "202602:12", + "month": "202602", + "no": 12, + "paper_link": "http://arxiv.org/abs/2602.07171v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202601:3", + "month": "202601", + "no": 3, + "paper_link": "http://arxiv.org/abs/2601.01447v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202511:4", + "month": "202511", + "no": 4, + "paper_link": "http://arxiv.org/abs/2511.23123v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202601:20", + "month": "202601", + "no": 20, + "paper_link": "http://arxiv.org/abs/2601.13212v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202601:42", + "month": "202601", + "no": 42, + "paper_link": "http://arxiv.org/abs/2601.09348v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202512:38", + "month": "202512", + "no": 38, + "paper_link": "http://arxiv.org/abs/2512.19831v2", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202512:4", + "month": "202512", + "no": 4, + "paper_link": "http://arxiv.org/abs/2512.03141v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202602:4", + "month": "202602", + "no": 4, + "paper_link": "http://arxiv.org/abs/2602.14368v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202511:15", + "month": "202511", + "no": 15, + "paper_link": "http://arxiv.org/abs/2511.17325v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202602:32", + "month": "202602", + "no": 32, + "paper_link": "http://arxiv.org/abs/2602.14817v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202512:51", + "month": "202512", + "no": 51, + "paper_link": "http://arxiv.org/abs/2512.14581v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202512:26", + "month": "202512", + "no": 26, + "paper_link": "http://arxiv.org/abs/2512.19586v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202601:13", + "month": "202601", + "no": 13, + "paper_link": "http://arxiv.org/abs/2601.10017v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202602:1", + "month": "202602", + "no": 1, + "paper_link": "http://arxiv.org/abs/2602.23137v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202511:18", + "month": "202511", + "no": 18, + "paper_link": "http://arxiv.org/abs/2511.10795v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202512:5", + "month": "202512", + "no": 5, + "paper_link": "http://arxiv.org/abs/2512.00348v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202511:19", + "month": "202511", + "no": 19, + "paper_link": "http://arxiv.org/abs/2511.06951v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202602:40", + "month": "202602", + "no": 40, + "paper_link": "http://arxiv.org/abs/2602.20462v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202602:29", + "month": "202602", + "no": 29, + "paper_link": "http://arxiv.org/abs/2602.10676v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202512:35", + "month": "202512", + "no": 35, + "paper_link": "http://arxiv.org/abs/2512.08840v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202512:48", + "month": "202512", + "no": 48, + "paper_link": "http://arxiv.org/abs/2512.03482v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202512:52", + "month": "202512", + "no": 52, + "paper_link": "http://arxiv.org/abs/2512.11246v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202512:44", + "month": "202512", + "no": 44, + "paper_link": "http://arxiv.org/abs/2512.10385v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202511:28", + "month": "202511", + "no": 28, + "paper_link": "http://arxiv.org/abs/2511.03812v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202601:43", + "month": "202601", + "no": 43, + "paper_link": "http://arxiv.org/abs/2601.22555v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202602:9", + "month": "202602", + "no": 9, + "paper_link": "http://arxiv.org/abs/2602.19882v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202512:23", + "month": "202512", + "no": 23, + "paper_link": "http://arxiv.org/abs/2512.09180v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202602:21", + "month": "202602", + "no": 21, + "paper_link": "http://arxiv.org/abs/2602.10509v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202511:5", + "month": "202511", + "no": 5, + "paper_link": "http://arxiv.org/abs/2511.20164v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202601:35", + "month": "202601", + "no": 35, + "paper_link": "http://arxiv.org/abs/2601.15606v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202602:50", + "month": "202602", + "no": 50, + "paper_link": "http://arxiv.org/abs/2602.05652v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202512:13", + "month": "202512", + "no": 13, + "paper_link": "http://arxiv.org/abs/2512.22861v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202602:49", + "month": "202602", + "no": 49, + "paper_link": "http://arxiv.org/abs/2602.07167v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202602:18", + "month": "202602", + "no": 18, + "paper_link": "http://arxiv.org/abs/2602.20124v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202601:15", + "month": "202601", + "no": 15, + "paper_link": "http://arxiv.org/abs/2601.05327v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202601:21", + "month": "202601", + "no": 21, + "paper_link": "http://arxiv.org/abs/2601.04994v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202601:32", + "month": "202601", + "no": 32, + "paper_link": "http://arxiv.org/abs/2601.09183v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202602:34", + "month": "202602", + "no": 34, + "paper_link": "http://arxiv.org/abs/2602.21118v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202602:20", + "month": "202602", + "no": 20, + "paper_link": "http://arxiv.org/abs/2602.16506v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202602:5", + "month": "202602", + "no": 5, + "paper_link": "http://arxiv.org/abs/2602.09806v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202512:40", + "month": "202512", + "no": 40, + "paper_link": "http://arxiv.org/abs/2512.16535v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202511:22", + "month": "202511", + "no": 22, + "paper_link": "http://arxiv.org/abs/2511.07607v2", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202601:36", + "month": "202601", + "no": 36, + "paper_link": "http://arxiv.org/abs/2601.12457v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202512:49", + "month": "202512", + "no": 49, + "paper_link": "http://arxiv.org/abs/2512.21565v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202511:10", + "month": "202511", + "no": 10, + "paper_link": "http://arxiv.org/abs/2511.06484v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202601:2", + "month": "202601", + "no": 2, + "paper_link": "http://arxiv.org/abs/2601.07068v4", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202602:19", + "month": "202602", + "no": 19, + "paper_link": "http://arxiv.org/abs/2602.18179v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202601:9", + "month": "202601", + "no": 9, + "paper_link": "http://arxiv.org/abs/2601.17765v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202512:6", + "month": "202512", + "no": 6, + "paper_link": "http://arxiv.org/abs/2512.23079v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202601:5", + "month": "202601", + "no": 5, + "paper_link": "http://arxiv.org/abs/2601.20344v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202602:14", + "month": "202602", + "no": 14, + "paper_link": "http://arxiv.org/abs/2602.09177v2", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202512:17", + "month": "202512", + "no": 17, + "paper_link": "http://arxiv.org/abs/2512.11657v2", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202512:19", + "month": "202512", + "no": 19, + "paper_link": "http://arxiv.org/abs/2512.16655v2", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202602:33", + "month": "202602", + "no": 33, + "paper_link": "http://arxiv.org/abs/2602.13734v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202512:18", + "month": "202512", + "no": 18, + "paper_link": "http://arxiv.org/abs/2512.22960v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202601:26", + "month": "202601", + "no": 26, + "paper_link": "http://arxiv.org/abs/2601.06814v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202601:1", + "month": "202601", + "no": 1, + "paper_link": "http://arxiv.org/abs/2601.18276v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202512:30", + "month": "202512", + "no": 30, + "paper_link": "http://arxiv.org/abs/2512.07260v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202602:44", + "month": "202602", + "no": 44, + "paper_link": "http://arxiv.org/abs/2602.01138v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202512:20", + "month": "202512", + "no": 20, + "paper_link": "http://arxiv.org/abs/2512.14575v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202511:13", + "month": "202511", + "no": 13, + "paper_link": "http://arxiv.org/abs/2511.16910v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202601:30", + "month": "202601", + "no": 30, + "paper_link": "http://arxiv.org/abs/2601.12140v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202601:40", + "month": "202601", + "no": 40, + "paper_link": "http://arxiv.org/abs/2601.05146v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202601:29", + "month": "202601", + "no": 29, + "paper_link": "http://arxiv.org/abs/2601.12846v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202511:11", + "month": "202511", + "no": 11, + "paper_link": "http://arxiv.org/abs/2511.17548v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202512:9", + "month": "202512", + "no": 9, + "paper_link": "http://arxiv.org/abs/2512.08817v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202601:18", + "month": "202601", + "no": 18, + "paper_link": "http://arxiv.org/abs/2601.01797v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202512:1", + "month": "202512", + "no": 1, + "paper_link": "http://arxiv.org/abs/2512.20055v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202601:4", + "month": "202601", + "no": 4, + "paper_link": "http://arxiv.org/abs/2601.21223v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202511:6", + "month": "202511", + "no": 6, + "paper_link": "http://arxiv.org/abs/2511.14959v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202602:38", + "month": "202602", + "no": 38, + "paper_link": "http://arxiv.org/abs/2602.08398v2", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202601:10", + "month": "202601", + "no": 10, + "paper_link": "http://arxiv.org/abs/2601.15524v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202602:11", + "month": "202602", + "no": 11, + "paper_link": "http://arxiv.org/abs/2602.11045v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202512:45", + "month": "202512", + "no": 45, + "paper_link": "http://arxiv.org/abs/2512.08395v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202601:12", + "month": "202601", + "no": 12, + "paper_link": "http://arxiv.org/abs/2601.11877v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202512:47", + "month": "202512", + "no": 47, + "paper_link": "http://arxiv.org/abs/2512.09683v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202511:21", + "month": "202511", + "no": 21, + "paper_link": "http://arxiv.org/abs/2511.21288v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202601:16", + "month": "202601", + "no": 16, + "paper_link": "http://arxiv.org/abs/2601.05008v2", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202512:3", + "month": "202512", + "no": 3, + "paper_link": "http://arxiv.org/abs/2512.13450v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202601:37", + "month": "202601", + "no": 37, + "paper_link": "http://arxiv.org/abs/2601.09443v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202511:12", + "month": "202511", + "no": 12, + "paper_link": "http://arxiv.org/abs/2511.04978v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202512:39", + "month": "202512", + "no": 39, + "paper_link": "http://arxiv.org/abs/2512.19003v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202601:8", + "month": "202601", + "no": 8, + "paper_link": "http://arxiv.org/abs/2601.19754v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202601:11", + "month": "202601", + "no": 11, + "paper_link": "http://arxiv.org/abs/2601.13552v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202511:25", + "month": "202511", + "no": 25, + "paper_link": "http://arxiv.org/abs/2511.10548v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202601:17", + "month": "202601", + "no": 17, + "paper_link": "http://arxiv.org/abs/2601.02655v2", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202602:36", + "month": "202602", + "no": 36, + "paper_link": "http://arxiv.org/abs/2602.13001v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202602:43", + "month": "202602", + "no": 43, + "paper_link": "http://arxiv.org/abs/2602.06897v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202601:6", + "month": "202601", + "no": 6, + "paper_link": "http://arxiv.org/abs/2601.04747v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202602:35", + "month": "202602", + "no": 35, + "paper_link": "http://arxiv.org/abs/2602.20938v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202512:11", + "month": "202512", + "no": 11, + "paper_link": "http://arxiv.org/abs/2512.03294v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202602:23", + "month": "202602", + "no": 23, + "paper_link": "http://arxiv.org/abs/2602.09201v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202601:7", + "month": "202601", + "no": 7, + "paper_link": "http://arxiv.org/abs/2601.02859v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202602:39", + "month": "202602", + "no": 39, + "paper_link": "http://arxiv.org/abs/2602.21659v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202512:12", + "month": "202512", + "no": 12, + "paper_link": "http://arxiv.org/abs/2512.00690v3", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202511:2", + "month": "202511", + "no": 2, + "paper_link": "http://arxiv.org/abs/2511.19681v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202512:43", + "month": "202512", + "no": 43, + "paper_link": "http://arxiv.org/abs/2512.10820v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202602:24", + "month": "202602", + "no": 24, + "paper_link": "http://arxiv.org/abs/2602.08680v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202601:34", + "month": "202601", + "no": 34, + "paper_link": "http://arxiv.org/abs/2601.07318v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202512:28", + "month": "202512", + "no": 28, + "paper_link": "http://arxiv.org/abs/2512.11294v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202601:27", + "month": "202601", + "no": 27, + "paper_link": "http://arxiv.org/abs/2601.05692v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202602:42", + "month": "202602", + "no": 42, + "paper_link": "http://arxiv.org/abs/2602.09749v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202512:22", + "month": "202512", + "no": 22, + "paper_link": "http://arxiv.org/abs/2512.11658v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202602:17", + "month": "202602", + "no": 17, + "paper_link": "http://arxiv.org/abs/2602.22504v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202602:48", + "month": "202602", + "no": 48, + "paper_link": "http://arxiv.org/abs/2602.08760v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202602:28", + "month": "202602", + "no": 28, + "paper_link": "http://arxiv.org/abs/2602.11595v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202602:3", + "month": "202602", + "no": 3, + "paper_link": "http://arxiv.org/abs/2602.17369v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202512:31", + "month": "202512", + "no": 31, + "paper_link": "http://arxiv.org/abs/2512.23668v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202512:27", + "month": "202512", + "no": 27, + "paper_link": "http://arxiv.org/abs/2512.16505v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202511:24", + "month": "202511", + "no": 24, + "paper_link": "http://arxiv.org/abs/2511.12549v2", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202511:8", + "month": "202511", + "no": 8, + "paper_link": "http://arxiv.org/abs/2511.12657v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202511:9", + "month": "202511", + "no": 9, + "paper_link": "http://arxiv.org/abs/2511.09015v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202601:28", + "month": "202601", + "no": 28, + "paper_link": "http://arxiv.org/abs/2601.14825v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202602:25", + "month": "202602", + "no": 25, + "paper_link": "http://arxiv.org/abs/2602.16048v3", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202511:23", + "month": "202511", + "no": 23, + "paper_link": "http://arxiv.org/abs/2511.06595v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202602:13", + "month": "202602", + "no": 13, + "paper_link": "http://arxiv.org/abs/2602.12261v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202511:27", + "month": "202511", + "no": 27, + "paper_link": "http://arxiv.org/abs/2511.04407v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202512:7", + "month": "202512", + "no": 7, + "paper_link": "http://arxiv.org/abs/2512.09490v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202512:29", + "month": "202512", + "no": 29, + "paper_link": "http://arxiv.org/abs/2512.08562v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202512:34", + "month": "202512", + "no": 34, + "paper_link": "http://arxiv.org/abs/2512.09598v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202512:42", + "month": "202512", + "no": 42, + "paper_link": "http://arxiv.org/abs/2512.10845v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202511:7", + "month": "202511", + "no": 7, + "paper_link": "http://arxiv.org/abs/2511.13976v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202511:29", + "month": "202511", + "no": 29, + "paper_link": "http://arxiv.org/abs/2511.03722v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202602:37", + "month": "202602", + "no": 37, + "paper_link": "http://arxiv.org/abs/2602.08644v1", + "source_file": "data/202602/qa_202602_final.json" + } +] diff --git a/data/livemathematicianbench_id_split/train/items.json b/data/livemathematicianbench_id_split/train/items.json new file mode 100644 index 0000000..d0f65ba --- /dev/null +++ b/data/livemathematicianbench_id_split/train/items.json @@ -0,0 +1,247 @@ +[ + { + "id": "202602:22", + "month": "202602", + "no": 22, + "paper_link": "http://arxiv.org/abs/2602.10700v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202512:8", + "month": "202512", + "no": 8, + "paper_link": "http://arxiv.org/abs/2512.08863v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202511:16", + "month": "202511", + "no": 16, + "paper_link": "http://arxiv.org/abs/2511.15668v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202601:44", + "month": "202601", + "no": 44, + "paper_link": "http://arxiv.org/abs/2601.21267v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202511:14", + "month": "202511", + "no": 14, + "paper_link": "http://arxiv.org/abs/2511.13447v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202602:30", + "month": "202602", + "no": 30, + "paper_link": "http://arxiv.org/abs/2602.16692v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202602:2", + "month": "202602", + "no": 2, + "paper_link": "http://arxiv.org/abs/2602.22933v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202601:41", + "month": "202601", + "no": 41, + "paper_link": "http://arxiv.org/abs/2601.01164v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202601:23", + "month": "202601", + "no": 23, + "paper_link": "http://arxiv.org/abs/2601.02528v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202511:20", + "month": "202511", + "no": 20, + "paper_link": "http://arxiv.org/abs/2511.02963v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202601:22", + "month": "202601", + "no": 22, + "paper_link": "http://arxiv.org/abs/2601.03984v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202512:14", + "month": "202512", + "no": 14, + "paper_link": "http://arxiv.org/abs/2512.22459v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202511:26", + "month": "202511", + "no": 26, + "paper_link": "http://arxiv.org/abs/2511.07817v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202511:3", + "month": "202511", + "no": 3, + "paper_link": "http://arxiv.org/abs/2511.11409v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202601:33", + "month": "202601", + "no": 33, + "paper_link": "http://arxiv.org/abs/2601.07747v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202602:7", + "month": "202602", + "no": 7, + "paper_link": "http://arxiv.org/abs/2602.22912v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202602:27", + "month": "202602", + "no": 27, + "paper_link": "http://arxiv.org/abs/2602.13968v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202602:31", + "month": "202602", + "no": 31, + "paper_link": "http://arxiv.org/abs/2602.15528v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202602:41", + "month": "202602", + "no": 41, + "paper_link": "http://arxiv.org/abs/2602.10707v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202512:25", + "month": "202512", + "no": 25, + "paper_link": "http://arxiv.org/abs/2512.04531v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202602:10", + "month": "202602", + "no": 10, + "paper_link": "http://arxiv.org/abs/2602.17863v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202602:16", + "month": "202602", + "no": 16, + "paper_link": "http://arxiv.org/abs/2602.02723v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202512:16", + "month": "202512", + "no": 16, + "paper_link": "http://arxiv.org/abs/2512.11601v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202512:2", + "month": "202512", + "no": 2, + "paper_link": "http://arxiv.org/abs/2512.16120v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202512:24", + "month": "202512", + "no": 24, + "paper_link": "http://arxiv.org/abs/2512.08391v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202512:32", + "month": "202512", + "no": 32, + "paper_link": "http://arxiv.org/abs/2512.23224v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202602:47", + "month": "202602", + "no": 47, + "paper_link": "http://arxiv.org/abs/2602.10391v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202602:46", + "month": "202602", + "no": 46, + "paper_link": "http://arxiv.org/abs/2602.13727v2", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202512:21", + "month": "202512", + "no": 21, + "paper_link": "http://arxiv.org/abs/2512.12835v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202512:33", + "month": "202512", + "no": 33, + "paper_link": "http://arxiv.org/abs/2512.19500v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202602:45", + "month": "202602", + "no": 45, + "paper_link": "http://arxiv.org/abs/2602.23912v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202602:26", + "month": "202602", + "no": 26, + "paper_link": "http://arxiv.org/abs/2602.14658v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202512:41", + "month": "202512", + "no": 41, + "paper_link": "http://arxiv.org/abs/2512.15177v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202601:38", + "month": "202601", + "no": 38, + "paper_link": "http://arxiv.org/abs/2601.07817v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202601:14", + "month": "202601", + "no": 14, + "paper_link": "http://arxiv.org/abs/2601.08704v1", + "source_file": "data/202601/qa_202601_final.json" + } +] diff --git a/data/livemathematicianbench_id_split/val/items.json b/data/livemathematicianbench_id_split/val/items.json new file mode 100644 index 0000000..4298592 --- /dev/null +++ b/data/livemathematicianbench_id_split/val/items.json @@ -0,0 +1,128 @@ +[ + { + "id": "202602:8", + "month": "202602", + "no": 8, + "paper_link": "http://arxiv.org/abs/2602.19529v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202512:50", + "month": "202512", + "no": 50, + "paper_link": "http://arxiv.org/abs/2512.15277v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202512:36", + "month": "202512", + "no": 36, + "paper_link": "http://arxiv.org/abs/2512.06696v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202511:1", + "month": "202511", + "no": 1, + "paper_link": "http://arxiv.org/abs/2511.04651v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202601:31", + "month": "202601", + "no": 31, + "paper_link": "http://arxiv.org/abs/2601.10298v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202511:17", + "month": "202511", + "no": 17, + "paper_link": "http://arxiv.org/abs/2511.13215v1", + "source_file": "data/202511/qa_202511_final.json" + }, + { + "id": "202512:37", + "month": "202512", + "no": 37, + "paper_link": "http://arxiv.org/abs/2512.20498v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202601:39", + "month": "202601", + "no": 39, + "paper_link": "http://arxiv.org/abs/2601.06601v2", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202601:25", + "month": "202601", + "no": 25, + "paper_link": "http://arxiv.org/abs/2601.10996v3", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202601:24", + "month": "202601", + "no": 24, + "paper_link": "http://arxiv.org/abs/2601.12250v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202601:45", + "month": "202601", + "no": 45, + "paper_link": "http://arxiv.org/abs/2601.12113v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202601:19", + "month": "202601", + "no": 19, + "paper_link": "http://arxiv.org/abs/2601.00779v1", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202512:10", + "month": "202512", + "no": 10, + "paper_link": "http://arxiv.org/abs/2512.07073v2", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202601:46", + "month": "202601", + "no": 46, + "paper_link": "http://arxiv.org/abs/2601.07793v2", + "source_file": "data/202601/qa_202601_final.json" + }, + { + "id": "202512:15", + "month": "202512", + "no": 15, + "paper_link": "http://arxiv.org/abs/2512.16165v1", + "source_file": "data/202512/qa_202512_final.json" + }, + { + "id": "202602:15", + "month": "202602", + "no": 15, + "paper_link": "http://arxiv.org/abs/2602.05303v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202602:6", + "month": "202602", + "no": 6, + "paper_link": "http://arxiv.org/abs/2602.01571v1", + "source_file": "data/202602/qa_202602_final.json" + }, + { + "id": "202512:46", + "month": "202512", + "no": 46, + "paper_link": "http://arxiv.org/abs/2512.05945v1", + "source_file": "data/202512/qa_202512_final.json" + } +] diff --git a/data/officeqa_id_split/split_manifest.json b/data/officeqa_id_split/split_manifest.json new file mode 100644 index 0000000..1054c19 --- /dev/null +++ b/data/officeqa_id_split/split_manifest.json @@ -0,0 +1,27 @@ +{ + "benchmark": "OfficeQA", + "manifest_type": "id_split", + "source_repo": "databricks/officeqa", + "source_repo_type": "dataset", + "source_url": "https://huggingface.co/datasets/databricks/officeqa", + "source_revision": "8ecbf18d3833daf4750a903d14963e4c4c1d4cd8", + "source_file": "officeqa_full.csv", + "source_split_name": "officeqa_split", + "counts": { + "train": 50, + "val": 24, + "test": 172 + }, + "item_fields": [ + "id", + "uid", + "category", + "source_files", + "source_docs", + "source_split" + ], + "notes": [ + "This is a split manifest, not the full OfficeQA payload.", + "The official OfficeQA CSV is gated on Hugging Face; materialization requires authorized access." + ] +} diff --git a/data/officeqa_id_split/test/items.json b/data/officeqa_id_split/test/items.json new file mode 100644 index 0000000..b40bb15 --- /dev/null +++ b/data/officeqa_id_split/test/items.json @@ -0,0 +1,1378 @@ +[ + { + "id": "UID0003", + "uid": "UID0003", + "category": "hard", + "source_files": "treasury_bulletin_1954_02.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1954-6685?page=14", + "source_split": "test" + }, + { + "id": "UID0004", + "uid": "UID0004", + "category": "hard", + "source_files": "treasury_bulletin_1941_01.txt\r\ntreasury_bulletin_1954_02.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1941-6529?page=15\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1954-6685?page=14", + "source_split": "test" + }, + { + "id": "UID0005", + "uid": "UID0005", + "category": "hard", + "source_files": "treasury_bulletin_1941_01.txt\r\ntreasury_bulletin_1954_02.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1941-6529?page=15\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1954-6685?page=14", + "source_split": "test" + }, + { + "id": "UID0006", + "uid": "UID0006", + "category": "easy", + "source_files": "treasury_bulletin_1998_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1998-7096?page=73&deep=true", + "source_split": "test" + }, + { + "id": "UID0008", + "uid": "UID0008", + "category": "easy", + "source_files": "treasury_bulletin_2012_06.txt\r\ntreasury_bulletin_2022_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2012-7151?page=18\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2022-616226?page=20", + "source_split": "test" + }, + { + "id": "UID0009", + "uid": "UID0009", + "category": "hard", + "source_files": "treasury_bulletin_2011_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2011-7148?page=50", + "source_split": "test" + }, + { + "id": "UID0010", + "uid": "UID0010", + "category": "hard", + "source_files": "treasury_bulletin_2025_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2025-687694?page=76", + "source_split": "test" + }, + { + "id": "UID0011", + "uid": "UID0011", + "category": "easy", + "source_files": "treasury_bulletin_1946_07.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/july-1946-6594?page=72&deep=true", + "source_split": "test" + }, + { + "id": "UID0012", + "uid": "UID0012", + "category": "hard", + "source_files": "treasury_bulletin_1958_10.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1958-6741?page=16", + "source_split": "test" + }, + { + "id": "UID0013", + "uid": "UID0013", + "category": "hard", + "source_files": "treasury_bulletin_1942_07.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/july-1942-6547?page=76", + "source_split": "test" + }, + { + "id": "UID0015", + "uid": "UID0015", + "category": "hard", + "source_files": "treasury_bulletin_1981_11.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1981-7019?page=24", + "source_split": "test" + }, + { + "id": "UID0016", + "uid": "UID0016", + "category": "easy", + "source_files": "treasury_bulletin_1982_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1982-7023?page=24", + "source_split": "test" + }, + { + "id": "UID0020", + "uid": "UID0020", + "category": "easy", + "source_files": "treasury_bulletin_1944_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1944-6570?page=14", + "source_split": "test" + }, + { + "id": "UID0021", + "uid": "UID0021", + "category": "easy", + "source_files": "treasury_bulletin_1982_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1982-7023?page=21&deep=true", + "source_split": "test" + }, + { + "id": "UID0022", + "uid": "UID0022", + "category": "hard", + "source_files": "treasury_bulletin_1999_03.txt\r\ntreasury_bulletin_1994_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1999-7097?page=18\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1994-7076?page=34", + "source_split": "test" + }, + { + "id": "UID0023", + "uid": "UID0023", + "category": "easy", + "source_files": "treasury_bulletin_1939_08.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/august-1939-6510?page=15", + "source_split": "test" + }, + { + "id": "UID0024", + "uid": "UID0024", + "category": "easy", + "source_files": "treasury_bulletin_1990_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1990-7063?page=43", + "source_split": "test" + }, + { + "id": "UID0025", + "uid": "UID0025", + "category": "hard", + "source_files": "treasury_bulletin_1942_10.txt\r\ntreasury_bulletin_1947_08.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1942-6550?page=18&deep=true\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/august-1947-6607?page=24&deep=true", + "source_split": "test" + }, + { + "id": "UID0029", + "uid": "UID0029", + "category": "hard", + "source_files": "treasury_bulletin_1970_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1970-6882?page=89&deep=true", + "source_split": "test" + }, + { + "id": "UID0032", + "uid": "UID0032", + "category": "hard", + "source_files": "treasury_bulletin_1941_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1941-6531?page=48", + "source_split": "test" + }, + { + "id": "UID0035", + "uid": "UID0035", + "category": "hard", + "source_files": "treasury_bulletin_1980_05.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1980-7001?page=41", + "source_split": "test" + }, + { + "id": "UID0036", + "uid": "UID0036", + "category": "hard", + "source_files": "treasury_bulletin_2011_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2011-7148?page=53", + "source_split": "test" + }, + { + "id": "UID0037", + "uid": "UID0037", + "category": "hard", + "source_files": "treasury_bulletin_2007_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2007-7132?page=12&deep=true", + "source_split": "test" + }, + { + "id": "UID0038", + "uid": "UID0038", + "category": "easy", + "source_files": "treasury_bulletin_2004_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2004-7117?page=66", + "source_split": "test" + }, + { + "id": "UID0040", + "uid": "UID0040", + "category": "easy", + "source_files": "treasury_bulletin_1981_04.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1981-7012?page=130", + "source_split": "test" + }, + { + "id": "UID0042", + "uid": "UID0042", + "category": "hard", + "source_files": "treasury_bulletin_2020_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2020-598551?page=29\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2020-598551?page=30", + "source_split": "test" + }, + { + "id": "UID0043", + "uid": "UID0043", + "category": "easy", + "source_files": "treasury_bulletin_2005_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2005-7121?page=123", + "source_split": "test" + }, + { + "id": "UID0045", + "uid": "UID0045", + "category": "easy", + "source_files": "treasury_bulletin_2003_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2003-7114?page=83", + "source_split": "test" + }, + { + "id": "UID0047", + "uid": "UID0047", + "category": "easy", + "source_files": "treasury_bulletin_1982_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1982-7029?page=86", + "source_split": "test" + }, + { + "id": "UID0048", + "uid": "UID0048", + "category": "easy", + "source_files": "treasury_bulletin_1939_02.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1939-6505?page=111", + "source_split": "test" + }, + { + "id": "UID0050", + "uid": "UID0050", + "category": "hard", + "source_files": "treasury_bulletin_1941_05.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1941-6533?page=32", + "source_split": "test" + }, + { + "id": "UID0051", + "uid": "UID0051", + "category": "easy", + "source_files": "treasury_bulletin_1969_07.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/july-1969-6871?page=77", + "source_split": "test" + }, + { + "id": "UID0053", + "uid": "UID0053", + "category": "hard", + "source_files": "treasury_bulletin_2000_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2000-7103?page=24", + "source_split": "test" + }, + { + "id": "UID0054", + "uid": "UID0054", + "category": "easy", + "source_files": "treasury_bulletin_2020_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2020-596188?page=74", + "source_split": "test" + }, + { + "id": "UID0055", + "uid": "UID0055", + "category": "hard", + "source_files": "treasury_bulletin_1960_07.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/july-1960-6763?page=68", + "source_split": "test" + }, + { + "id": "UID0057", + "uid": "UID0057", + "category": "hard", + "source_files": "treasury_bulletin_1969_10.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1969-6874?page=32, https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1970-6877?page=32, https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1971-6889?page=24, https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1972-6901?page=32, https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1973-6913?page=38, https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1974-6924?page=37, https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1975-6937?page=28, https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1976-6949?page=27, https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1977-6961?page=27, https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1978-6973?page=31, https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1979-6985?page=29, https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1980-6997?page=30, https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1981-7009?page=39", + "source_split": "test" + }, + { + "id": "UID0058", + "uid": "UID0058", + "category": "hard", + "source_files": "treasury_bulletin_2003_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2003-7115?page=106&deep=true", + "source_split": "test" + }, + { + "id": "UID0059", + "uid": "UID0059", + "category": "hard", + "source_files": "treasury_bulletin_1953_02.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1953-6673?page=25", + "source_split": "test" + }, + { + "id": "UID0060", + "uid": "UID0060", + "category": "easy", + "source_files": "treasury_bulletin_1953_02.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1953-6673?page=25", + "source_split": "test" + }, + { + "id": "UID0061", + "uid": "UID0061", + "category": "easy", + "source_files": "treasury_bulletin_1949_01.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1949-6624?page=37", + "source_split": "test" + }, + { + "id": "UID0062", + "uid": "UID0062", + "category": "hard", + "source_files": "treasury_bulletin_1948_04.txt\r\ntreasury_bulletin_1952_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1948-6615?page=13\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1952-6671?page=15", + "source_split": "test" + }, + { + "id": "UID0064", + "uid": "UID0064", + "category": "easy", + "source_files": "treasury_bulletin_1941_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1941-6531?page=58&deep=true\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1941-6531?page=59&deep=true", + "source_split": "test" + }, + { + "id": "UID0066", + "uid": "UID0066", + "category": "easy", + "source_files": "treasury_bulletin_2020_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2020-598551?page=29\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2020-598551?page=30&deep=true", + "source_split": "test" + }, + { + "id": "UID0067", + "uid": "UID0067", + "category": "easy", + "source_files": "treasury_bulletin_1940_04.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1940-6524?page=71", + "source_split": "test" + }, + { + "id": "UID0068", + "uid": "UID0068", + "category": "hard", + "source_files": "treasury_bulletin_2016_12.txt\r\ntreasury_bulletin_2017_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2016-535293?page=22&deep=true\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2017-552379?page=22", + "source_split": "test" + }, + { + "id": "UID0069", + "uid": "UID0069", + "category": "hard", + "source_files": "treasury_bulletin_2000_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2000-7102?page=55\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2000-7102?page=56\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2000-7102?page=57", + "source_split": "test" + }, + { + "id": "UID0071", + "uid": "UID0071", + "category": "hard", + "source_files": "treasury_bulletin_2007_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2007-7133?page=53", + "source_split": "test" + }, + { + "id": "UID0074", + "uid": "UID0074", + "category": "easy", + "source_files": "treasury_bulletin_1969_01.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1969-6865?page=30", + "source_split": "test" + }, + { + "id": "UID0075", + "uid": "UID0075", + "category": "easy", + "source_files": "treasury_bulletin_1975_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1975-6939?page=11&deep=true", + "source_split": "test" + }, + { + "id": "UID0076", + "uid": "UID0076", + "category": "easy", + "source_files": "treasury_bulletin_1990_03.txt\r\ntreasury_bulletin_1991_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1990-7060?page=27\r\n\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1991-7064?page=29", + "source_split": "test" + }, + { + "id": "UID0077", + "uid": "UID0077", + "category": "hard", + "source_files": "treasury_bulletin_2011_03.txt\r\ntreasury_bulletin_2012_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2011-7147?page=20\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2011-7147?page=21\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2012-7150?page=21\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2012-7150?page=22", + "source_split": "test" + }, + { + "id": "UID0078", + "uid": "UID0078", + "category": "easy", + "source_files": "treasury_bulletin_2010_12.txt\r\ntreasury_bulletin_2015_12.txt\r\ntreasury_bulletin_2024_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2010-7146?page=26\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2015-519209?page=24\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2024-679984?page=28", + "source_split": "test" + }, + { + "id": "UID0080", + "uid": "UID0080", + "category": "easy", + "source_files": "treasury_bulletin_1955_01.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1955-6696?page=51", + "source_split": "test" + }, + { + "id": "UID0081", + "uid": "UID0081", + "category": "easy", + "source_files": "treasury_bulletin_2023_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2023-628984?page=31", + "source_split": "test" + }, + { + "id": "UID0082", + "uid": "UID0082", + "category": "easy", + "source_files": "treasury_bulletin_2011_12.txt\r\ntreasury_bulletin_2016_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2011-7149?page=57&deep=true\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2016-535293?page=56&deep=true", + "source_split": "test" + }, + { + "id": "UID0084", + "uid": "UID0084", + "category": "hard", + "source_files": "treasury_bulletin_1969_01.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1969-6865?page=30", + "source_split": "test" + }, + { + "id": "UID0088", + "uid": "UID0088", + "category": "easy", + "source_files": "treasury_bulletin_2012_12.txt\r\ntreasury_bulletin_2017_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2012-7142?page=24\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2017-575188?page=22", + "source_split": "test" + }, + { + "id": "UID0089", + "uid": "UID0089", + "category": "easy", + "source_files": "treasury_bulletin_2016_12.txt\r\ntreasury_bulletin_2017_12.txt\r\ntreasury_bulletin_2020_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2016-535293?page=21\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2017-575188?page=21\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2020-598551?page=24", + "source_split": "test" + }, + { + "id": "UID0090", + "uid": "UID0090", + "category": "easy", + "source_files": "treasury_bulletin_1985_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1985-7043?page=24", + "source_split": "test" + }, + { + "id": "UID0093", + "uid": "UID0093", + "category": "hard", + "source_files": "treasury_bulletin_1988_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1988-7055?page=95", + "source_split": "test" + }, + { + "id": "UID0094", + "uid": "UID0094", + "category": "hard", + "source_files": "treasury_bulletin_1939_12.txt\r\ntreasury_bulletin_1941_01.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1939-6513?page=31\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1941-6529?page=26", + "source_split": "test" + }, + { + "id": "UID0095", + "uid": "UID0095", + "category": "easy", + "source_files": "treasury_bulletin_1939_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1939-6513?page=51", + "source_split": "test" + }, + { + "id": "UID0096", + "uid": "UID0096", + "category": "hard", + "source_files": "treasury_bulletin_1940_12.txt\r\ntreasury_bulletin_1941_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1940-6528?page=21\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1941-6540?page=64", + "source_split": "test" + }, + { + "id": "UID0097", + "uid": "UID0097", + "category": "hard", + "source_files": "treasury_bulletin_1989_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1989-7059?page=117&deep=true", + "source_split": "test" + }, + { + "id": "UID0099", + "uid": "UID0099", + "category": "easy", + "source_files": "treasury_bulletin_2020_12.txt\r\ntreasury_bulletin_2024_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2020-598551?page=24\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2024-679984?page=25", + "source_split": "test" + }, + { + "id": "UID0100", + "uid": "UID0100", + "category": "hard", + "source_files": "treasury_bulletin_2011_12.txt\r\ntreasury_bulletin_2014_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2011-7149?page=23 \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2014-487465?page=22", + "source_split": "test" + }, + { + "id": "UID0102", + "uid": "UID0102", + "category": "hard", + "source_files": "treasury_bulletin_2021_03.txt\r\ntreasury_bulletin_2021_06.txt\r\ntreasury_bulletin_2021_09.txt\r\ntreasury_bulletin_2021_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2021-601654?page=16\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2021-603946?page=20\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2021-605026?page=20\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2021-610144?page=22", + "source_split": "test" + }, + { + "id": "UID0103", + "uid": "UID0103", + "category": "hard", + "source_files": "treasury_bulletin_1960_12.txt\r\ntreasury_bulletin_1967_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1960-6768?page=71\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1967-6852?page=83", + "source_split": "test" + }, + { + "id": "UID0104", + "uid": "UID0104", + "category": "easy", + "source_files": "treasury_bulletin_1970_12.txt\r\ntreasury_bulletin_1978_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1970-6888?page=94\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1978-6984?page=91", + "source_split": "test" + }, + { + "id": "UID0105", + "uid": "UID0105", + "category": "easy", + "source_files": "treasury_bulletin_1948_12.txt\r\ntreasury_bulletin_1950_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1948-6623?page=54\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1950-6647?page=56", + "source_split": "test" + }, + { + "id": "UID0106", + "uid": "UID0106", + "category": "easy", + "source_files": "treasury_bulletin_1960_12.txt\r\ntreasury_bulletin_1954_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1960-6768?page=13\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1954-6695?page=13", + "source_split": "test" + }, + { + "id": "UID0107", + "uid": "UID0107", + "category": "easy", + "source_files": "treasury_bulletin_1962_12.txt\r\ntreasury_bulletin_1955_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1962-6792?page=91\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1955-6707?page=57", + "source_split": "test" + }, + { + "id": "UID0108", + "uid": "UID0108", + "category": "hard", + "source_files": "treasury_bulletin_2018_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2018-581283?page=18", + "source_split": "test" + }, + { + "id": "UID0111", + "uid": "UID0111", + "category": "hard", + "source_files": "treasury_bulletin_2015_09.txt\r\ntreasury_bulletin_2020_09.txt\r\ntreasury_bulletin_2024_09.txt\r\ntreasury_bulletin_2025_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2015-519208?page=18\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2020-596188?page=21\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2024-677156?page=21\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2025-701334?page=22", + "source_split": "test" + }, + { + "id": "UID0112", + "uid": "UID0112", + "category": "hard", + "source_files": "treasury_bulletin_1996_09.txt\r\ntreasury_bulletin_2001_09.txt\r\ntreasury_bulletin_2006_09.txt\r\ntreasury_bulletin_2011_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1996-7087?page=12\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2001-7107?page=16&deep=true\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2006-7127?page=19\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2011-7148?page=20", + "source_split": "test" + }, + { + "id": "UID0113", + "uid": "UID0113", + "category": "hard", + "source_files": "treasury_bulletin_1982_11.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1982-7031?page=66", + "source_split": "test" + }, + { + "id": "UID0114", + "uid": "UID0114", + "category": "hard", + "source_files": "treasury_bulletin_2003_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2003-7109?page=58", + "source_split": "test" + }, + { + "id": "UID0116", + "uid": "UID0116", + "category": "easy", + "source_files": "treasury_bulletin_1969_12.txt\r\ntreasury_bulletin_1974_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1969-6876?page=182\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1974-6936?page=126", + "source_split": "test" + }, + { + "id": "UID0117", + "uid": "UID0117", + "category": "hard", + "source_files": "treasury_bulletin_1972_03.txt\r\ntreasury_bulletin_1973_03.txt\r\ntreasury_bulletin_1974_03.txt\r\ntreasury_bulletin_1975_03.txt\r\ntreasury_bulletin_1976_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1972-6903?page=45\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1973-6915?page=49\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1974-6926?page=38\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1975-6939?page=36\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1976-6951?page=38", + "source_split": "test" + }, + { + "id": "UID0118", + "uid": "UID0118", + "category": "hard", + "source_files": "treasury_bulletin_1968_07.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/july-1968-6859?page=83", + "source_split": "test" + }, + { + "id": "UID0119", + "uid": "UID0119", + "category": "easy", + "source_files": "treasury_bulletin_1960_08.txt\r\ntreasury_bulletin_1961_08.txt\r\ntreasury_bulletin_1962_08.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/august-1960-6764?page=77\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/august-1961-6776?page=84\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/august-1962-6788?page=98", + "source_split": "test" + }, + { + "id": "UID0120", + "uid": "UID0120", + "category": "hard", + "source_files": "treasury_bulletin_1970_05.txt\r\ntreasury_bulletin_1970_06.txt\r\ntreasury_bulletin_1970_07.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1970-6758?page=79\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1970-6882?page=76\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/july-1970-6883?page=73", + "source_split": "test" + }, + { + "id": "UID0121", + "uid": "UID0121", + "category": "hard", + "source_files": "treasury_bulletin_1980_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1980-6999?page=81", + "source_split": "test" + }, + { + "id": "UID0124", + "uid": "UID0124", + "category": "hard", + "source_files": "treasury_bulletin_2000_06.txt\r\ntreasury_bulletin_2005_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2000-7102?page=48\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2000-7102?page=49\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2005-7122?page=50\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2005-7122?page=51", + "source_split": "test" + }, + { + "id": "UID0125", + "uid": "UID0125", + "category": "easy", + "source_files": "treasury_bulletin_2011_03.txt\r\ntreasury_bulletin_2012_03.txt\r\ntreasury_bulletin_2013_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2011-7147?page=106\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2012-7150?page=105\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2013-6928?page=107", + "source_split": "test" + }, + { + "id": "UID0126", + "uid": "UID0126", + "category": "easy", + "source_files": "treasury_bulletin_1970_01.txt\r\ntreasury_bulletin_1970_02.txt\r\ntreasury_bulletin_1970_03.txt\r\ntreasury_bulletin_1970_04.txt\r\ntreasury_bulletin_1970_05.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1970-6877?page=91\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1970-6878?page=98\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1970-6880?page=93\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1970-6881?page=96\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1970-6758?page=90", + "source_split": "test" + }, + { + "id": "UID0127", + "uid": "UID0127", + "category": "easy", + "source_files": "treasury_bulletin_1991_03.txt\r\ntreasury_bulletin_1992_03.txt\r\ntreasury_bulletin_1993_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1991-7064?page=122\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1992-7068?page=145\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1993-7072?page=132", + "source_split": "test" + }, + { + "id": "UID0128", + "uid": "UID0128", + "category": "easy", + "source_files": "treasury_bulletin_1941_01.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1941-6529?page=17", + "source_split": "test" + }, + { + "id": "UID0129", + "uid": "UID0129", + "category": "easy", + "source_files": "treasury_bulletin_1994_03.txt\r\ntreasury_bulletin_1999_03.txt\r\ntreasury_bulletin_2004_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1994-7076?page=78\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1999-7097?page=55\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2004-7117?page=53", + "source_split": "test" + }, + { + "id": "UID0130", + "uid": "UID0130", + "category": "hard", + "source_files": "treasury_bulletin_1980_04.txt\r\ntreasury_bulletin_1981_04.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1980-7000?page=85\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1981-7012?page=81", + "source_split": "test" + }, + { + "id": "UID0131", + "uid": "UID0131", + "category": "easy", + "source_files": "treasury_bulletin_2003_12.txt\r\ntreasury_bulletin_2008_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2003-7116?page=32\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2008-7137?page=38", + "source_split": "test" + }, + { + "id": "UID0132", + "uid": "UID0132", + "category": "easy", + "source_files": "treasury_bulletin_1994_03.txt\r\ntreasury_bulletin_1995_03.txt\r\ntreasury_bulletin_1996_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1994-7076?page=28\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1995-7081?page=29\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1996-7085?page=11", + "source_split": "test" + }, + { + "id": "UID0134", + "uid": "UID0134", + "category": "hard", + "source_files": "treasury_bulletin_1963_02.txt\r\ntreasury_bulletin_1964_02.txt\r\ntreasury_bulletin_1965_02.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1963-6794?page=45\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1964-6806?page=38\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1965-6818?page=42", + "source_split": "test" + }, + { + "id": "UID0135", + "uid": "UID0135", + "category": "hard", + "source_files": "treasury_bulletin_1996_06.txt\r\ntreasury_bulletin_1997_06.txt\r\ntreasury_bulletin_1998_06.txt\r\ntreasury_bulletin_2000_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1996-7086?page=70\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1997-7090?page=68\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1998-7094?page=70\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2000-7102?page=65", + "source_split": "test" + }, + { + "id": "UID0136", + "uid": "UID0136", + "category": "hard", + "source_files": "treasury_bulletin_1953_10.txt\r\ntreasury_bulletin_1954_10.txt\r\ntreasury_bulletin_1955_10.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1953-6681?page=8\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1954-6693?page=11\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1955-6705?page=9", + "source_split": "test" + }, + { + "id": "UID0137", + "uid": "UID0137", + "category": "easy", + "source_files": "treasury_bulletin_1939_05.txt\r\ntreasury_bulletin_1944_05.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1939-6507?page=65\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1944-6569?page=91", + "source_split": "test" + }, + { + "id": "UID0138", + "uid": "UID0138", + "category": "easy", + "source_files": "treasury_bulletin_1943_04.txt\r\ntreasury_bulletin_1944_04.txt\r\ntreasury_bulletin_1945_04.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1943-6556?page=51\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1944-6568?page=55\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1945-6578?page=63", + "source_split": "test" + }, + { + "id": "UID0139", + "uid": "UID0139", + "category": "easy", + "source_files": "treasury_bulletin_1970_01.txt\r\ntreasury_bulletin_1970_02.txt\r\ntreasury_bulletin_1970_03.txt\r\ntreasury_bulletin_1970_04.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1970-6877?page=28\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1970-6878?page=28\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1970-6880?page=30\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1970-6881?page=31", + "source_split": "test" + }, + { + "id": "UID0140", + "uid": "UID0140", + "category": "hard", + "source_files": "treasury_bulletin_1994_06.txt\r\ntreasury_bulletin_1999_06.txt\r\ntreasury_bulletin_2004_06.txt\r\ntreasury_bulletin_2009_06.txt\r\ntreasury_bulletin_2014_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1994-7077?page=28\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1999-7098?page=15\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2004-7118?page=18\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2009-7139?page=20\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2014-7157?page=19", + "source_split": "test" + }, + { + "id": "UID0143", + "uid": "UID0143", + "category": "easy", + "source_files": "treasury_bulletin_1943_01.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1943-6553?page=33", + "source_split": "test" + }, + { + "id": "UID0146", + "uid": "UID0146", + "category": "easy", + "source_files": "treasury_bulletin_1960_01.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1960-6756?page=60", + "source_split": "test" + }, + { + "id": "UID0147", + "uid": "UID0147", + "category": "hard", + "source_files": "treasury_bulletin_1948_03.txt\r\ntreasury_bulletin_1949_03.txt\r\ntreasury_bulletin_1950_03.txt\r\ntreasury_bulletin_1951_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1948-6614?page=28\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1949-6626?page=29\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1950-6638?page=31\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1951-6650?page=32", + "source_split": "test" + }, + { + "id": "UID0148", + "uid": "UID0148", + "category": "hard", + "source_files": "treasury_bulletin_1972_05.txt\r\ntreasury_bulletin_1973_05.txt\r\ntreasury_bulletin_1974_05.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1972-6906?page=89\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1973-6898?page=95\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1974-6929?page=83", + "source_split": "test" + }, + { + "id": "UID0149", + "uid": "UID0149", + "category": "hard", + "source_files": "treasury_bulletin_1962_03.txt\r\ntreasury_bulletin_1963_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1962-6783?page=74\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1963-6795?page=81", + "source_split": "test" + }, + { + "id": "UID0152", + "uid": "UID0152", + "category": "easy", + "source_files": "treasury_bulletin_1939_01.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1939-6518?page=15", + "source_split": "test" + }, + { + "id": "UID0153", + "uid": "UID0153", + "category": "easy", + "source_files": "treasury_bulletin_1996_03.txt\r\ntreasury_bulletin_1997_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1996-7085?page=113\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1997-7089?page=106", + "source_split": "test" + }, + { + "id": "UID0155", + "uid": "UID0155", + "category": "hard", + "source_files": "treasury_bulletin_2010_09.txt\r\ntreasury_bulletin_2011_09.txt\r\ntreasury_bulletin_2012_09.txt\r\ntreasury_bulletin_2013_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2010-7145?page=52\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2011-7148?page=52\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2012-7152?page=52\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2013-7154?page=51", + "source_split": "test" + }, + { + "id": "UID0156", + "uid": "UID0156", + "category": "easy", + "source_files": "treasury_bulletin_1947_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1947-6602?page=34", + "source_split": "test" + }, + { + "id": "UID0157", + "uid": "UID0157", + "category": "easy", + "source_files": "treasury_bulletin_1961_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1961-6774?page=75", + "source_split": "test" + }, + { + "id": "UID0158", + "uid": "UID0158", + "category": "easy", + "source_files": "treasury_bulletin_1970_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1970-6888?page=44", + "source_split": "test" + }, + { + "id": "UID0160", + "uid": "UID0160", + "category": "easy", + "source_files": "treasury_bulletin_1980_03.txt\r\ntreasury_bulletin_1980_04.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1980-6999?page=9\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1980-7000?page=9", + "source_split": "test" + }, + { + "id": "UID0164", + "uid": "UID0164", + "category": "easy", + "source_files": "treasury_bulletin_1948_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1948-6614?page=37", + "source_split": "test" + }, + { + "id": "UID0167", + "uid": "UID0167", + "category": "easy", + "source_files": "treasury_bulletin_1950_05.txt\r\ntreasury_bulletin_1955_05.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1950-6640?page=30\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1955-6700?page=27", + "source_split": "test" + }, + { + "id": "UID0168", + "uid": "UID0168", + "category": "hard", + "source_files": "treasury_bulletin_1939_08.txt\r\ntreasury_bulletin_1939_09.txt\r\ntreasury_bulletin_1939_10.txt\r\ntreasury_bulletin_1939_11.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/august-1939-6510?page=79\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1939-6511?page=45\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1939-6520?page=45\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1939-6512?page=49", + "source_split": "test" + }, + { + "id": "UID0171", + "uid": "UID0171", + "category": "easy", + "source_files": "treasury_bulletin_1963_12.txt\r\ntreasury_bulletin_1964_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1963-6804?page=39\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1964-6816?page=34", + "source_split": "test" + }, + { + "id": "UID0172", + "uid": "UID0172", + "category": "hard", + "source_files": "treasury_bulletin_2000_12.txt\r\ntreasury_bulletin_2001_12.txt\r\ntreasury_bulletin_2002_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2000-7104?page=68\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2001-7108?page=73\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2002-7113?page=73", + "source_split": "test" + }, + { + "id": "UID0173", + "uid": "UID0173", + "category": "hard", + "source_files": "treasury_bulletin_1980_08.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/august-1980-7004?page=95", + "source_split": "test" + }, + { + "id": "UID0174", + "uid": "UID0174", + "category": "hard", + "source_files": "treasury_bulletin_1960_04.txt\r\ntreasury_bulletin_1960_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1960-6760?page=69\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1960-6762?page=71", + "source_split": "test" + }, + { + "id": "UID0175", + "uid": "UID0175", + "category": "hard", + "source_files": "treasury_bulletin_1947_02.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1947-6601?page=87", + "source_split": "test" + }, + { + "id": "UID0176", + "uid": "UID0176", + "category": "easy", + "source_files": "treasury_bulletin_1992_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1992-7068?page=42", + "source_split": "test" + }, + { + "id": "UID0177", + "uid": "UID0177", + "category": "hard", + "source_files": "treasury_bulletin_1950_04.txt\r\ntreasury_bulletin_1951_04.txt\r\ntreasury_bulletin_1952_04.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1950-6639?page=30\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1951-6652?page=33\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1952-6663?page=29", + "source_split": "test" + }, + { + "id": "UID0178", + "uid": "UID0178", + "category": "easy", + "source_files": "treasury_bulletin_2014_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2014-487465?page=19", + "source_split": "test" + }, + { + "id": "UID0179", + "uid": "UID0179", + "category": "hard", + "source_files": "treasury_bulletin_1977_03.txt\r\ntreasury_bulletin_1977_04.txt\r\ntreasury_bulletin_1977_05.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1977-6963?page=135\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1977-6964?page=137\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1977-6965?page=130", + "source_split": "test" + }, + { + "id": "UID0180", + "uid": "UID0180", + "category": "hard", + "source_files": "treasury_bulletin_2010_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2010-7145?page=47", + "source_split": "test" + }, + { + "id": "UID0181", + "uid": "UID0181", + "category": "easy", + "source_files": "treasury_bulletin_1960_02.txt\r\ntreasury_bulletin_1961_02.txt\r\ntreasury_bulletin_1962_02.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1960-6757?page=37\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1961-6770?page=36\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1962-6782?page=39", + "source_split": "test" + }, + { + "id": "UID0182", + "uid": "UID0182", + "category": "hard", + "source_files": "treasury_bulletin_2011_03.txt\r\ntreasury_bulletin_2012_03.txt\r\ntreasury_bulletin_2013_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2011-7147?page=59\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2012-7150?page=58\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2013-6928?page=60", + "source_split": "test" + }, + { + "id": "UID0183", + "uid": "UID0183", + "category": "hard", + "source_files": "treasury_bulletin_1964_03.txt\r\ntreasury_bulletin_1965_03.txt\r\ntreasury_bulletin_1966_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1964-6807?page=80\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1965-6819?page=79\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1966-6831?page=86", + "source_split": "test" + }, + { + "id": "UID0184", + "uid": "UID0184", + "category": "hard", + "source_files": "treasury_bulletin_1948_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1948-6614?page=45", + "source_split": "test" + }, + { + "id": "UID0185", + "uid": "UID0185", + "category": "easy", + "source_files": "treasury_bulletin_1990_09.txt\r\ntreasury_bulletin_1991_09.txt\r\ntreasury_bulletin_1992_09.txt\r\ntreasury_bulletin_1993_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1990-7062?page=72\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1991-7066?page=76\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1992-7070?page=69\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1993-7074?page=73", + "source_split": "test" + }, + { + "id": "UID0186", + "uid": "UID0186", + "category": "easy", + "source_files": "treasury_bulletin_1974_11.txt\r\ntreasury_bulletin_1975_11.txt\r\ntreasury_bulletin_1976_11.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1974-6935?page=80\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1975-6947?page=75\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1976-6960?page=83", + "source_split": "test" + }, + { + "id": "UID0187", + "uid": "UID0187", + "category": "hard", + "source_files": "treasury_bulletin_1940_10.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1940-6516?page=74", + "source_split": "test" + }, + { + "id": "UID0188", + "uid": "UID0188", + "category": "hard", + "source_files": "treasury_bulletin_1939_01.txt\r\ntreasury_bulletin_1949_01.txt\r\ntreasury_bulletin_1959_01.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1939-6518?page=67\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1949-6624?page=57\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1959-6744?page=71", + "source_split": "test" + }, + { + "id": "UID0191", + "uid": "UID0191", + "category": "easy", + "source_files": "treasury_bulletin_1980_11.txt\r\ntreasury_bulletin_1981_11.txt\r\ntreasury_bulletin_1982_11.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1980-7007?page=148\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1981-7019?page=140\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1982-7031?page=112", + "source_split": "test" + }, + { + "id": "UID0192", + "uid": "UID0192", + "category": "easy", + "source_files": "treasury_bulletin_1990_06.txt\r\ntreasury_bulletin_1991_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1990-7061?page=42\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1991-7065?page=46", + "source_split": "test" + }, + { + "id": "UID0193", + "uid": "UID0193", + "category": "hard", + "source_files": "treasury_bulletin_1939_03.txt\r\ntreasury_bulletin_1940_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1939-6519?page=115\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1940-6523?page=71", + "source_split": "test" + }, + { + "id": "UID0194", + "uid": "UID0194", + "category": "hard", + "source_files": "treasury_bulletin_2003_09.txt\r\ntreasury_bulletin_2013_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2003-7115?page=69\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2013-7154?page=62", + "source_split": "test" + }, + { + "id": "UID0196", + "uid": "UID0196", + "category": "hard", + "source_files": "treasury_bulletin_1980_05.txt\r\ntreasury_bulletin_1980_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1980-7001?page=41\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1980-7002?page=35", + "source_split": "test" + }, + { + "id": "UID0197", + "uid": "UID0197", + "category": "easy", + "source_files": "treasury_bulletin_1970_11.txt\r\ntreasury_bulletin_1971_11.txt\r\ntreasury_bulletin_1972_11.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1970-6887?page=74\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1971-6899?page=70\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1972-6911?page=74", + "source_split": "test" + }, + { + "id": "UID0198", + "uid": "UID0198", + "category": "easy", + "source_files": "treasury_bulletin_1978_05.txt\r\ntreasury_bulletin_1979_05.txt\r\ntreasury_bulletin_1980_05.txt\r\ntreasury_bulletin_1981_05.txt\r\ntreasury_bulletin_1982_05.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1978-6977?page=91\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1979-6989?page=83\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1980-7001?page=99\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1981-7013?page=81\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1982-7025?page=76", + "source_split": "test" + }, + { + "id": "UID0199", + "uid": "UID0199", + "category": "easy", + "source_files": "treasury_bulletin_1939_01.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1939-6518?page=49", + "source_split": "test" + }, + { + "id": "UID0200", + "uid": "UID0200", + "category": "easy", + "source_files": "treasury_bulletin_1939_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1939-6513?page=54", + "source_split": "test" + }, + { + "id": "UID0201", + "uid": "UID0201", + "category": "hard", + "source_files": "treasury_bulletin_1975_11.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1975-6947?page=147", + "source_split": "test" + }, + { + "id": "UID0203", + "uid": "UID0203", + "category": "hard", + "source_files": "treasury_bulletin_1960_04.txt\r\ntreasury_bulletin_1961_04.txt\r\ntreasury_bulletin_1962_04.txt\r\ntreasury_bulletin_1963_04.txt\r\ntreasury_bulletin_1964_04.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1960-6760?page=36\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1961-6772?page=40\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1962-6784?page=41\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1963-6796?page=45\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1964-6808?page=40", + "source_split": "test" + }, + { + "id": "UID0204", + "uid": "UID0204", + "category": "hard", + "source_files": "treasury_bulletin_2010_09.txt\r\ntreasury_bulletin_2011_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2010-7145?page=13\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2011-7148?page=13", + "source_split": "test" + }, + { + "id": "UID0205", + "uid": "UID0205", + "category": "hard", + "source_files": "treasury_bulletin_1960_10.txt\r\ntreasury_bulletin_1965_10.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1960-6766?page=15\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1965-6826?page=15", + "source_split": "test" + }, + { + "id": "UID0206", + "uid": "UID0206", + "category": "easy", + "source_files": "treasury_bulletin_2010_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2010-7143?page=31", + "source_split": "test" + }, + { + "id": "UID0207", + "uid": "UID0207", + "category": "hard", + "source_files": "treasury_bulletin_1980_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1980-7002?page=16", + "source_split": "test" + }, + { + "id": "UID0208", + "uid": "UID0208", + "category": "easy", + "source_files": "treasury_bulletin_1973_07.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/july-1973-6918?page=88", + "source_split": "test" + }, + { + "id": "UID0209", + "uid": "UID0209", + "category": "easy", + "source_files": "treasury_bulletin_1970_08.txt\r\ntreasury_bulletin_1970_09.txt\r\ntreasury_bulletin_1970_10.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/august-1970-6884?page=93\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1970-6885?page=86\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1970-6886?page=83", + "source_split": "test" + }, + { + "id": "UID0210", + "uid": "UID0210", + "category": "easy", + "source_files": "treasury_bulletin_1994_06.txt\r\ntreasury_bulletin_1995_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1994-7077?page=69\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1995-7082?page=63", + "source_split": "test" + }, + { + "id": "UID0211", + "uid": "UID0211", + "category": "hard", + "source_files": "treasury_bulletin_1956_08.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/august-1956-6715?page=43", + "source_split": "test" + }, + { + "id": "UID0214", + "uid": "UID0214", + "category": "hard", + "source_files": "treasury_bulletin_1970_01.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1970-6877?page=31", + "source_split": "test" + }, + { + "id": "UID0215", + "uid": "UID0215", + "category": "hard", + "source_files": "treasury_bulletin_1988_03.txt\r\ntreasury_bulletin_1989_03.txt\r\ntreasury_bulletin_1990_03.txt\r\ntreasury_bulletin_1991_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1988-7052?page=52\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1988-7052?page=53\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1988-7052?page=54\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1989-7056?page=53\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1989-7056?page=54\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1989-7056?page=55\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1990-7060?page=51\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1990-7060?page=52\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1990-7060?page=53\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1991-7064?page=53\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1991-7064?page=54\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1991-7064?page=55", + "source_split": "test" + }, + { + "id": "UID0216", + "uid": "UID0216", + "category": "hard", + "source_files": "treasury_bulletin_1941_10.txt\r\ntreasury_bulletin_1942_10.txt\r\ntreasury_bulletin_1943_10.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1941-6538?page=12\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1942-6550?page=12\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1943-6562?page=22", + "source_split": "test" + }, + { + "id": "UID0218", + "uid": "UID0218", + "category": "hard", + "source_files": "treasury_bulletin_1962_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1962-6783?page=67", + "source_split": "test" + }, + { + "id": "UID0219", + "uid": "UID0219", + "category": "hard", + "source_files": "treasury_bulletin_2013_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2013-7153?page=47", + "source_split": "test" + }, + { + "id": "UID0221", + "uid": "UID0221", + "category": "hard", + "source_files": "treasury_bulletin_1950_01.txt\r\ntreasury_bulletin_1950_02.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1950-6636?page=32\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1950-6637?page=30", + "source_split": "test" + }, + { + "id": "UID0223", + "uid": "UID0223", + "category": "hard", + "source_files": "treasury_bulletin_1991_09.txt\r\ntreasury_bulletin_1996_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1991-7066?page=99\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1996-7087?page=69", + "source_split": "test" + }, + { + "id": "UID0224", + "uid": "UID0224", + "category": "hard", + "source_files": "treasury_bulletin_1943_01.txt\r\ntreasury_bulletin_1944_01.txt\r\ntreasury_bulletin_1945_01.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1943-6553?page=74\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1944-6565?page=68\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1945-6575?page=83", + "source_split": "test" + }, + { + "id": "UID0225", + "uid": "UID0225", + "category": "hard", + "source_files": "treasury_bulletin_1982_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1982-7023?page=74", + "source_split": "test" + }, + { + "id": "UID0226", + "uid": "UID0226", + "category": "hard", + "source_files": "treasury_bulletin_1953_03.txt\r\ntreasury_bulletin_1954_03.txt\r\ntreasury_bulletin_1955_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1953-6674?page=31\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1954-6686?page=25\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1955-6698?page=27", + "source_split": "test" + }, + { + "id": "UID0227", + "uid": "UID0227", + "category": "hard", + "source_files": "treasury_bulletin_1982_11.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1982-7031?page=66", + "source_split": "test" + }, + { + "id": "UID0230", + "uid": "UID0230", + "category": "hard", + "source_files": "treasury_bulletin_1960_10.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1960-6766?page=14", + "source_split": "test" + }, + { + "id": "UID0231", + "uid": "UID0231", + "category": "easy", + "source_files": "treasury_bulletin_1961_12.txt\r\ntreasury_bulletin_1962_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1961-6780?page=35\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1962-6792?page=44", + "source_split": "test" + }, + { + "id": "UID0232", + "uid": "UID0232", + "category": "easy", + "source_files": "treasury_bulletin_1949_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1949-6626?page=18", + "source_split": "test" + }, + { + "id": "UID0236", + "uid": "UID0236", + "category": "easy", + "source_files": "treasury_bulletin_1982_04.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1982-7024?page=77", + "source_split": "test" + }, + { + "id": "UID0237", + "uid": "UID0237", + "category": "hard", + "source_files": "treasury_bulletin_2007_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2007-7132?page=47", + "source_split": "test" + }, + { + "id": "UID0242", + "uid": "UID0242", + "category": "easy", + "source_files": "treasury_bulletin_2010_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2010-7144?page=57\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2010-7144?page=59\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2010-7144?page=61\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2010-7144?page=62\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2010-7144?page=64\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2010-7144?page=86\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2010-7144?page=87\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2010-7144?page=88\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2010-7144?page=89", + "source_split": "test" + }, + { + "id": "UID0243", + "uid": "UID0243", + "category": "hard", + "source_files": "treasury_bulletin_1970_01.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1970-6877?page=32", + "source_split": "test" + }, + { + "id": "UID0244", + "uid": "UID0244", + "category": "hard", + "source_files": "treasury_bulletin_1960_01.txt\r\ntreasury_bulletin_1960_02.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1960-6756?page=20\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1960-6757?page=22", + "source_split": "test" + }, + { + "id": "UID0245", + "uid": "UID0245", + "category": "hard", + "source_files": "treasury_bulletin_1982_08.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/august-1982-7028?page=82", + "source_split": "test" + }, + { + "id": "UID0246", + "uid": "UID0246", + "category": "hard", + "source_files": "treasury_bulletin_1970_03.txt\r\ntreasury_bulletin_1975_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1970-6880?page=88\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1975-6939?page=71", + "source_split": "test" + } +] diff --git a/data/officeqa_id_split/train/items.json b/data/officeqa_id_split/train/items.json new file mode 100644 index 0000000..ea8265a --- /dev/null +++ b/data/officeqa_id_split/train/items.json @@ -0,0 +1,402 @@ +[ + { + "id": "UID0002", + "uid": "UID0002", + "category": "easy", + "source_files": "treasury_bulletin_1944_01.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1944-6565?page=18", + "source_split": "train" + }, + { + "id": "UID0007", + "uid": "UID0007", + "category": "hard", + "source_files": "treasury_bulletin_1950_02.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1950-6637?page=15", + "source_split": "train" + }, + { + "id": "UID0014", + "uid": "UID0014", + "category": "easy", + "source_files": "treasury_bulletin_1942_07.txt\r\ntreasury_bulletin_2001_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/july-1942-6547?page=76\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2001-7108?page=17&deep=true", + "source_split": "train" + }, + { + "id": "UID0017", + "uid": "UID0017", + "category": "hard", + "source_files": "treasury_bulletin_1982_08.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/august-1982-7028?page=13", + "source_split": "train" + }, + { + "id": "UID0018", + "uid": "UID0018", + "category": "hard", + "source_files": "treasury_bulletin_1985_03.txt\r\ntreasury_bulletin_1986_03.txt\r\ntreasury_bulletin_1987_03.txt\r\ntreasury_bulletin_1988_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1985-7040?page=22\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1986-7045?page=26\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1987-7049?page=24\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1988-7052?page=36", + "source_split": "train" + }, + { + "id": "UID0019", + "uid": "UID0019", + "category": "hard", + "source_files": "treasury_bulletin_2016_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2016-533966?page=54\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2016-533966?page=58", + "source_split": "train" + }, + { + "id": "UID0026", + "uid": "UID0026", + "category": "easy", + "source_files": "treasury_bulletin_1963_01.txt\r\ntreasury_bulletin_1962_01.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1963-6793?page=88\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1962-6781?page=82&deep=true", + "source_split": "train" + }, + { + "id": "UID0028", + "uid": "UID0028", + "category": "hard", + "source_files": "treasury_bulletin_1970_06.txt\r\ntreasury_bulletin_1964_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1970-6882?page=89&deep=true\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1964-6816?page=25&deep=true", + "source_split": "train" + }, + { + "id": "UID0030", + "uid": "UID0030", + "category": "hard", + "source_files": "treasury_bulletin_1990_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1990-7062?page=19&deep=true", + "source_split": "train" + }, + { + "id": "UID0031", + "uid": "UID0031", + "category": "hard", + "source_files": "treasury_bulletin_1992_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1992-7068?page=158&deep=true", + "source_split": "train" + }, + { + "id": "UID0033", + "uid": "UID0033", + "category": "easy", + "source_files": "treasury_bulletin_1977_04.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1977-6964?page=9", + "source_split": "train" + }, + { + "id": "UID0034", + "uid": "UID0034", + "category": "easy", + "source_files": "treasury_bulletin_1992_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1992-7069?page=32", + "source_split": "train" + }, + { + "id": "UID0044", + "uid": "UID0044", + "category": "hard", + "source_files": "treasury_bulletin_1939_04.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1939-6506?page=61", + "source_split": "train" + }, + { + "id": "UID0046", + "uid": "UID0046", + "category": "easy", + "source_files": "treasury_bulletin_1988_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1988-7054?page=37", + "source_split": "train" + }, + { + "id": "UID0049", + "uid": "UID0049", + "category": "hard", + "source_files": "treasury_bulletin_1942_02.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1942-6542?page=19&deep=true", + "source_split": "train" + }, + { + "id": "UID0056", + "uid": "UID0056", + "category": "hard", + "source_files": "treasury_bulletin_1991_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1991-7066?page=30&deep=true", + "source_split": "train" + }, + { + "id": "UID0063", + "uid": "UID0063", + "category": "easy", + "source_files": "treasury_bulletin_1990_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1990-7061?page=127", + "source_split": "train" + }, + { + "id": "UID0065", + "uid": "UID0065", + "category": "hard", + "source_files": "treasury_bulletin_1998_06.txt\r\ntreasury_bulletin_1995_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1998-7094?page=7\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1995-7084?page=16", + "source_split": "train" + }, + { + "id": "UID0073", + "uid": "UID0073", + "category": "hard", + "source_files": "treasury_bulletin_1982_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1982-7023?page=24", + "source_split": "train" + }, + { + "id": "UID0079", + "uid": "UID0079", + "category": "easy", + "source_files": "treasury_bulletin_2011_12.txt\r\ntreasury_bulletin_2013_12.txt\r\ntreasury_bulletin_2015_12.txt\r\ntreasury_bulletin_2017_12.txt\r\ntreasury_bulletin_2019_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2011-7149?page=25\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2013-7155?page=24\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2015-519209?page=23\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2017-575188?page=23\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2019-584842?page=22", + "source_split": "train" + }, + { + "id": "UID0083", + "uid": "UID0083", + "category": "hard", + "source_files": "treasury_bulletin_1981_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1981-7020?page=24", + "source_split": "train" + }, + { + "id": "UID0085", + "uid": "UID0085", + "category": "hard", + "source_files": "treasury_bulletin_2019_12.txt\r\ntreasury_bulletin_2018_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2019-584842?page=23\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2018-581283?page=22", + "source_split": "train" + }, + { + "id": "UID0087", + "uid": "UID0087", + "category": "easy", + "source_files": "treasury_bulletin_2013_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2013-7155?page=17", + "source_split": "train" + }, + { + "id": "UID0092", + "uid": "UID0092", + "category": "easy", + "source_files": "treasury_bulletin_1987_12.txt\r\ntreasury_bulletin_1992_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1987-7051?page=69\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1992-7071?page=84", + "source_split": "train" + }, + { + "id": "UID0098", + "uid": "UID0098", + "category": "easy", + "source_files": "treasury_bulletin_2020_12.txt\r\ntreasury_bulletin_2024_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2020-598551?page=21\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2024-679984?page=22", + "source_split": "train" + }, + { + "id": "UID0101", + "uid": "UID0101", + "category": "hard", + "source_files": "treasury_bulletin_2011_12.txt\r\ntreasury_bulletin_2019_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2011-7149?page=25\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2019-584842?page=22", + "source_split": "train" + }, + { + "id": "UID0110", + "uid": "UID0110", + "category": "hard", + "source_files": "treasury_bulletin_2020_03.txt\r\ntreasury_bulletin_2016_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2020-587316?page=10\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2016-527290?page=9", + "source_split": "train" + }, + { + "id": "UID0115", + "uid": "UID0115", + "category": "easy", + "source_files": "treasury_bulletin_1980_02.txt\r\ntreasury_bulletin_1981_02.txt\r\ntreasury_bulletin_1982_02.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1980-6998?page=27\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1981-7010?page=38\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1982-7022?page=31", + "source_split": "train" + }, + { + "id": "UID0122", + "uid": "UID0122", + "category": "hard", + "source_files": "treasury_bulletin_2001_03.txt\r\ntreasury_bulletin_2002_03.txt\r\ntreasury_bulletin_2003_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2001-7105?page=112\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2002-7110?page=115\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2003-7109?page=113", + "source_split": "train" + }, + { + "id": "UID0123", + "uid": "UID0123", + "category": "hard", + "source_files": "treasury_bulletin_1941_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1941-6540?page=91", + "source_split": "train" + }, + { + "id": "UID0133", + "uid": "UID0133", + "category": "hard", + "source_files": "treasury_bulletin_2004_09.txt\r\ntreasury_bulletin_2013_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2004-7119?page=63\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2013-7155?page=68", + "source_split": "train" + }, + { + "id": "UID0141", + "uid": "UID0141", + "category": "easy", + "source_files": "treasury_bulletin_1962_04.txt\r\ntreasury_bulletin_1963_04.txt\r\ntreasury_bulletin_1964_04.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1962-6784?page=75\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1963-6796?page=79\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1964-6808?page=82", + "source_split": "train" + }, + { + "id": "UID0144", + "uid": "UID0144", + "category": "easy", + "source_files": "treasury_bulletin_1980_11.txt\r\ntreasury_bulletin_1981_11.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1980-7007?page=76\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1981-7019?page=67", + "source_split": "train" + }, + { + "id": "UID0145", + "uid": "UID0145", + "category": "easy", + "source_files": "treasury_bulletin_1943_01.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1943-6553?page=41", + "source_split": "train" + }, + { + "id": "UID0150", + "uid": "UID0150", + "category": "hard", + "source_files": "treasury_bulletin_1972_04.txt\r\ntreasury_bulletin_1973_04.txt\r\ntreasury_bulletin_1974_04.txt\r\ntreasury_bulletin_1975_04.txt\r\ntreasury_bulletin_1976_04.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1972-6905?page=89\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1973-6916?page=91\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1974-6927?page=88\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1975-6940?page=88\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1976-6952?page=104", + "source_split": "train" + }, + { + "id": "UID0151", + "uid": "UID0151", + "category": "easy", + "source_files": "treasury_bulletin_1953_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1953-6674?page=54", + "source_split": "train" + }, + { + "id": "UID0162", + "uid": "UID0162", + "category": "easy", + "source_files": "treasury_bulletin_2011_06.txt\r\ntreasury_bulletin_2012_06.txt\r\ntreasury_bulletin_2013_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2011-7129?page=105\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2012-7151?page=105\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2013-7153?page=104", + "source_split": "train" + }, + { + "id": "UID0163", + "uid": "UID0163", + "category": "easy", + "source_files": "treasury_bulletin_1981_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1981-7020?page=28", + "source_split": "train" + }, + { + "id": "UID0165", + "uid": "UID0165", + "category": "hard", + "source_files": "treasury_bulletin_2010_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2010-7143?page=49", + "source_split": "train" + }, + { + "id": "UID0166", + "uid": "UID0166", + "category": "easy", + "source_files": "treasury_bulletin_1943_03.txt\r\ntreasury_bulletin_1944_03.txt\r\ntreasury_bulletin_1945_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1943-6555?page=68\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1944-6567?page=83\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1945-6577?page=71", + "source_split": "train" + }, + { + "id": "UID0169", + "uid": "UID0169", + "category": "hard", + "source_files": "treasury_bulletin_1982_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1982-7023?page=73", + "source_split": "train" + }, + { + "id": "UID0189", + "uid": "UID0189", + "category": "easy", + "source_files": "treasury_bulletin_1970_08.txt\r\ntreasury_bulletin_1970_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/august-1970-6884?page=70\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1970-6885?page=70", + "source_split": "train" + }, + { + "id": "UID0195", + "uid": "UID0195", + "category": "hard", + "source_files": "treasury_bulletin_1956_08.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/august-1956-6715?page=59", + "source_split": "train" + }, + { + "id": "UID0202", + "uid": "UID0202", + "category": "easy", + "source_files": "treasury_bulletin_1939_07.txt\r\ntreasury_bulletin_1939_08.txt\r\ntreasury_bulletin_1939_09.txt\r\ntreasury_bulletin_1939_10.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/july-1939-6509?page=99\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/august-1939-6510?page=107\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1939-6511?page=60\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1939-6520?page=62", + "source_split": "train" + }, + { + "id": "UID0212", + "uid": "UID0212", + "category": "hard", + "source_files": "treasury_bulletin_1964_01.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1964-6805?page=99", + "source_split": "train" + }, + { + "id": "UID0222", + "uid": "UID0222", + "category": "hard", + "source_files": "treasury_bulletin_2001_06.txt\r\ntreasury_bulletin_2006_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2001-7106?page=50\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2006-7126?page=50", + "source_split": "train" + }, + { + "id": "UID0228", + "uid": "UID0228", + "category": "hard", + "source_files": "treasury_bulletin_1956_03.txt\r\ntreasury_bulletin_1956_04.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1956-6710?page=22\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1956-6711?page=22", + "source_split": "train" + }, + { + "id": "UID0229", + "uid": "UID0229", + "category": "easy", + "source_files": "treasury_bulletin_2005_03.txt\r\ntreasury_bulletin_2006_03.txt\r\ntreasury_bulletin_2007_03.txt\r\ntreasury_bulletin_2008_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2005-7121?page=109\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2006-7125?page=106\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2007-7130?page=109\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2008-7134?page=107", + "source_split": "train" + }, + { + "id": "UID0238", + "uid": "UID0238", + "category": "hard", + "source_files": "treasury_bulletin_1982_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1982-7023?page=44", + "source_split": "train" + }, + { + "id": "UID0241", + "uid": "UID0241", + "category": "easy", + "source_files": "treasury_bulletin_1963_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1963-6798?page=13", + "source_split": "train" + } +] diff --git a/data/officeqa_id_split/val/items.json b/data/officeqa_id_split/val/items.json new file mode 100644 index 0000000..290a52d --- /dev/null +++ b/data/officeqa_id_split/val/items.json @@ -0,0 +1,194 @@ +[ + { + "id": "UID0001", + "uid": "UID0001", + "category": "hard", + "source_files": "treasury_bulletin_1941_01.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1941-6529?page=15", + "source_split": "val" + }, + { + "id": "UID0027", + "uid": "UID0027", + "category": "hard", + "source_files": "treasury_bulletin_1970_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1970-6882?page=89&deep=true", + "source_split": "val" + }, + { + "id": "UID0039", + "uid": "UID0039", + "category": "hard", + "source_files": "treasury_bulletin_2004_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2004-7117?page=20\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-2004-7117?page=21&deep=true", + "source_split": "val" + }, + { + "id": "UID0041", + "uid": "UID0041", + "category": "easy", + "source_files": "treasury_bulletin_1970_10.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1970-6886?page=35", + "source_split": "val" + }, + { + "id": "UID0052", + "uid": "UID0052", + "category": "easy", + "source_files": "treasury_bulletin_2000_06.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/june-2000-7102?page=56", + "source_split": "val" + }, + { + "id": "UID0070", + "uid": "UID0070", + "category": "easy", + "source_files": "treasury_bulletin_1939_01.txt\r\ntreasury_bulletin_1939_02.txt\r\ntreasury_bulletin_1939_03.txt\r\ntreasury_bulletin_1939_04.txt\r\ntreasury_bulletin_1939_05.txt\r\ntreasury_bulletin_1939_06.txt\r\ntreasury_bulletin_1939_07.txt\r\ntreasury_bulletin_1939_08.txt\r\ntreasury_bulletin_1939_09.txt\r\ntreasury_bulletin_1939_10.txt\r\ntreasury_bulletin_1939_11.txt\r\ntreasury_bulletin_1939_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1939-6518?page=81\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1939-6505?page=111\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1939-6519?page=117\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1939-6506?page=95\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1939-6507?page=109\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/june-1939-6508?page=117\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/july-1939-6509?page=109\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/august-1939-6510?page=117\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/september-1939-6511?page=66&deep=true \r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1939-6520?page=68\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1939-6512?page=70\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1939-6513?page=72", + "source_split": "val" + }, + { + "id": "UID0072", + "uid": "UID0072", + "category": "easy", + "source_files": "treasury_bulletin_2011_12.txt\r\ntreasury_bulletin_2016_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2011-7149?page=58\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2016-535293?page=57", + "source_split": "val" + }, + { + "id": "UID0086", + "uid": "UID0086", + "category": "hard", + "source_files": "treasury_bulletin_2022_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2022-627778?page=86", + "source_split": "val" + }, + { + "id": "UID0091", + "uid": "UID0091", + "category": "easy", + "source_files": "treasury_bulletin_1940_12.txt\r\ntreasury_bulletin_1941_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1940-6528?page=21\r\n\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1941-6540?page=64", + "source_split": "val" + }, + { + "id": "UID0109", + "uid": "UID0109", + "category": "hard", + "source_files": "treasury_bulletin_2015_12.txt\r\ntreasury_bulletin_2020_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2015-519209?page=21\r\n \r\n https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-2020-598551?page=24", + "source_split": "val" + }, + { + "id": "UID0142", + "uid": "UID0142", + "category": "easy", + "source_files": "treasury_bulletin_1944_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1944-6567?page=93", + "source_split": "val" + }, + { + "id": "UID0154", + "uid": "UID0154", + "category": "hard", + "source_files": "treasury_bulletin_1977_03.txt\r\ntreasury_bulletin_1978_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1977-6963?page=83\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1978-6975?page=84", + "source_split": "val" + }, + { + "id": "UID0159", + "uid": "UID0159", + "category": "easy", + "source_files": "treasury_bulletin_2000_09.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/september-2000-7103?page=109", + "source_split": "val" + }, + { + "id": "UID0161", + "uid": "UID0161", + "category": "hard", + "source_files": "treasury_bulletin_1980_03.txt\r\ntreasury_bulletin_1985_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1980-6999?page=88\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1985-7040?page=48", + "source_split": "val" + }, + { + "id": "UID0170", + "uid": "UID0170", + "category": "hard", + "source_files": "treasury_bulletin_1960_03.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/march-1960-6759?page=64", + "source_split": "val" + }, + { + "id": "UID0190", + "uid": "UID0190", + "category": "hard", + "source_files": "treasury_bulletin_1939_10.txt\r\ntreasury_bulletin_1939_11.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1939-6520?page=14\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/november-1939-6512?page=14", + "source_split": "val" + }, + { + "id": "UID0213", + "uid": "UID0213", + "category": "hard", + "source_files": "treasury_bulletin_1947_04.txt\r\ntreasury_bulletin_1948_04.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1947-6603?page=28\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1948-6615?page=18", + "source_split": "val" + }, + { + "id": "UID0217", + "uid": "UID0217", + "category": "easy", + "source_files": "treasury_bulletin_1963_10.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/october-1963-6802?page=15", + "source_split": "val" + }, + { + "id": "UID0220", + "uid": "UID0220", + "category": "hard", + "source_files": "treasury_bulletin_1939_02.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1939-6505?page=25", + "source_split": "val" + }, + { + "id": "UID0233", + "uid": "UID0233", + "category": "hard", + "source_files": "treasury_bulletin_1948_04.txt\r\ntreasury_bulletin_1958_04.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1948-6615?page=42\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1958-6735?page=54", + "source_split": "val" + }, + { + "id": "UID0234", + "uid": "UID0234", + "category": "easy", + "source_files": "treasury_bulletin_1958_01.txt\r\ntreasury_bulletin_1958_02.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1958-6732?page=28\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/february-1958-6733?page=32", + "source_split": "val" + }, + { + "id": "UID0235", + "uid": "UID0235", + "category": "easy", + "source_files": "treasury_bulletin_1948_04.txt\r\ntreasury_bulletin_1948_05.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/april-1948-6615?page=27\r\nhttps://fraser.stlouisfed.org/title/treasury-bulletin-407/may-1948-6616?page=27", + "source_split": "val" + }, + { + "id": "UID0239", + "uid": "UID0239", + "category": "easy", + "source_files": "treasury_bulletin_1953_01.txt\r\ntreasury_bulletin_1954_01.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1953-6672?page=62\r\nhttp://fraser.stlouisfed.org/title/treasury-bulletin-407/january-1954-6684?page=51", + "source_split": "val" + }, + { + "id": "UID0240", + "uid": "UID0240", + "category": "hard", + "source_files": "treasury_bulletin_1957_12.txt", + "source_docs": "https://fraser.stlouisfed.org/title/treasury-bulletin-407/december-1957-6731?page=26", + "source_split": "val" + } +] diff --git a/data/searchqa_id_split/split_manifest.json b/data/searchqa_id_split/split_manifest.json new file mode 100644 index 0000000..9b87645 --- /dev/null +++ b/data/searchqa_id_split/split_manifest.json @@ -0,0 +1,21 @@ +{ + "benchmark": "SearchQA", + "manifest_type": "id_split", + "source_repo": "lucadiliello/searchqa", + "source_repo_type": "dataset", + "source_url": "https://huggingface.co/datasets/lucadiliello/searchqa", + "source_id_field": "key", + "counts": { + "train": 400, + "val": 200, + "test": 1400 + }, + "item_fields": [ + "id" + ], + "notes": [ + "This is a split manifest, not the full SearchQA payload.", + "Materialize full split items from lucadiliello/searchqa before evaluation.", + "The IDs in items.json exactly match the key field in lucadiliello/searchqa." + ] +} diff --git a/data/searchqa_id_split/test/items.json b/data/searchqa_id_split/test/items.json new file mode 100644 index 0000000..45f3f0b --- /dev/null +++ b/data/searchqa_id_split/test/items.json @@ -0,0 +1,4202 @@ +[ + { + "id": "5093b6d997674d25be29a4c94fcd5185" + }, + { + "id": "99dc9de2fd094b8e9de41a9040081c1d" + }, + { + "id": "979e3f5aac91446b8550161dd89e9e26" + }, + { + "id": "3bccee60c0c149e08451db23b0963238" + }, + { + "id": "7cd88e67c6c64595a44707e44a33c57c" + }, + { + "id": "feee13cb93224a3297a7b87ddb424640" + }, + { + "id": "48e0ed33e6794e50a1894269c8110e47" + }, + { + "id": "ed76c0f0111f48a8bddb25039aeb2ad0" + }, + { + "id": "fd75250e93884f6a83463d93b2108db6" + }, + { + "id": "a34708851fbd4ddaba3dce11d13bed71" + }, + { + "id": "6371fd8b1f9a400784076e296ec015ba" + }, + { + "id": "458f4f9ec50b4233b211b12f1eeca2a0" + }, + { + "id": "6db35769ae81425781881647d14b9533" + }, + { + "id": "e166749f42c946f7a18160e48f3c9c77" + }, + { + "id": "34f4c188587e4118a9c65bf31bb1cfdb" + }, + { + "id": "fe5a8c72fb9e4c55933786c6e49072d0" + }, + { + "id": "e5f27592956c4b0582eedfb37045a266" + }, + { + "id": "1bad2fba539343539ccc0666d70f4da3" + }, + { + "id": "1200fe70bda14116affdd48427cc63ee" + }, + { + "id": "c09dc55e403141348ba0778942aef804" + }, + { + "id": "d5bd248435fd48858beef6626bbadbdb" + }, + { + "id": "75815be29bed4f45aa300fd41e9018ef" + }, + { + "id": "4185d3cd150d40c9920c37e627eb5153" + }, + { + "id": "633265a55d45428c9866b45e75c0b67e" + }, + { + "id": "f771a66784f2494b8c0a5aa81f9bd09f" + }, + { + "id": "442ca352dc31418a9548fddc8259ef66" + }, + { + "id": "00090d662c4c4db9b088922fef9fe061" + }, + { + "id": "29043bc89e504948b706b4c502a39a27" + }, + { + "id": "db3126341393421caf287bf600f97a46" + }, + { + "id": "215ec23d85f74b059a2e55edab5da89b" + }, + { + "id": "525ee6acd75f4ec98aa1c86eb067e49a" + }, + { + "id": "203cf01361a54663adf565df1070296e" + }, + { + "id": "10fac0924a5e425098992c27d77da289" + }, + { + "id": "35b8579be49e4ac598df516858dda290" + }, + { + "id": "962d456952b14ca4a40adb6a665e4655" + }, + { + "id": "e2a6f8f3f7fd468dbd6c8accff13a62f" + }, + { + "id": "633c88e4bffa422ea29193c32eecd3ca" + }, + { + "id": "c37a6b4a6320407089cfcf6effca6f2f" + }, + { + "id": "b7317ee96b234164b583906a8daf293b" + }, + { + "id": "d05ced73459848598523c0de21f63fb9" + }, + { + "id": "657fd4c82a584e22a4c49c116950f142" + }, + { + "id": "3e31c94908e041c7a855dd7ac5659f2e" + }, + { + "id": "34b216269ba5429d9077cdc2c7bc3e42" + }, + { + "id": "9eebfa2316cd49b4912c6764cb21480c" + }, + { + "id": "917d841d16014bbba49d0d3368223e43" + }, + { + "id": "5811f1e8130d41fb87b71b4b86c0588f" + }, + { + "id": "3bf21733e43c4b5fac681d0e9a4a3b83" + }, + { + "id": "0429a0309ef94df8a59959beb9e87fa9" + }, + { + "id": "a0034b2ceb474e749fc178de8e297e27" + }, + { + "id": "285b2733a61f4fb19680ee93b7b80392" + }, + { + "id": "b818cb227aee48e6b44b1d5dbeaf179b" + }, + { + "id": "0f6d2de288e348779e8c5d7515792449" + }, + { + "id": "7d249f57e28e4719aaec9d223e0a27e7" + }, + { + "id": "0de314b893ea4c7fb7294a61e3792bfb" + }, + { + "id": "f75bce4f743c4a6d984b64e71bb00261" + }, + { + "id": "5b015d8aaea04ca68fb2150fad53f044" + }, + { + "id": "0216d5236aeb43348eaea3284961edad" + }, + { + "id": "50072c2bf1e7453e8f3e59fd50e111cd" + }, + { + "id": "dbea47b42dec407f98ae495bb3adb83e" + }, + { + "id": "212743a2ca8448998afb34fe71c6f704" + }, + { + "id": "6c43e20b398d49b5be1a34ec82a28e2a" + }, + { + "id": "638753d6628e47258d18dc18e1a99623" + }, + { + "id": "b23add0ac57b4bc39756687a895d3736" + }, + { + "id": "a7ecab7cd26b40a6a22254f18e9823c6" + }, + { + "id": "ce664a256193487bb4eba33daa4bc8b3" + }, + { + "id": "f4ecbbfd207c46dc9a72ea0b83b0c3da" + }, + { + "id": "6a67adec151c4625969743e37ad6393d" + }, + { + "id": "d054ab0a3bb945a3a156b9b097fa360d" + }, + { + "id": "344bf9a4bb1a4005a4461ae840c13d4c" + }, + { + "id": "c1e6d716026c4caca73f73628224a1ed" + }, + { + "id": "e8fc1c24fcc34b74ac2048a5d34c9e4a" + }, + { + "id": "4e11443e24ce44f58ff3ea558f121165" + }, + { + "id": "3331ad327364476aa685018cc01293ea" + }, + { + "id": "731cafd017b843f78bd4c1b8114a5e4d" + }, + { + "id": "e5cf2bd4da0e4489ba32862bcaa41647" + }, + { + "id": "5003ecf728f441f786d421fcabc73105" + }, + { + "id": "e87835a26a3b447cac651c8c7e8d200b" + }, + { + "id": "33c1f72dbc194869beed5bf366a6720a" + }, + { + "id": "93f29dc1c04c4d50937ce66a8d373825" + }, + { + "id": "dfe37094074c4f18840cbd3ddc3f5272" + }, + { + "id": "7c9086aad3f14242bf9a239b0dcc7dfa" + }, + { + "id": "be19f4b2a21e492ea622d31c29dd01b4" + }, + { + "id": "16e363a272714ecda717539f291b3f89" + }, + { + "id": "83b243debd0e4114a279556f70a2a0b5" + }, + { + "id": "b2e6b86c2e54434a85e0f2e6527cf6db" + }, + { + "id": "2c76127f56ba4abda3e8ed09146c562d" + }, + { + "id": "eaa107b4f1224f229132381f53bb4588" + }, + { + "id": "b83ea47cda2b4428861c4a86eed64ad2" + }, + { + "id": "9097a4bf75784749a1be32d1e5c0b875" + }, + { + "id": "c5232cc0d59147dea6d8cff9c57428a6" + }, + { + "id": "9a94c07609304bc890685fb0d6ff799d" + }, + { + "id": "a2743674044c4315be9b5a6151cc5f63" + }, + { + "id": "77016bca1c08489493e49af1086680ce" + }, + { + "id": "a95d37788dab44fd9249c25f84e2a068" + }, + { + "id": "aaef2082129b4cfdb2de0a281b65cab6" + }, + { + "id": "bd1eb94035884688ac84e585fa2b1088" + }, + { + "id": "86a2c3181d54411fa035b4d950354130" + }, + { + "id": "30363d00e669430bb626175fcce58d58" + }, + { + "id": "ac3f4d90416d446996962b0020b9c635" + }, + { + "id": "dedcc7b25ccf4ec388788d84adbebe07" + }, + { + "id": "0f72f41ca4ea444a97be577a1beb9bcb" + }, + { + "id": "a5f3de4b150341e08b67e7dc71204703" + }, + { + "id": "2d937a8e07f1428f9bad2047ae13753c" + }, + { + "id": "d9aacf57ec6c4710973206838deca5f1" + }, + { + "id": "cc7512bd9a034ff786f3ee0a7da05baf" + }, + { + "id": "414b1faf561e44e291e1cba1d53d4e0e" + }, + { + "id": "e8148b60bf094f7f995b7254b8e18657" + }, + { + "id": "dd00ef077b1e4d1883cf381ed260e4c4" + }, + { + "id": "2a022332b2604c7bb184920b069ff82e" + }, + { + "id": "5774b5d48ecc41c884be95aea47e04db" + }, + { + "id": "accb1302cc6143be94dd440f7f50c447" + }, + { + "id": "ce906c622e204ba6818fe9b27aac5086" + }, + { + "id": "bfea66d8093f4a459969f18b65950d5d" + }, + { + "id": "67c2645726af424ebef4df2229368abf" + }, + { + "id": "ebd72a59dc8d4c6f829b690ab8211ec5" + }, + { + "id": "2e32f9a614fe44c488f5f3c973b956ee" + }, + { + "id": "1bbc678c2c2144a0b98d1db7bd97aab8" + }, + { + "id": "8dc2205b82664584a345454726172232" + }, + { + "id": "de3a5ad837144fae8529d750fcfe6222" + }, + { + "id": "d3a34a078fa941fb93afb7cb5ee930e3" + }, + { + "id": "0f4a622654634b7987233ee50d5f72c5" + }, + { + "id": "b441b6baebff401eb4b4d84921f39ff9" + }, + { + "id": "00f461b32ab54882926444d14c1c3477" + }, + { + "id": "a790b16e8712437f9650d552069ab5f1" + }, + { + "id": "957b0fabbe7d4d9184a31d3290258509" + }, + { + "id": "8e981230d1a340cea3439ecda7c2ea73" + }, + { + "id": "0243fa2f9e5d48dc8c38f1e9380133a3" + }, + { + "id": "3045f82703e348d9b7355a602dc2653c" + }, + { + "id": "452f87d64e68457b89435f3595a3f788" + }, + { + "id": "e19f0550cc464e6aa8a343ac4905db8b" + }, + { + "id": "b996c427bf094af6b711b737166e5f1c" + }, + { + "id": "a7dce4e10bc94faebb98ca90b963b026" + }, + { + "id": "fb48d375beb54c80b9b658c3346aa0f2" + }, + { + "id": "1f9e3d57a12d4308bf7476b2ea0e9c9f" + }, + { + "id": "ab585940f21543cdb602c7f34606b808" + }, + { + "id": "57614fd6dbb04777b14b5c88baa3cb66" + }, + { + "id": "18a7d5d6288145d98f37834834de5b60" + }, + { + "id": "de4c635a93fd49b8829dcf3926407b70" + }, + { + "id": "0bf04282c3634ddc9384ae26caadd1c2" + }, + { + "id": "4483bb93ad36402ba3e5e3257abfb415" + }, + { + "id": "34874f99ede8490f8cb0a6989cb2a2e5" + }, + { + "id": "7be287bcb7614aa3afd1932f498cc515" + }, + { + "id": "12692cce4f124fc2893d040bc78b1f90" + }, + { + "id": "8c983ce1cbae4056ae03d78d8b67053a" + }, + { + "id": "10fbb659301f4b829e5c29ae08195da9" + }, + { + "id": "65ac6a0f59b540adbb13949a79c487c9" + }, + { + "id": "f3ead327d41f4d099f6bafde717d9479" + }, + { + "id": "b26721de2d9a44f495accae7093d4f3a" + }, + { + "id": "24eb5a009ee64909bad741ae3afe0456" + }, + { + "id": "3c5ee2e049864ca98c7ff8cd1c86942f" + }, + { + "id": "7209c6f378a14c31aa517a773c7869b6" + }, + { + "id": "abd5eadacaa840bd97c69e5cc1beb1e7" + }, + { + "id": "b75da932a89d47d8a88a7516e446ed21" + }, + { + "id": "beac9af49fe24592abc9acc2c575e6e3" + }, + { + "id": "b7a3b748667348b7bca76958e7ce110a" + }, + { + "id": "96c7d722c5454846a324682d10dcd4e0" + }, + { + "id": "d26f3e0fce49475595ab98354c941d49" + }, + { + "id": "86f594b52f7148ee9ae371cb580c00ce" + }, + { + "id": "491fe356172c4f0f8899595cd13426db" + }, + { + "id": "08fee65a34684a9ba7f3dced52e2eb69" + }, + { + "id": "540f3cb9af4e4c3c8ce39b2c0ee94522" + }, + { + "id": "2db7671583be4f56be84f68677b6c925" + }, + { + "id": "f517a2fbdcd24448b12f40613cff19dc" + }, + { + "id": "05883443928c47a2aa0c15e84cc3efce" + }, + { + "id": "b0d285b94a5f4f2cb0605919d0567703" + }, + { + "id": "c3af854e40fc4dfab301bb95e9d9e3d2" + }, + { + "id": "51a410837fa748c0ad59b078f5b96ef7" + }, + { + "id": "bc05ded989c342ff8a9058bfaf2fd2f5" + }, + { + "id": "ec656d35dc024d7197c664055e09a4cd" + }, + { + "id": "29479a10bb7a409d8ec13616f197f5c9" + }, + { + "id": "796799cc49d2488c9300266c324a0001" + }, + { + "id": "8af2fd8cef0c454594e4f3a568588472" + }, + { + "id": "9d7d95de350d49a783ae35644ffcbec3" + }, + { + "id": "642513b51abe48cc9734dc6164c65a60" + }, + { + "id": "d6c08fd858e14dfd98b924c5aa317752" + }, + { + "id": "b1934fb8f0b84a2bbfd90811a7610018" + }, + { + "id": "35ca318c97e44a5fbacec652c6aa178a" + }, + { + "id": "d979fed6fede41ceaa780d115d1d0124" + }, + { + "id": "741b6eda4535483cb9c80242089d8eb2" + }, + { + "id": "3da5b2392db8414bba6d0c3da2ab3f79" + }, + { + "id": "a7bfde0104e34c739685cddd40203085" + }, + { + "id": "9167af082d2742d282753d9175fbeec4" + }, + { + "id": "851efecede9a4223bd660d986d272388" + }, + { + "id": "9e8fc31beb22409189a62f1cbbaf95c9" + }, + { + "id": "440bce5338e34f74961b06d4e2f538ad" + }, + { + "id": "6fab6fbb778c4ad784fe0ea2fced512e" + }, + { + "id": "6e2e7a60038547609427b5cc58b35de0" + }, + { + "id": "faeeb90e9c3f47edb21d0d7ad41c22a2" + }, + { + "id": "30cc22b7eab4467f8f68ae92ee761191" + }, + { + "id": "dca652e3f8db4a06b132d57c30c5c77b" + }, + { + "id": "94a90d73a1ed45ab99840bfd4cf10a32" + }, + { + "id": "40c86f5f19654755b76a872cad12e8fe" + }, + { + "id": "d6e95e41b53943a4a2caf08178634e94" + }, + { + "id": "f26ef14e89244472a71ad384cf5214db" + }, + { + "id": "c54e6e0f897645c18a28a0637f193908" + }, + { + "id": "5218ad458ee3405eb9c00a67cbc1e17d" + }, + { + "id": "9a2ffbc032454b858f83cb50793a4c83" + }, + { + "id": "7c18240ce2aa47fcb1230cb11f9baa38" + }, + { + "id": "0deb5a98a74b4f8c826c8096984a8e0f" + }, + { + "id": "bffb34366c9d400abba4d4e99017c345" + }, + { + "id": "69904776d81641888e4bd93ea9574275" + }, + { + "id": "77c033ce24b14a38b2131274c99a409f" + }, + { + "id": "b499643a00034309bb74ace2469a01f1" + }, + { + "id": "30c1985fb1704367bb35b80a6666cd10" + }, + { + "id": "0ffdf0ac392c4050bc585c32b7a48aec" + }, + { + "id": "0505b21549f74c7c89a368a9b41b81c3" + }, + { + "id": "3574ef631b4d48f7b4b2e304c1b7eba0" + }, + { + "id": "f233a9c1c2fb4650bae56cfc02ad8cde" + }, + { + "id": "3806f3917ad74a7abaf1ac64016cf9b2" + }, + { + "id": "fb928910625f422ba1897825b5ab5f4f" + }, + { + "id": "cb1b48bb73ee438081bb7b3dc484dc3d" + }, + { + "id": "b439e5783bd64c238c306b520a44aa04" + }, + { + "id": "1c168ba59de44d469fba6009f8c14e9e" + }, + { + "id": "606268049245493e960f6a2fe6ddf9ba" + }, + { + "id": "6f76f4ec0a1c46f3824786e43acd1a42" + }, + { + "id": "6e33a868ec674f59bc2e0a7fa4686f4d" + }, + { + "id": "0ad6f400c12a4da8815213c1d6aada43" + }, + { + "id": "07782e76e7ec403b883dbe704dc16ba2" + }, + { + "id": "d6254d08e9c44ac1b94eea9be6624549" + }, + { + "id": "5287b59ad7184131a93c827769df7ae9" + }, + { + "id": "51b4c515dadb49da82b22445a69a9881" + }, + { + "id": "0cd00484d33645b0a58f51dfec568995" + }, + { + "id": "db1c3a9cc3df4782adc936838679d1c0" + }, + { + "id": "0ea40b8199ce485e97511ed1a0f4b329" + }, + { + "id": "a8e6016661be494ebc65e7a80a1d7ff1" + }, + { + "id": "7821675f0b614f5b89828e5004ce01a4" + }, + { + "id": "783e9f82f9c044999fbe04aaed091a80" + }, + { + "id": "f41714b7a8db4810ba3ddbcfdeaec344" + }, + { + "id": "345d4dc06c7e4bd0bf02d54c83af22f0" + }, + { + "id": "a9600ebd49684f158a292964aef91319" + }, + { + "id": "3ed3ab8bdddb466a9001f8282173d0fe" + }, + { + "id": "c1cd1e093cf54677a83e671a9a332b98" + }, + { + "id": "e95bc48e53df41cda7d0f253149bab48" + }, + { + "id": "0878e886ed474c4197ae7b01383a73e9" + }, + { + "id": "9389182a49724617b0f54be9213d0d33" + }, + { + "id": "d62c4b22ddb244608a60d467aa5c35e8" + }, + { + "id": "c5dcf70c3a47476699dbe6811140cb58" + }, + { + "id": "2c474544cb4543688efecf4d0b9393c4" + }, + { + "id": "fc5312021cfd4dd998dfa3f7342ab62b" + }, + { + "id": "31a472d2b8a942c9a7d7f8050233a960" + }, + { + "id": "9d31050c4b73430485803dd317f7a987" + }, + { + "id": "88243e966a8b44fc96c8fa1d8ee78ff6" + }, + { + "id": "2fd1897c4e914a4b8c1e759123ebc1c3" + }, + { + "id": "8b9d892de0e143c69dc410f5123e7520" + }, + { + "id": "25e3396cd2404cddb1f5a01c7e6e7553" + }, + { + "id": "81ff2cf8bf2e4a0cbff1aeff5c0e67df" + }, + { + "id": "ab628aaebd1149a596554d215f8207d9" + }, + { + "id": "ff142be20d6e44aabecda244878d8406" + }, + { + "id": "f59615e5df1c4b02a8cee40b10c61d8c" + }, + { + "id": "a7312ddf63744c37b682fd71a7a30698" + }, + { + "id": "2f64879434954dc380f9213f868e8338" + }, + { + "id": "32f12b8717454d6c9401909b517875d9" + }, + { + "id": "7a83242b5eda4715a578d5eece0712f4" + }, + { + "id": "a4f828a3afb942da92913cc3e7f36c24" + }, + { + "id": "007e0f579f49464d90e43b6e24cefbb4" + }, + { + "id": "7adeaa03d9384dcc8aff7919d46bb858" + }, + { + "id": "5a59d6e1cec84d6ba54c03a3910af065" + }, + { + "id": "54d99ac9fa03443e99f77d6f9f208713" + }, + { + "id": "b94c8c938de9476f89661dc6d95d1c86" + }, + { + "id": "d29e5124d9364d75a7a6a2973a616c64" + }, + { + "id": "1aa44299fa434210937bbaa1eaf40f5e" + }, + { + "id": "ad9d13ba642c4ede85353eafe933faaf" + }, + { + "id": "ca692ac8c8d1446c90ed8a4c9d44dde1" + }, + { + "id": "6f1d2a3ec3df471987e4b3c6492c1880" + }, + { + "id": "fcba0d52a55b4d9587eb8442ccf98928" + }, + { + "id": "6d30545d099e4ccc89f1ce4b98c6ff24" + }, + { + "id": "1efda6b3a760493ea90f2a7b5b8bcd4f" + }, + { + "id": "bc0f71c7bb4c42a885543244834267fc" + }, + { + "id": "9cce6a8e69214436bdb7a923d2cfa82a" + }, + { + "id": "b167ddfc10bd436aae34f2bab13405ec" + }, + { + "id": "9e1e036499dc4e0eaa569116c6abb0ae" + }, + { + "id": "832a75b67b3448818c5f87a733494d9f" + }, + { + "id": "27a3d2219dd647dcb8832d1960346ce9" + }, + { + "id": "388c301702f24869a50b31b3ea08ab43" + }, + { + "id": "01723c627507432bbc50d7cadef3b4db" + }, + { + "id": "aee570424f86443691e340cc80dc9063" + }, + { + "id": "80b349e3d51f4adabec627ed444b90e7" + }, + { + "id": "2f8054d5ed06439e8deb919c9c2449c6" + }, + { + "id": "008613ff6fba4f7a98143b17b9da7d2d" + }, + { + "id": "c8f835ba31104b738f8becbcc79adec2" + }, + { + "id": "a42d556eb5d04305a6e25c875f8f0746" + }, + { + "id": "ba3d5db296944affafda05f4d628c2de" + }, + { + "id": "b0ea89ee6dbf4d19a264528fb0b5d2ed" + }, + { + "id": "9430ec5597fd47a6a90f9db7da75bbae" + }, + { + "id": "5ae6225b818a41fc8e812560874921fb" + }, + { + "id": "ee8fc58e226e4b9582ed9dc2efaf0e6f" + }, + { + "id": "aa8d1a9cc4c24910a729bfddfa1ad4e8" + }, + { + "id": "2ae7fad172b94cf9bf4cb7eaa3a85ee5" + }, + { + "id": "ba6a74f9cd8f4a079cca008178933650" + }, + { + "id": "26dda87f8ce44abe9b89e780c0ed4007" + }, + { + "id": "7dd589a7e5da4b4cb383f057781e1889" + }, + { + "id": "7e92bf0c1c234cc1a162cefe55db3214" + }, + { + "id": "a716cc3098294cbd9a6e00529a414b41" + }, + { + "id": "cc73517365b94e149cbfc12550b3ae88" + }, + { + "id": "4e3710d2b4264c54b0824f4961715d63" + }, + { + "id": "d8b9fa1f5cc2426ba74f3bb683d39064" + }, + { + "id": "5ad93ba20a924f1fa75167c1b59cd102" + }, + { + "id": "7eac903ab82446cbbd0e55db99585f64" + }, + { + "id": "fa8f20b61bda46b98977383e6005869e" + }, + { + "id": "77d63f024ef7435daef2003a2df801a3" + }, + { + "id": "902560ea8e394e839d513351a0f9c2a9" + }, + { + "id": "28ddc28a1e224b9aa3be88052db91e2b" + }, + { + "id": "45356ddb6e944e30a15526922f123ed1" + }, + { + "id": "b1321e17fa1f436292adc0ea256bd280" + }, + { + "id": "76886f1a93914e37bb30e1ca06f555ae" + }, + { + "id": "39fe55b58957486a9531449bc6a9b0e1" + }, + { + "id": "943c20e85f324584b597e50e0b7d2d05" + }, + { + "id": "e4a6d8e2126b4fb6b767debdc234a86d" + }, + { + "id": "30866cba68bb479d8d5a3017dbf63095" + }, + { + "id": "af5ec5e74b2f4a8685bb7f25fbcbb9b0" + }, + { + "id": "006813695f494d42b49438c8a0057f73" + }, + { + "id": "3fd44eac02b5430ab355a5aa5e2b7f66" + }, + { + "id": "43cd4f4251f04af1abb6e1977eee6ddc" + }, + { + "id": "2a24afd4f4df46ce9481cbc8be5b55c8" + }, + { + "id": "add3ee5dd6454eac91efd33a2a7a6163" + }, + { + "id": "b315d931dfb2440e92cf1f9e063be50c" + }, + { + "id": "174f39c3019f42c29c17c2b62b6c9e46" + }, + { + "id": "1f9b96014e71418cbc0ba20a6422f91e" + }, + { + "id": "9a9ed6bd43684cf6a74f78a8e4aa8431" + }, + { + "id": "f03134f8cfb74c6b9d883b69a587c101" + }, + { + "id": "1c7f885133ce4b6eb2af794a2d430d2d" + }, + { + "id": "7773b38bf1b846818f4bba066f7b5d07" + }, + { + "id": "8ae5b2a6fbe744929bf4f15b5af9d5f5" + }, + { + "id": "dbd0d9898ec74cc6a5d38835a9c3f6e1" + }, + { + "id": "16ae8a29bc944d4e970f0d89ba5b2dcb" + }, + { + "id": "61a6fdc4ec9c4834a833c8b6ed438b61" + }, + { + "id": "261ebacdeb1f4a0ea230f79306532be5" + }, + { + "id": "100e71fde3674af69110d1d286af2550" + }, + { + "id": "f8d0e0ecdc7c4420b7f1f513e18429a6" + }, + { + "id": "7622138eea5d4073a7e5a6acc5beebed" + }, + { + "id": "f40a291848ff41c5adba099e4987078d" + }, + { + "id": "43f44abdf90b4a2ba258141dedb67a06" + }, + { + "id": "45ebb1b3c46140d09682cb291cd38d6f" + }, + { + "id": "1af0be8661d64af09e1065d652312aae" + }, + { + "id": "42b2368515d94cd4bb295e157ed3cbe6" + }, + { + "id": "ccd7fad85ef94625bb2b2cd1dcbbc9c9" + }, + { + "id": "b28bf588eed3447a8c9fdbf57dc2fc1f" + }, + { + "id": "a0ada661569e4ab2a15db06560ec7b45" + }, + { + "id": "e30ef51251a847138f4826c6146e982d" + }, + { + "id": "85d92d7d7410408780bc11c741de1676" + }, + { + "id": "2eb12073b96b4fcab7db4f7c0f183627" + }, + { + "id": "1212b9bbd8db40c989cf30794f638eb5" + }, + { + "id": "8f427af6ba8d4055947300f8781815d0" + }, + { + "id": "9ae85053dd2949a9ab57a8963eef3b02" + }, + { + "id": "01eac94b21a24dcead6f9c3efec2a58d" + }, + { + "id": "aa790318578147efac1e0e1ab82d6368" + }, + { + "id": "e29870210b084e41a6e65fff44bd5f75" + }, + { + "id": "229a49365ff4462d8bc951ddb68ff1f1" + }, + { + "id": "e9433b099ee14acf96e5cd7422e6c577" + }, + { + "id": "43eaa27f886b4573959d36b086d83631" + }, + { + "id": "46b71fad5fff409cb48a1d3493bbb2e5" + }, + { + "id": "47ea5e3fbfea4969a8e4d6b8167daf7e" + }, + { + "id": "857ad42459c14613b984c39ca68430dd" + }, + { + "id": "28d672eed13241539963231a521343ee" + }, + { + "id": "1bca1e13fe4c455b888d98f248eed26d" + }, + { + "id": "8512ed5d054544cfaec47e568a5e099b" + }, + { + "id": "21a0bd79a801484098879d6dce609641" + }, + { + "id": "3ddbc52a1228479bba3296be9a652bb6" + }, + { + "id": "6d65e0f818d64a24a17d4ba6408d58ae" + }, + { + "id": "e3bc691db8544b53a257438e4076e07e" + }, + { + "id": "1c03c07e617e4f92981e569fce970642" + }, + { + "id": "03cda26c9db54f219e91f5901e725dda" + }, + { + "id": "2f0b7b6952ee4110ac5fb9f91c318650" + }, + { + "id": "bd6bb9b794784cbea8c7ac17e19d119d" + }, + { + "id": "61e95a62a8104bd49be3c8bde39d4f5e" + }, + { + "id": "63d0331e94bb40fbb212adc3d586480c" + }, + { + "id": "dccec34c17324c79a7a24c6f55edc4d5" + }, + { + "id": "381cee68ee3146b3bfdda9b2db468a21" + }, + { + "id": "fbea4c11417a4361adeb09ad77c5076b" + }, + { + "id": "414e2f13970c40c4916add89f9cfa235" + }, + { + "id": "6c71cf873c0142b48c5fe720db98eba0" + }, + { + "id": "1d4d71abca5c48b4a6f3722b09da8643" + }, + { + "id": "1b623543bd15458d80e07eae21850457" + }, + { + "id": "4735c55c7d9f45688c00b6f18aaf9071" + }, + { + "id": "4aac711a9ae143968a32b462772a4830" + }, + { + "id": "5b3065ec8dfd4136a45bb6842419130d" + }, + { + "id": "a95dda91cd1f47ab8473faf8c6d370b5" + }, + { + "id": "09610f7c19cd4da480f84cd47fad368c" + }, + { + "id": "b4efa8b30168419cbb1e460342c15062" + }, + { + "id": "d18899bc5ff04f099b43a18bf4feb87f" + }, + { + "id": "b8eb90ad5e1f4c93abffc3452a756bfe" + }, + { + "id": "295f574b0bd6480eaa67181d0f684dc5" + }, + { + "id": "b5068f901df64f1f9d531cd0b22c5072" + }, + { + "id": "ba57a86593cb4c0781e2b39cc5a4405e" + }, + { + "id": "eefa42e9cdbd4619998877f1e95eaf52" + }, + { + "id": "2845343a14194de2bc49a207eb280d9d" + }, + { + "id": "b4c605da009b4af1ab94dce00ffaf73c" + }, + { + "id": "c63517e2f3b543afb9157c6e926f3a22" + }, + { + "id": "83702fa1bfbb40eb894c505260c23913" + }, + { + "id": "e2e5aaa18d614e0094723f4d57fa1086" + }, + { + "id": "7917a5e7448447688f273dd89c4b289e" + }, + { + "id": "3b9b31a7fe0849a18e4d75d6e1d7a915" + }, + { + "id": "8d269142648140508ab96ac735141a80" + }, + { + "id": "b5fdffb7cf664532a0de7444ad3bed4f" + }, + { + "id": "f024764eb6f44680852b89af7af64fbc" + }, + { + "id": "e50d5eee9eac47dd819070f8c206725a" + }, + { + "id": "99c441596f1c40f398aff4e4451f428c" + }, + { + "id": "dffd548d127c4ccfa56872c36e79deea" + }, + { + "id": "c8535d02c6354d5caf99f91107e9cb0b" + }, + { + "id": "e4615445187d4054bd1751442077dfde" + }, + { + "id": "2d87e6cb50ab4c3ba5df1c11245e5991" + }, + { + "id": "66f76ae1211e41569d562f37fb2b19de" + }, + { + "id": "c80e413e7a1e4a02ba048cc5246859be" + }, + { + "id": "d313db2759024237bbc3e0263f823646" + }, + { + "id": "916d6c683689416f920c33fbada5e93f" + }, + { + "id": "71c2617a69f845edacd806df4f07f3f1" + }, + { + "id": "d3f8ca7e8c444f63acc92433810f5209" + }, + { + "id": "eebc59492b3d4da3a99f616eee08f4a9" + }, + { + "id": "05bb7fe82bee447fb0c2f7886686c5e8" + }, + { + "id": "667a5aaeb91c4ca79152cfe70aa132c4" + }, + { + "id": "1c0986d2f8b34d71a803209e1c1af4b2" + }, + { + "id": "bbf82dbaa08b4b8587eb3872b8f7a876" + }, + { + "id": "015cbdf5c4b4412caf81e8b0fa8d0d2d" + }, + { + "id": "a527ced709fa4ddc9bb894a740ce5b8c" + }, + { + "id": "b7258e5a3a2440a6bce0a2f255e1cbcd" + }, + { + "id": "4816238548e944ecb6cbace1042fc0c4" + }, + { + "id": "207951638c3e41b8b33768284cca03d7" + }, + { + "id": "a19dc82cbad1465baacf847c78322862" + }, + { + "id": "7085fa663e24471494b27184d9729b3b" + }, + { + "id": "66bdd306b3794732980b97a5dda113cc" + }, + { + "id": "aaa7234299e2425c8b8242bb3d3dcaeb" + }, + { + "id": "516e0f59e3e7453789815ebab013bda0" + }, + { + "id": "c5a87bda6ee74fea8bc0a238af242296" + }, + { + "id": "65c309bf461c46e4885d28d56dd9e4f7" + }, + { + "id": "1afc488880414f70a478f861e1bb181b" + }, + { + "id": "a549a72dcb53436faa67c8b681b85735" + }, + { + "id": "e5b3512a238c4b12a749ab93a8845fbb" + }, + { + "id": "bc382a25e8da4f51a5a9cba127614b4d" + }, + { + "id": "50ba2a5324884e88a4011d8772de0fd0" + }, + { + "id": "7ca14fe0dfee4063a1298c77da63f7a9" + }, + { + "id": "c0649d456e0840769b8447d227e1c105" + }, + { + "id": "f7d3654582cd4886906bbeb267d4f3a7" + }, + { + "id": "794123ece5e24a198b589321f65b401f" + }, + { + "id": "483466929e33400bbfaf264af8db19cf" + }, + { + "id": "f4d1c6d45e0f4f4e8e0403bc7ef6b104" + }, + { + "id": "33410f4502a141ba98c15b3972a2e11a" + }, + { + "id": "9935b8b08a8e4f0d9ed37a330a6cf0fe" + }, + { + "id": "d09e46053b20406e882ae6d1597069be" + }, + { + "id": "aaf7b5f9f4fb4dbea1d57e8aa6cfe228" + }, + { + "id": "3a44d3746b9b447c814217d21fbef168" + }, + { + "id": "2c0fe5eea1484a689b357d345abba854" + }, + { + "id": "96e5b8ae45b74e3b84ea0dcb32f6e597" + }, + { + "id": "f26e7f72328a4556a9fb5655e7770e28" + }, + { + "id": "d392df665e19486594f19652ce61a96b" + }, + { + "id": "3c8e14348a394c04943897987ca68f8c" + }, + { + "id": "57a9c41a87254962a153f091182dc1c0" + }, + { + "id": "696aa36338a346d4929808d4679b19d4" + }, + { + "id": "2cf19e5998f946c084069a90b86cad3b" + }, + { + "id": "f5bf85a20abb40d3b781b55fc5ce9f66" + }, + { + "id": "218c4e7c031049cf8e40caa3ef4d03a5" + }, + { + "id": "a087419e91c74230abfb085080ae9c19" + }, + { + "id": "c03c5949ce35458d871675a72436337f" + }, + { + "id": "4605bddda00745439785211508887438" + }, + { + "id": "d047dfa72f104317a4c1c0738089f3ff" + }, + { + "id": "12dec4196bb143a1aefa79d446a748f6" + }, + { + "id": "b339d5fef74349958e739a5f3d7adbd9" + }, + { + "id": "e76f303b7fb641a2b5dec35240758f37" + }, + { + "id": "7a6a2d268f7a4f43936e9d63959b50f4" + }, + { + "id": "6a93924d3a9342568f4e36545ac83e72" + }, + { + "id": "6cf5711da68d491e9595988d01824176" + }, + { + "id": "eaa61dddceac4f74a015cd031e4de67a" + }, + { + "id": "059c6343bd5840f4b1b2325ff37ed4a0" + }, + { + "id": "74c896e331cf459a91549812a39057cd" + }, + { + "id": "9e9fc47587634a1d9b023a391fcae693" + }, + { + "id": "629697cab0d844a985ed3c1df7f39678" + }, + { + "id": "c93cf43111fb474ba0320e103f2aa01b" + }, + { + "id": "de14df3fb3f54ab88707902d3c120396" + }, + { + "id": "ebd09c008e884480b7162148fe0d1cf0" + }, + { + "id": "52c26a3da62741cdba8754a3d78d864a" + }, + { + "id": "4a819c8efa8145aca95fd7f10edf2a14" + }, + { + "id": "043508ba85cd4a0eb37a6d3e5d4a5d14" + }, + { + "id": "9cf670af0d5547f1af4ba354a41c7065" + }, + { + "id": "5bcb91f5fee34bec8df82e051e6eee87" + }, + { + "id": "7326e94d6d5a4e0a8d2017fbd1f333c6" + }, + { + "id": "cc319ec003cb41978b34756b47b9780c" + }, + { + "id": "7cc0e48d6b9a4e5286b7bf50c81e8d9b" + }, + { + "id": "255522dc506241a18970f20c859f4431" + }, + { + "id": "c9416e5462f245f19697e82bcb9f0b02" + }, + { + "id": "e5fa40d54ba04726b5a9883c5af26081" + }, + { + "id": "292d47a397f748a0a35a81618994b95b" + }, + { + "id": "a108b3fd35a4478490e0d86e65fda61f" + }, + { + "id": "70006ef9fefc4702911bfd3e8c31ebb9" + }, + { + "id": "3d6d34d50ff1405d9bba93c58ec512cf" + }, + { + "id": "a8a3a1697d9d4157b3f1d41bd510a70d" + }, + { + "id": "0cbceee80bee448c8c5d40f39d12d434" + }, + { + "id": "f3b7ae7f137f4b5291ec44919a269874" + }, + { + "id": "2e3b022cdde049439745ddea0bb64dd7" + }, + { + "id": "3deaa9f97883456d9904db5281da8ddd" + }, + { + "id": "8b10c96b05fe4a969f451ca45dea9e67" + }, + { + "id": "5e9c599e2f874b5baac79b49174f52ca" + }, + { + "id": "145a9ce734fd4d328a4be6f9deec64b9" + }, + { + "id": "e521910199f74b7192684bea87fa32d8" + }, + { + "id": "2402a4574a0944ccb31b2fa5af4d0b3e" + }, + { + "id": "68b20a57f30d4561b7f552c22be69343" + }, + { + "id": "af97973283df4d8eb059e58bf9b44cee" + }, + { + "id": "2b6038b0f9a14a089f3df83666f3a7e0" + }, + { + "id": "3cc2460d92a54e71ac0fc71b973918f4" + }, + { + "id": "77615b944ac746b5b2e019713abe5eb5" + }, + { + "id": "09d5b7de3015405c83f27e785ff9e834" + }, + { + "id": "c062204c69b94a59b3fb9026b1490616" + }, + { + "id": "4108ae63026e4db5b233293c88731044" + }, + { + "id": "fe759df9dc274228a80ac6eb905c2748" + }, + { + "id": "82bfb03bcd88419eb44235581240a2d5" + }, + { + "id": "e1db7097131542138d2b1cfc8012b5d6" + }, + { + "id": "3addd7957eac46eca1cada2999b225de" + }, + { + "id": "672c19cc7c9046caa525b2eb9222b63c" + }, + { + "id": "bb2fcca2557e4a9fb39b466710542338" + }, + { + "id": "40a3c8c8f1f4455fa159207cc1231e59" + }, + { + "id": "4fa7c2c886ce4f9790d8a402c9694dcd" + }, + { + "id": "8fdaafb1da1d49f0abc9772185c545d4" + }, + { + "id": "e72e6a8ac4e14372991a131bd3aec1de" + }, + { + "id": "1aab59c3467d486484e0a964728d6fe2" + }, + { + "id": "3be48ee90f3d4a769cd4213a5fd10ca9" + }, + { + "id": "10645748088b4ab0a421f05e992aee53" + }, + { + "id": "9addc8cd4aaa4ec995e7d66e314640a4" + }, + { + "id": "e6e6fdb13a5d482db07eef5e5c02d4f0" + }, + { + "id": "8a48a3e0e4d543fc90493445b3b50f5d" + }, + { + "id": "918c7ca9502243d6b410aab5e3258bfb" + }, + { + "id": "5b85faf48c5445c795c56d06d072af59" + }, + { + "id": "dedfb6cc72b44d74a2adebfd9bbbdf8e" + }, + { + "id": "857b8456009640359d2c10e4d19abd19" + }, + { + "id": "32f2ca6f0f214d8a9bda87a315d57b91" + }, + { + "id": "23a6c1fda8fc4c5fbb3b7740bee62ead" + }, + { + "id": "593f7e68ece24d0d93e4743547f84227" + }, + { + "id": "1ef613f6c25b4d43b8eb623746066663" + }, + { + "id": "c9cf94088d08447195c9ec8ee3e669f5" + }, + { + "id": "42ddac486eed46c9883ba0550cf719b9" + }, + { + "id": "1e94d21f33884494ab304834955241ea" + }, + { + "id": "53dabd2e810a44e09bf78a3d14a41811" + }, + { + "id": "aa7ef353a8c04cb18d00dd74ea5dc4f4" + }, + { + "id": "f6180c31a1184b27a7ec86663ff0eaca" + }, + { + "id": "cbbe39cda03e4f958df5f7f10b458c70" + }, + { + "id": "e3c89907500f4580ad3bcdb4062661e5" + }, + { + "id": "af49be3e777748e194c3b811d02bd69b" + }, + { + "id": "189bf0be3c9a44a39b977d5ee38375f6" + }, + { + "id": "b55ad2773d2b4de9971e7760cc36b8d0" + }, + { + "id": "8b0d46cd4a314123843ca2e8769d9b51" + }, + { + "id": "4ed4090ba9cd449a95f74e9d19aff030" + }, + { + "id": "8d372d89ff964c5a81828bebea3b72b7" + }, + { + "id": "8b649492cc804c6badfd043b2122ccba" + }, + { + "id": "d30dbfb5a24b4a848df9d79b5775f6ef" + }, + { + "id": "0036f2df628845669edefeb70bfe672d" + }, + { + "id": "39141e8c2276481cbec3aeec69da73a8" + }, + { + "id": "e921b02da68c4f3d86a7834ef62a909d" + }, + { + "id": "5fb265572a0349ceab2a01dce4db25ae" + }, + { + "id": "db51999e46d34d20ac3b6cc07de3ea48" + }, + { + "id": "7f67766516a34b9b8cf9480ae031e7c0" + }, + { + "id": "1b7abcf2e5334a52a8a6d454a9a7a422" + }, + { + "id": "c64e0787ce4545a487b103d88994afcd" + }, + { + "id": "b24e907a56894606aa1a65db91d5a0f0" + }, + { + "id": "9f047b722c954616a87f8a75fb05ea1e" + }, + { + "id": "268bdc9af5244092ac50e46d0e58da14" + }, + { + "id": "cadd1ac7cfc54dea8196aa83ea5f8a77" + }, + { + "id": "3fbfce94e0414297a6571b37a4c373a9" + }, + { + "id": "c5f9976c9718450bb1b7f75387b0460f" + }, + { + "id": "df1005681f55423a8bfca735b00db270" + }, + { + "id": "30befaa3a1844891a4fe5ea61ac440a0" + }, + { + "id": "ba4b4222d75e4579a1222714226d3159" + }, + { + "id": "39cb7d676f7049ed976f8986a41c27bb" + }, + { + "id": "8ed17c48fb864ff6a4bb5ba04055c1f7" + }, + { + "id": "7b50344caff24795bd6141f952e6eeeb" + }, + { + "id": "1f34e8356f244ec090c04c17c6bd2461" + }, + { + "id": "1e5097d79888462caaa39ba4db594a75" + }, + { + "id": "dcd3432c065c42e6aa0889fe00ce1551" + }, + { + "id": "0bed2137c44d4075bed61696b3a43f86" + }, + { + "id": "021af5c97c3e4b858d38d473b460e193" + }, + { + "id": "d025cc670b164233b35937e971cbe2b9" + }, + { + "id": "29c4441eb3de4e4081e7b021b7b21ae8" + }, + { + "id": "25c3b674680241ec8ece39507edecf5e" + }, + { + "id": "c32dc525fe3b476583cd61345d7cdab1" + }, + { + "id": "daacd026f4c74f05ad54a81156a73fc7" + }, + { + "id": "8fd31d7bf9f34ee18483bde98c3a7ab2" + }, + { + "id": "37f57f0d6bec4ca8a98ebf3674181eab" + }, + { + "id": "0bf30b86754b402e8edb15e06deea6d7" + }, + { + "id": "26f0839b53df4fde898bd35ad64d1b8e" + }, + { + "id": "9c43041246554b0f9371e45cbaf1b9bf" + }, + { + "id": "3e7b755e768b446cb6ce765d1bb23e7e" + }, + { + "id": "de04866a2e794178848e01632d12cc37" + }, + { + "id": "7472c5a21123400596962aec79a5e9b3" + }, + { + "id": "8067a787d569470497c0cb93213f7e07" + }, + { + "id": "8c3bc713dc724f5f907694e2d2583526" + }, + { + "id": "4e0652d328f640b6b8a36f47d508a05a" + }, + { + "id": "f4eb6ed857034ddb8caede4a50126916" + }, + { + "id": "3a635690d173489fbb5da0bbc4e67f0a" + }, + { + "id": "8e29a1330c7444029122dcabc01d4c29" + }, + { + "id": "33c1e8acc17f476bbaea3ced919f7261" + }, + { + "id": "00bae619959f47cb9b81cf3bacba0a38" + }, + { + "id": "1a4c64b7e55f49198320442e20da31a7" + }, + { + "id": "fbb328204c534e5e81a03f43009eb3ee" + }, + { + "id": "1b6c95d31f6245f5a8817dc8e5129c3e" + }, + { + "id": "aa09b8584da945bb8e799bfd52f71c8f" + }, + { + "id": "fe06a70fb7bb4f6fa665e96fe6617b56" + }, + { + "id": "4c425dc6992c413d9c1521f758c0d85a" + }, + { + "id": "39505c228a0d42ee86d84683483be26e" + }, + { + "id": "b54c0dde6f774d9cab7c34d47de7ac72" + }, + { + "id": "9c831d5f63eb406aa414bc8cbcc65702" + }, + { + "id": "d848ec91a45a47e9ad5b7951b9fa328c" + }, + { + "id": "a9091f1b92fe4f988f7dd362f226f24a" + }, + { + "id": "e22ffc29e8624e9d899f6b2066b22cca" + }, + { + "id": "565fbcb59be4452c98e09c3c0f5b5ab6" + }, + { + "id": "39ded4807d8649ac8148c560cdc9be59" + }, + { + "id": "41ba06c7459e468c8877cfd21a65a845" + }, + { + "id": "d08c12ce91114920b1792de8c4d55f66" + }, + { + "id": "18bbeafcf1a345c297c73e0043e4c884" + }, + { + "id": "353217467129452597e5b903092164e5" + }, + { + "id": "78bd57d4f88e45678b12a7f201c08b91" + }, + { + "id": "4accac8592a54df1b1b36fb49bf1b4df" + }, + { + "id": "02d096413cc84c3d8ea71b4740db3c83" + }, + { + "id": "7e3cea5135b34efab66c1ea8c4610361" + }, + { + "id": "1728772ad1b545e48f1033a908075f50" + }, + { + "id": "b6b16e7c963241d3819a28cd4d54df69" + }, + { + "id": "97ea770391534ffd9d71beb48d620a2d" + }, + { + "id": "23e28bc8e6c0466c9f1c7427b37b8d8f" + }, + { + "id": "986c1a379bfa4fe08dd2453eace5f752" + }, + { + "id": "efe119e8c6a64b8ab3377201e682c5ce" + }, + { + "id": "88e4101e4cc043c1a1790407124d6d2b" + }, + { + "id": "5c3f466eaa8a4032bdf797970e57f7cf" + }, + { + "id": "32c460e50841419e86a4ff76b28d3ac6" + }, + { + "id": "cf035f229bb84ceb8116c26d87a74299" + }, + { + "id": "1bcbb6bc51104330894359a3aa0ba675" + }, + { + "id": "2b73d6c930d24ff99e0a8d7caefde3fe" + }, + { + "id": "9ab071c46a9f49729bc7ced0b8143032" + }, + { + "id": "7757b82b4af04f12a3bea7655cae0e08" + }, + { + "id": "c55531425e38481c87dad17bb5a7eb61" + }, + { + "id": "da85bfd141ae4c97b03144145067dbc7" + }, + { + "id": "3ed3ed1a2ed24826998f5a377a0eebe5" + }, + { + "id": "5ada43ba21394ff58c8875db16c2fb4a" + }, + { + "id": "a7ebc31e16924865b483b408fa340abb" + }, + { + "id": "98e248b0cd0b4fa483949dbe09a7ceaa" + }, + { + "id": "592684577f2f4d0185be7aff295266f9" + }, + { + "id": "059fe45574ce48e3becc3b230e4546cb" + }, + { + "id": "bfba2cb68c9b4db4a71db7b1ca1daf1f" + }, + { + "id": "58b49f67d6ca40e19df7fe3f9410d4e6" + }, + { + "id": "d0a7c4d69828416da83a28c969e1d850" + }, + { + "id": "de54b5a1fe6947f993d16474d14f760f" + }, + { + "id": "8f88cd07ca714a469314bfd0dfed46db" + }, + { + "id": "880c4f8fe57445e281ddf15e3e47eecd" + }, + { + "id": "cb15bedf505045608c84edea71d6d658" + }, + { + "id": "2b32a64c14dd45fa88d3022bafb66916" + }, + { + "id": "7ef04f636a284045bab6c1dca3490190" + }, + { + "id": "996d3c20195f43da98f0b5aba98c955f" + }, + { + "id": "6423c6aff9d342ba9b280cfe66dcb5ff" + }, + { + "id": "45224e0c3ca4482992e8be46e0a67f64" + }, + { + "id": "8e6cdba3e47d4be38094c354cc9b7529" + }, + { + "id": "bf13df65693e41a58ffe9668a2100305" + }, + { + "id": "6d1f497464e8443a84562d229749027e" + }, + { + "id": "69cff2fb4eaf4a54af589d23d5f5e851" + }, + { + "id": "5af67300706c4224b111bdf8d3e498a2" + }, + { + "id": "0b0636e515c2405b8a03690ddb23484e" + }, + { + "id": "bc0eb04777904f4683de2a681201e3ee" + }, + { + "id": "04d3b44f40ad45289d0414990690a8ab" + }, + { + "id": "552af585b5004a0a9a56820d059be3e9" + }, + { + "id": "e9b52d99692b4cb39db22db4f6b79d7c" + }, + { + "id": "20a9fdc3a7e3452abc98ea834c7c8287" + }, + { + "id": "e0f0e3482b4d42218bf7856e590ea65c" + }, + { + "id": "3d229189bbfe401c928650ea4a6aa666" + }, + { + "id": "7d35fcb607d144308e59842d5fb0e6c7" + }, + { + "id": "ee02a0e1352b41e28836b95b6d62ae9c" + }, + { + "id": "1bf2e8f4324d47bf804194ae3eb748aa" + }, + { + "id": "69d1ac690138404b8a30e66b4af39e0b" + }, + { + "id": "f55cde33b3c34d17b152fe3f001b879c" + }, + { + "id": "a51dbb7a319b4cfe82ae927d6fa2a378" + }, + { + "id": "5209bb3e28874b6ab044698bd558f856" + }, + { + "id": "e225921262ae4952ae88f316adab17fa" + }, + { + "id": "da940922cae640058dcd8333a529a70f" + }, + { + "id": "1957afc2a46e439a8624866564299c77" + }, + { + "id": "86879fe499e54b40b24406c7feeda67c" + }, + { + "id": "dfec9961833848cbb53dfd96120c5c67" + }, + { + "id": "3cb9999f46154bb3b7d14b2ef04a3c7c" + }, + { + "id": "9ba40aba1544434b900db2758db2eded" + }, + { + "id": "a18fc294f8ea407880c30cf70c7ae290" + }, + { + "id": "5a9531bad7d049b6ad6290ad7ee0203e" + }, + { + "id": "8728ca39ca6c4a69ad00664e9ddfee0e" + }, + { + "id": "dd28ea5ffc9b49149a8b54366a25ad2f" + }, + { + "id": "7bd7a98633694059a03bb80a35cfacff" + }, + { + "id": "d3bc24bbf7a64cfcbd7817b9c5d6ca54" + }, + { + "id": "ad48db8d48f24570b35c8f4d990547d3" + }, + { + "id": "e868ea3304524abea45d9388a323e271" + }, + { + "id": "74b6042e0798491c81d482aec435e83b" + }, + { + "id": "384cb3de0b714c049e97f1659df6c38f" + }, + { + "id": "95b675e8b63a4f39b3bc13d3e926e270" + }, + { + "id": "db80db13a92c488e920b53bc3cdcf58a" + }, + { + "id": "1907544aa6ab4ba28951dc4b0389608f" + }, + { + "id": "544b9ff112ec462cbc2365d4534f2877" + }, + { + "id": "fc200c82aef548ddbf14823397d3defd" + }, + { + "id": "db406004734c46d28b5237cef711e2fb" + }, + { + "id": "0788e394a0d4417798eeb8c3f1939b85" + }, + { + "id": "a5fb869ba1534a1fbaf6f7d2daf8c592" + }, + { + "id": "e92a6a5f47b344e9956d2f242fd0494a" + }, + { + "id": "c1d0ebcfc30d4588bc539e1491bdfda2" + }, + { + "id": "99db7766aa6948779a5a50e754ff2eed" + }, + { + "id": "ece2f199f83f4c6f989911abdee7871a" + }, + { + "id": "e7249a74dc344c88b3b10adfedfd2b2c" + }, + { + "id": "1647038e04294ba293bafc1dff2754d7" + }, + { + "id": "1ab6058b99fb4263a34d46ab834a899d" + }, + { + "id": "dd95a0955ce14d69994a3834225ac485" + }, + { + "id": "9ad0ab836cd145df95b8967e286bf57b" + }, + { + "id": "1af943cca718482ea5e270215ce3ae98" + }, + { + "id": "593939638fbd4669a2f3343f64f1d414" + }, + { + "id": "4cccc854d8e54bd2ae5f20d2df617925" + }, + { + "id": "317d3ca2b3b049c38f0599105b631cb2" + }, + { + "id": "d33d8073aa6149dd9e1933b0fbdc9c0c" + }, + { + "id": "878d7d4e5b024fa394d5551a74bf74ce" + }, + { + "id": "b35426ec7d4a454889a577624d5f37e3" + }, + { + "id": "5c5525ecc07b45eeb0385e22ff04da2c" + }, + { + "id": "8049aa445ccf4e02848687895819b1a8" + }, + { + "id": "aa9245ea61e4446b99430c7e496e3399" + }, + { + "id": "b63bb97b6419445fb4f54984847ce532" + }, + { + "id": "9288303957c24d2f89bd30ee0ec1b099" + }, + { + "id": "4a8bc0ccc06749aa839a49d991e3983d" + }, + { + "id": "a125832f958b4b949d53701e7e7de41d" + }, + { + "id": "596acfb6acae4c0e917dce07c055168f" + }, + { + "id": "5f52a6cf393c4f48a852f94e4d035270" + }, + { + "id": "1f84b610c0904423ad9a44d86c9adf5b" + }, + { + "id": "dff449177b0444fb9fb3d3074c47dfdb" + }, + { + "id": "94aee49633d343d7868124a082fd4e79" + }, + { + "id": "b12327b5ebbe4ce48f26173d175358d2" + }, + { + "id": "bd472ffaee064326a2b7e7437785eea5" + }, + { + "id": "e4af38570ede4d60975fe42cbee53740" + }, + { + "id": "2d15ac4929a24bb9b28be47692ec0e31" + }, + { + "id": "47c9a3f482a64f4caf0fa08bb40111fe" + }, + { + "id": "0aa8fff3215c4ddd89526bc8e8b49950" + }, + { + "id": "dad2a74bc1cf453d9348235f071dfcea" + }, + { + "id": "4e2bde6cceaf4589a1827023bc70df00" + }, + { + "id": "07aaefcffd8e4569b12c80884ef0d24b" + }, + { + "id": "4e09d6c162414151b95d3c8fecc6ab73" + }, + { + "id": "c5cb2da2f1344ef4aeefbc76ba172a4f" + }, + { + "id": "010b367b8f5e49d3be4fea7439de0514" + }, + { + "id": "21cdade1fc4648a6955eb16c68b3c188" + }, + { + "id": "315ce0c9509e40e9ada49582c4c6dcb0" + }, + { + "id": "7cefa39a689a4ab89f749733f4573e14" + }, + { + "id": "4c62187235714ac8915d93d00b8b86ce" + }, + { + "id": "83a49886c0224008bb794d5c92caf0dd" + }, + { + "id": "c8d81dd0e7dc4bde9969db769a8be21b" + }, + { + "id": "fd6c948ba43c490c945177ed804fba30" + }, + { + "id": "b8186b46d7d54de5b0e9b942723f4fb1" + }, + { + "id": "07c95deaf6164d4380e8c5083b99307c" + }, + { + "id": "5537ded80516456582e58a06027b8c06" + }, + { + "id": "734c12b3c61f499eb696ab2e7e7521cd" + }, + { + "id": "7cbc24d809ce4acf974c2a67b475bcb2" + }, + { + "id": "6c4b1e619f9047f5be345a1934fb00a2" + }, + { + "id": "aae4ba6a4f184647b881304d15a0127b" + }, + { + "id": "18a2740593b247688ca64e5c347040a3" + }, + { + "id": "a79aeb2cd44f4d7db90713d72023d094" + }, + { + "id": "b3813b34d25e4dc592e344a9f226958e" + }, + { + "id": "3b44d20c36344e2cba9635596cfd47fe" + }, + { + "id": "e939d716042547b9a5c0e859a57537ce" + }, + { + "id": "e060051f352a46f78ec5444cab1399c7" + }, + { + "id": "0386b55f3628443484d92c35fb0bb200" + }, + { + "id": "30acb98661c6410aaf6c6ff5b2f6a3ea" + }, + { + "id": "5290f39935a047d9a6176f83190fdc75" + }, + { + "id": "9bb5aa2f3e9940eea6f4f04f471601c3" + }, + { + "id": "2a5524dbccb6469bb8be6357f68769b2" + }, + { + "id": "0a7cadd8c4a841f78f7679a402758028" + }, + { + "id": "741f6007092a4251bbb733f8251d6ff8" + }, + { + "id": "a3db01e563ff481288627ed7f9ed2110" + }, + { + "id": "68ad1d5549fa41d1a6473e25b8e32e95" + }, + { + "id": "03275e1352884f2cb6ff1e70031ea96b" + }, + { + "id": "2df9c599b03a4600b53839efc8c12d03" + }, + { + "id": "cf4dc2bacd2544738999a500551e695d" + }, + { + "id": "39eb0eee961849f980899754daa0e6fc" + }, + { + "id": "52adaf3083cf4e9d97bdd9fe1d9063cd" + }, + { + "id": "c97304fe63f44271b054e480071f3bda" + }, + { + "id": "6d183cab5f4946cc9693d1bcd464e438" + }, + { + "id": "f52e36d88d8141589b7f57be7141d2ce" + }, + { + "id": "ed9f9cea823345fc9cadc16d151a9bbd" + }, + { + "id": "89eb757291114ed8ac5bb0df63cff34b" + }, + { + "id": "0599cd7767b242f78cfb9bf6bb49f259" + }, + { + "id": "8ae40dd4f4db4ba3965d510abfdbfc53" + }, + { + "id": "b3db70bb7a1746eba2857bdcd77aec1e" + }, + { + "id": "37b7a09bb7ed4b7ca9471b847c3b89f7" + }, + { + "id": "399282083ee8402eb87ace6200235232" + }, + { + "id": "17aa9d61ef8545cab274be711be12545" + }, + { + "id": "d9bf831ca84a4d778e20e5dfa93c4bdd" + }, + { + "id": "89412bf5de8446cabc4d5a96cbf7be68" + }, + { + "id": "5f6158c674304dc48612c1c90777f6ef" + }, + { + "id": "d274307a500d4d8c8cbf30faf5589cf0" + }, + { + "id": "f5ee56d993c147c4b84fe7a2e820cafe" + }, + { + "id": "ad1053172799410dbbba6e41ff9304a0" + }, + { + "id": "2760e2dfcb7d48308c0e7ff3d66adc03" + }, + { + "id": "55de755622204878942357d6f0752184" + }, + { + "id": "96b182a87733443d900d573aa6a3daf5" + }, + { + "id": "e5a2400ddabe4751a3a39adfb948c411" + }, + { + "id": "5b6925fbab6a4b369f69c19075efb029" + }, + { + "id": "756276d1375f4a4e9b16282e8b915b51" + }, + { + "id": "a7674332b87849f4b6cbc64be28b233b" + }, + { + "id": "ea6c17bb22ba4ac08420f2395e28ed79" + }, + { + "id": "e2801d2a3baa454f90ef2f954c07989c" + }, + { + "id": "4f393e1af76a4211855c954e7a48910c" + }, + { + "id": "2711f4e089ee430c8cbeb3498bf95473" + }, + { + "id": "9625aec4508b46ab8ff49452ae77c08b" + }, + { + "id": "b57a65e684504d9db9a6878292c0313e" + }, + { + "id": "7be9991d6f6947e4acc72ea8644b82f5" + }, + { + "id": "a0dbc05648a74890a966350c81704622" + }, + { + "id": "9e8622f9a9184715982fd8dd1c6e3ba0" + }, + { + "id": "67e7f225634c4cba8a31bb6fbc51a107" + }, + { + "id": "f285b18402cc4d1ea3cdbab6ece293df" + }, + { + "id": "3b3c239c629640659e5eba502a817ced" + }, + { + "id": "da6a69045302434d8aec0c0ddbee1a49" + }, + { + "id": "24faa13b2b7748f4babea2d6320d458e" + }, + { + "id": "730d1b61b948484394b9c56703bdb2b3" + }, + { + "id": "da2feb04ad764ada9fb88189716f1142" + }, + { + "id": "649fdc580ea74665a873f6a0e29907a5" + }, + { + "id": "7c69a9380360429d91cf729201196b16" + }, + { + "id": "ed3126e9b1c046a88952849cde65acd2" + }, + { + "id": "eecd780002e6481f93ab585f1ba80ab0" + }, + { + "id": "21df075c378849ab9411a0ffaddb3697" + }, + { + "id": "aa5ccdb669c54275a3ac03c724a4f2e9" + }, + { + "id": "2d909b42ef974963b8918ab1c6771b84" + }, + { + "id": "15d8c4fb9ed1486f8e6d80ffcd7d0948" + }, + { + "id": "5bd2e40ac0a54e1db12f2b37f3a86f6c" + }, + { + "id": "7e6a0f89411346e2ad88dc04a2be77e3" + }, + { + "id": "18f57112ccf54db680526fe8d04f0a0f" + }, + { + "id": "7497bd11227f4bd8b027c5f737c3f23c" + }, + { + "id": "142c46ffc40d484d9a9805811fa8d989" + }, + { + "id": "fa3a3d392e1f435f996add8371a798bf" + }, + { + "id": "380905a8be3140d2b8c53427fe923b73" + }, + { + "id": "511231befba84f3baaf198cd8b32f541" + }, + { + "id": "f359127d25b646d6b1cf595ca1f44c62" + }, + { + "id": "dbcadf420ab94399b8137e3bab4eb2c8" + }, + { + "id": "09a3c29bd469417c9ee3cec55c1d4b73" + }, + { + "id": "87d921a79a624189b7fdbce4ba1c3409" + }, + { + "id": "8e0843e73a22433c845d554d79bac4ab" + }, + { + "id": "b34b82628e4b4361846043c8dcfb8aac" + }, + { + "id": "a0d9bcba64594ec0aaeb2d1cad1a8b59" + }, + { + "id": "c421ee82b0594e76b762cd052e3dfa6f" + }, + { + "id": "e04f70f907f0419ba627c6889342eeb1" + }, + { + "id": "579ae7a5dde34d1f8d20f3c5b644f5f4" + }, + { + "id": "c35fa3ed34bf4b0e8116f10d1913d75c" + }, + { + "id": "dac2c24d81ce4899823c9d4fd9f47db9" + }, + { + "id": "dc0df73d4fe3474cabd047b65c70d911" + }, + { + "id": "4fdd00881ae641ba9535e22d64c3a0ff" + }, + { + "id": "49f06cb45863458e82c2cc98d6496fe6" + }, + { + "id": "08377c21beb04e8db4a3a2687dba7c33" + }, + { + "id": "16f4d1a28f264c15b4c93460cf2d985a" + }, + { + "id": "93269a66ef94416f8fa08257efc9d30e" + }, + { + "id": "85a8040e8294493b8d038621e6a04930" + }, + { + "id": "ff2942e7dcec4b54bb5b4d1c35678ae0" + }, + { + "id": "a534d740d99c4f4ebaf7745907603113" + }, + { + "id": "dc15fa04af0040f1b7999769ac1da899" + }, + { + "id": "e4c43c28c96f40a893765744915965da" + }, + { + "id": "f257a2c9688441719bc8b80bb03ed3a1" + }, + { + "id": "2912f5399c2f4fb6a7c520306d8caf3a" + }, + { + "id": "ceaa70ad97bc40cabc6b6fd99e37b0e4" + }, + { + "id": "9d65ae3eff6940dd88b0b66996c9360d" + }, + { + "id": "777b196aeee34792b1c79c7d022ca04a" + }, + { + "id": "dcd5ede5eac44e12b8da2c1f88d05596" + }, + { + "id": "87662c5037b04f44a5e20fcc7c925f98" + }, + { + "id": "979feb65f2b643589a974182664cbeb6" + }, + { + "id": "b92a6b9037e74c47a219a30a0a813683" + }, + { + "id": "fa1600f545b14703b72b253b24ba1673" + }, + { + "id": "4a641c9367f84b589e5be486a01c9078" + }, + { + "id": "5ae3072b6bbd40cab41d2d0782aa9bef" + }, + { + "id": "bf2db66e80ff469e979ac4e9f6b255c4" + }, + { + "id": "4412a02b02114936b8bb7a0f366770e2" + }, + { + "id": "d104edea8ff5470a98dc6f81520e39e1" + }, + { + "id": "b956696a520f4490ba457c175cf49b4f" + }, + { + "id": "e5a37e657d344835a95788179c1266eb" + }, + { + "id": "1e57515dad4841e3b0cca4d623cb78ff" + }, + { + "id": "8bf26a55b8e24999a71cec5c22dd6003" + }, + { + "id": "75a1657798b3412e83c28343a93c6a2e" + }, + { + "id": "3201d8f450dd47debc62e21b5fb3c490" + }, + { + "id": "aac65b2641484039b5af31a876f03f10" + }, + { + "id": "11b2ebf9061545afa597e33b12888dd0" + }, + { + "id": "47e0542a24954b67a5d540dc56c1a80d" + }, + { + "id": "5c26e2dde8e2465ab44fc9bbcc0aefb5" + }, + { + "id": "4419b5d125bd4eeea134f2cc05561f0d" + }, + { + "id": "6b8be20329f043138222e3bef58cd516" + }, + { + "id": "15e2f1ce0d4a4cfc80335a7d931fa6c0" + }, + { + "id": "6bc9eb6be6ef40488546f5c5a2656b0b" + }, + { + "id": "f9eb03ae442c4afa9ea03d58106ec819" + }, + { + "id": "3d47899ab972454cb664fa33109370c2" + }, + { + "id": "b7daed3b13b24c2c900b3db34091f5af" + }, + { + "id": "b54b3d6f34644ea2aaf37de39682368b" + }, + { + "id": "de08797cda7e4a928bebeb62b07788ec" + }, + { + "id": "3829f770d336410dbfb6091efdbcad94" + }, + { + "id": "5135571c42dd41f49eb9b6df45fc00db" + }, + { + "id": "592401ec18a74311b063864e6bc63c81" + }, + { + "id": "94cdf72aba2b4b06a1c9ce934cd407ca" + }, + { + "id": "f1872aa085684205b01c8b044be9c405" + }, + { + "id": "54e7aaba9cbb454a86af6d916a71109b" + }, + { + "id": "2e44b7a3015a4079a8460c2c84963a6b" + }, + { + "id": "4068384dadbf49afb3e663a099fbd619" + }, + { + "id": "25c0e5f8a53f4bafae2db6feb35ae667" + }, + { + "id": "4cfac23b2d114f91a2ecebc36feadcb5" + }, + { + "id": "9b0f54e4d4b24faca22ba89f1275b675" + }, + { + "id": "3c2897b03e0844a19fd5b3d49e99e3a5" + }, + { + "id": "4e040c91a514411f869042bd1bb74701" + }, + { + "id": "fa35104992c04600b07ec126b0d8f794" + }, + { + "id": "e72957b206954563842b8921bb239815" + }, + { + "id": "52842cb313f445e399029f5e78bc5cd1" + }, + { + "id": "8b933c2dc3ae468b88d3a744bd052757" + }, + { + "id": "36fda1c2c2504199bed202f61b06ecf9" + }, + { + "id": "51d16b79273d4676bd38079610968af0" + }, + { + "id": "a5c6e8c4c78449cea0fd71f7d908d88a" + }, + { + "id": "77ceea3648764a8e8561ffe69f284d9c" + }, + { + "id": "5ee940c2357946a6ba68ea0713517bca" + }, + { + "id": "7b6242e63e9042fbbe2e72a7414addb7" + }, + { + "id": "d11365f5d8ba4aee988258f9340a7009" + }, + { + "id": "0574d25fac6244c3885304a2112a0c15" + }, + { + "id": "b52a5d5925504e7a9b0eb7b9a2e808fa" + }, + { + "id": "2372fcaa483d4384a68645f3fa9ca173" + }, + { + "id": "92d41fc8d92e4b82becee6215520d0f6" + }, + { + "id": "39a9394a0123427ea52b07c9ec6c3093" + }, + { + "id": "0cdf302a5c1c4fe0ac9a499470765108" + }, + { + "id": "bfb38a93888b4c0f8c6b83c8d127eb2a" + }, + { + "id": "7614ea4125f34dcba0d17cfe071d42d3" + }, + { + "id": "95757ee9efaa497f90b0c6d79640865d" + }, + { + "id": "4af582d60f3e463caa1c8b1669e931a4" + }, + { + "id": "858e95fc230148ba8bb554e35b25cbe5" + }, + { + "id": "177ebf6c9a854c78a73eb21e23402fef" + }, + { + "id": "6241e981d4b74d3282534ad5972ff3f2" + }, + { + "id": "b0f37c93bb7b440fba1eb339b05c31a1" + }, + { + "id": "e006f1243e0e486e8bab439af1d1ffd0" + }, + { + "id": "51c5b397478b4a7bb93232e7229a6a2d" + }, + { + "id": "e7066c6fbdb1495c8f57b0a40040f837" + }, + { + "id": "d2aae288da2240479ac8fd64380e8181" + }, + { + "id": "b8d6403bc2cd48cda7413a3f5a4ba723" + }, + { + "id": "ea5e1d5687754fc19c7e5e5ec47b0ccb" + }, + { + "id": "c92ffc250da04dadbd4091e7caff083d" + }, + { + "id": "847e5a2123aa4430aad88f8869cd3005" + }, + { + "id": "938a1a20b96245e6a0c687bf010a20c3" + }, + { + "id": "662149e9d00747989aabcdf269c496a3" + }, + { + "id": "48e2633ad90147a1bdc751f7387034eb" + }, + { + "id": "8b68dfd9a467473897fa9d1434418267" + }, + { + "id": "1489805b557742a0972dd1add3b3f3a8" + }, + { + "id": "092f0450ae72424e80d1b7baca5d2eab" + }, + { + "id": "bb525e3475144796acac6b4db0591ec4" + }, + { + "id": "fc3c34a4613c447dbfed5a531b4a9a15" + }, + { + "id": "89ca79133c4643ddbb782bf195c27661" + }, + { + "id": "c7b1b174463847aebbfd540d05759332" + }, + { + "id": "4aad23ed67974f77ab643d52928b9f13" + }, + { + "id": "61eda64250e44e7da58229538c783ccc" + }, + { + "id": "d3135a3432a24c1b8d22ef8e1753800f" + }, + { + "id": "b2991dbc96b84be3a3ea13a765cb0289" + }, + { + "id": "47b66a02f30d4ca89ab37f7bfb96e426" + }, + { + "id": "1b8886a267e04b48ab393f0703af9532" + }, + { + "id": "afaf746196804a32aaf822eb592528cc" + }, + { + "id": "a31a3da4cb2f47ab81fd576ce09e56df" + }, + { + "id": "df0a8252d982464597f62bd806caf39c" + }, + { + "id": "df5c7bb060c442689a7a7fccc7079811" + }, + { + "id": "a6652c364b6e4e8db83b3e26b2dd1c1f" + }, + { + "id": "f09586515a2044019052af9bf87096cf" + }, + { + "id": "4fc6ad573cba408e933068e8d51cc444" + }, + { + "id": "44777e2c7f8c42b7a2252b71c64aa422" + }, + { + "id": "773848a7e56340a29b03a44794814582" + }, + { + "id": "39f0fe99cde247619b4057d0084ba9e1" + }, + { + "id": "a2137a7b8c7c426ba4b49bae0d61521e" + }, + { + "id": "c548b9c001f4403b88be2af3d566ef35" + }, + { + "id": "197ee5401f554bcfbfce4dc751ec3343" + }, + { + "id": "e1a0e75cc5cf409a9fc1488b071b2eb6" + }, + { + "id": "c9cf7304b2c84aebb6f9af8bfc2dac9d" + }, + { + "id": "47a7a24cbbeb4cc980fc15aaedb03bf3" + }, + { + "id": "c8b56dbe7602481199557313bf404b9f" + }, + { + "id": "bbb1f8afa57049488402d1d02e211878" + }, + { + "id": "05bb48c9c9094f2f9374148f2756fd4f" + }, + { + "id": "7b8f504bff1240bf983c7f28c50da9e4" + }, + { + "id": "ffa2d3fd7c8641d5b88780ab0c980057" + }, + { + "id": "82959bc544334b70819123eb60817b19" + }, + { + "id": "6894daf9cdfe4d2081847134f34ea3fa" + }, + { + "id": "3e27e48cd7ed4f828b1506c626917c3f" + }, + { + "id": "59d7d4ec3377454b9a5d8a63c4d67a29" + }, + { + "id": "f4c1aaec12fa4c61b379f1a9272d1ff4" + }, + { + "id": "04d1b7f7695c403497dec7423a68a36c" + }, + { + "id": "3a976786b4fd4a9a9da2119af1c28d36" + }, + { + "id": "1dccbb3c4bc04d87a2dea2eb10cb996f" + }, + { + "id": "5a1069e2fa7342ee80f28ed79ef14db9" + }, + { + "id": "aeb2c8943702445aac674889d64af7a0" + }, + { + "id": "4fa03c33d4d2467096cb5159d9b32d0e" + }, + { + "id": "6e1f69f84d5c442a89d09d578cd00ff7" + }, + { + "id": "89678f2e2fcf40eb8bf563880994add2" + }, + { + "id": "f84af13b9f8f4f3d95607952fb694e5a" + }, + { + "id": "5cc9d4417dcb47ff90663dcc8546bcf2" + }, + { + "id": "7d6bd0d727e54335be7c1c69886fc82f" + }, + { + "id": "6bff5f3522d84b2183e903e3157112c8" + }, + { + "id": "2b49d26a7efb4a2c8159efbe3439e12b" + }, + { + "id": "67848315b1fe4feabe12fd05d9de288a" + }, + { + "id": "2bda147f9ebb430a8968ff5aae713585" + }, + { + "id": "f1123b7bd1e94b98b4a1022070fb8479" + }, + { + "id": "b2890ea2eda540d1b74d36f4ef13ec76" + }, + { + "id": "21938e691b2a412ebb7a3548209ecbcb" + }, + { + "id": "74e7110ca4f149b1b1359cc75acd69e7" + }, + { + "id": "1a09c7ead5c7476cb80618a977ff8af3" + }, + { + "id": "a757b55a3f83439db423887bd8143f90" + }, + { + "id": "d8496e597ea843a386431d8baa798edb" + }, + { + "id": "37439a48301c4dcfa3a3ac80902c2e0b" + }, + { + "id": "152527eaf57e4c11991713d44c1dfa7f" + }, + { + "id": "ab94aa55c6314e9b8af69bdfac48dadf" + }, + { + "id": "c385599879b24c4abc06e53f98863d46" + }, + { + "id": "29320213cd1d46d38ff39332227412bf" + }, + { + "id": "1d24871af6db449099d45e17b6341dcd" + }, + { + "id": "8907263a3c0846469f4bba527c65c45c" + }, + { + "id": "fa215eac88e94be3ae508faecdf8e161" + }, + { + "id": "05c45056631d408ea58b9008026e33a2" + }, + { + "id": "e965f5490a7e48029a7c9dee649c6dcb" + }, + { + "id": "5c562c75aa9f465587b466f96f47c930" + }, + { + "id": "46d2aa3ada6f4a5d8a499e4a5c07385e" + }, + { + "id": "bd49e52726674480873c9f6528094f03" + }, + { + "id": "a75f12ade8544e1a89f2d48ccc815a0d" + }, + { + "id": "a5473315502f4e198965fc83e7c263b0" + }, + { + "id": "4aa8fc7557fa4930b5e2e78a896b1084" + }, + { + "id": "d8fc70c4924e4422859a7058f11a5f5a" + }, + { + "id": "276fe486a5ef4a809d2a36d07285a6f0" + }, + { + "id": "a064125fa3674c049e473e4d7c2afcd1" + }, + { + "id": "d8833f3292614d978fbe84fe928fb138" + }, + { + "id": "18a6e062325242e58fa8a5cb5f3d2a32" + }, + { + "id": "86acb5b661bd4724a5d404e5b17e6a46" + }, + { + "id": "113ad57d593f482e96e30af68b3aab9f" + }, + { + "id": "12b51ad292af470ebd99f1d043357ebc" + }, + { + "id": "1e7a668213c4433d8b7970a32e3c0af1" + }, + { + "id": "884f995169b3419b888c6ad6e2f6bcd8" + }, + { + "id": "80e6e7b117394115b0de70672b547307" + }, + { + "id": "abd5b61589364b07bbf7299c49e70e01" + }, + { + "id": "419b76eb0c8c45568814af128c318830" + }, + { + "id": "886f2801a4fc43b9adb3ab9f551555d4" + }, + { + "id": "64a22f0e29224f86b10c0fa0928fa96c" + }, + { + "id": "867f309535fc4b788320e6c2b23fd89e" + }, + { + "id": "d67c9156a9b04624910dea2f510f8047" + }, + { + "id": "7292cdeca2b949a6bef8c03a9901f5a4" + }, + { + "id": "efcc54e12eff4d83866b4f58c4b3df01" + }, + { + "id": "238903697f2f4d3da4cf0e24a2761db0" + }, + { + "id": "bd104461e8ca4017aaacd928f8ae5784" + }, + { + "id": "8e31e394ac6c45c4acfeda5424aceb64" + }, + { + "id": "3d01568b168a4f0bb899e8b1c2239944" + }, + { + "id": "5e838c1d51044e77ac345c32b2a8cfee" + }, + { + "id": "c128b8c91c024ab0a592dcda590a11f9" + }, + { + "id": "1a4e5bce9bc940918d6635479d944d6d" + }, + { + "id": "aa87cb64b6f54159927506f28ec62f4e" + }, + { + "id": "33117d6f9f5e412788148baef55f4b2c" + }, + { + "id": "3d4b3af120844bce9ee06e3fb56f2936" + }, + { + "id": "4abb0e9595604633a05a61c344d88b2d" + }, + { + "id": "af3c4b517336436681e03e57ce731b31" + }, + { + "id": "eb0fb0cca9684810ac6840dd9b8ebb91" + }, + { + "id": "c5b64798f0314be897a348c3196681ac" + }, + { + "id": "7aced3eb67c74eba971a2530ce81074c" + }, + { + "id": "e7071a0a197c471d8026449e74a9233e" + }, + { + "id": "35ee959204c1457da47d572c5ffd92a6" + }, + { + "id": "f2bce9e28cdd43e6a888bafd35d47f1a" + }, + { + "id": "759f644fc9e34509b7b3c7e588a157cd" + }, + { + "id": "273e54aff0a149e08852c43ca2c5ea89" + }, + { + "id": "a1fff358ccfa4bf2b53a911e082e4cd3" + }, + { + "id": "d1c87cfadcda4e4a9c21fd93fd267ad8" + }, + { + "id": "10b15b8fe4324bd48b481506d0925863" + }, + { + "id": "9e1a028c042a43a4aad803789594d801" + }, + { + "id": "61eddea900ce4ba4bf01cc9eb5b68736" + }, + { + "id": "67488f8f226d4d278ed359cea5b80cf9" + }, + { + "id": "972feb1557d445e098a9a0bc4c8080db" + }, + { + "id": "40a28f0fbc384b16a4ccccecfa95d96a" + }, + { + "id": "b643995b3c124c28b370cc483af30897" + }, + { + "id": "9615057cf59f4fa3a92da965e12c7a40" + }, + { + "id": "b5ed7cbabbbb41949a7965e7a98ccb49" + }, + { + "id": "8b090fa84d1744e9952401d6cacb09f6" + }, + { + "id": "d8eba1df0e5b45159283a1f457542c0f" + }, + { + "id": "78211f5b18e9465796b288ddda7768da" + }, + { + "id": "b1fe478a71c943459dcf55119d6f3f4f" + }, + { + "id": "4e876d2abc10451483f9275829e780a0" + }, + { + "id": "c159edd48871458095d1bf54d7a50a5f" + }, + { + "id": "140eb9010112494aad095739adb735d6" + }, + { + "id": "a09c6188fae74726aed2bbf66ca193ce" + }, + { + "id": "b9aef446e8ca412bbcb69b8f94249dea" + }, + { + "id": "e72677bbc791425387075a3a7188230f" + }, + { + "id": "499b5f088dc14ce0a44b5f4cf7118616" + }, + { + "id": "80f499f83e834cc99e31318d31aeaf47" + }, + { + "id": "74aa3ba7ab104fe8bbcc426c64520d0f" + }, + { + "id": "fd1a54d5872246719e28ba3fab3f5e8d" + }, + { + "id": "3725ccc6e524405f81722ceffead540a" + }, + { + "id": "84fa5a0c7f7f42fc846f79589a14112f" + }, + { + "id": "ae072e1d30fa4694b14ff98e301fd768" + }, + { + "id": "327c1b33798e4afda8686955e302afaa" + }, + { + "id": "337965b613194614a282322ac5ad6cec" + }, + { + "id": "8a67adbeab504bceadbcf5848e613469" + }, + { + "id": "bc098dd8220e474bb29845ce026fb599" + }, + { + "id": "2f702f4dd2574c02addd404d82953d15" + }, + { + "id": "6eea972e7c134cadb65132643a906ba1" + }, + { + "id": "068575aee7224d4ab849f9d138731fdc" + }, + { + "id": "84024891efbf49308fd2fe87c30e396a" + }, + { + "id": "a58acf29767e4b939a81581b60f3e215" + }, + { + "id": "eef234c3f1844fae80ba0c6820ef33a9" + }, + { + "id": "a0f4f67f76034e49a4153768c89364da" + }, + { + "id": "dca3a642962b4b119f7aa555978dd41e" + }, + { + "id": "ea8d2f98ab2b41329f96ffd259484819" + }, + { + "id": "9ec6f9f9af044b988a7d21f9d2b88a20" + }, + { + "id": "f9d719fa59344d7c9b23952b5fb79989" + }, + { + "id": "cd52f29813ed467fa6b62b10895844d8" + }, + { + "id": "1e08a80bcb084a368495349bfa155a1f" + }, + { + "id": "44cd5c74a4544123a44553bd5faaa219" + }, + { + "id": "a0dea12493ec4f3fad8587be85e5d0e0" + }, + { + "id": "de3cc412c77c49d2b1c90c72b3e2c4f1" + }, + { + "id": "1666f5c898ee4e7f83c8916ac0a1a369" + }, + { + "id": "58b3523293b74ff7ad6037676aac940d" + }, + { + "id": "e01ccc8683af45998149ed184e896067" + }, + { + "id": "550812d86ec842d18632765827acb71c" + }, + { + "id": "2352016546794050829f1b07f25f9090" + }, + { + "id": "fbd263a6701c48a8ba24cadff0542457" + }, + { + "id": "dcf0c2c053ad499faac8a13f71296557" + }, + { + "id": "8cec7fcf760149dfb56506296bb3dbbb" + }, + { + "id": "021a105fde2040e6878900394ac1fe98" + }, + { + "id": "19803226726c4e5592cc9584cc09c1ca" + }, + { + "id": "d3205343ca1d4166b6cd70984408127f" + }, + { + "id": "33c5b638c7c0474798042991babdb389" + }, + { + "id": "98e59cbf1c7047ecbf36b4b7382d3948" + }, + { + "id": "2d9a9efee3994717a1db7cf02e0f206d" + }, + { + "id": "e5edc854c80343fe8fd747bb7bf31459" + }, + { + "id": "791836977d0143509c7b8b00ccfae43d" + }, + { + "id": "4641f3042c1746abb5c9ac68570e7d59" + }, + { + "id": "265083cfc2bc481e8d8ed466bdb62543" + }, + { + "id": "3486bf0bc72d438b9627e2ec2b6ce4bb" + }, + { + "id": "330fb4fb30c4475f85872be34c1d2117" + }, + { + "id": "53846a14ec1e49918818c1bcdcddd918" + }, + { + "id": "a4b854f73b0644d1b72679fd808db834" + }, + { + "id": "acbe0bdbebb0454c9b5a96a537e7b414" + }, + { + "id": "d43c698a4a4444c8a51ca58fea216d22" + }, + { + "id": "73726187be7f46a0ab4e6a0a8e9ce810" + }, + { + "id": "87d2dfd838c94dd5b8462fa868edda8f" + }, + { + "id": "5291a269c9de4712b4900236eb4fe5d4" + }, + { + "id": "b43617fa1fc145e6809f24a7a652772c" + }, + { + "id": "1d6d2c9e18184226911b9d16a97f3530" + }, + { + "id": "0b43263922b84f66ab78b0df30dc751e" + }, + { + "id": "d702c769002043d08be0f9187d32ac74" + }, + { + "id": "e16de15cd4ce49ae9730d2d50de9f032" + }, + { + "id": "d79569103b7740f0a5684e2ec335e6b5" + }, + { + "id": "7a8912ce2dd54b09b9bb42f96dc073b4" + }, + { + "id": "5be6164a777b48f4bd10f228f27fd1fb" + }, + { + "id": "fbabb7185c374402801f5257f49b5309" + }, + { + "id": "ea747bee47074bf980f47568a81425b1" + }, + { + "id": "4b90f476ede9474aa7eb7e904961c163" + }, + { + "id": "4b6ab1f198474d779f28bddbb79937a2" + }, + { + "id": "bf17e4448a5a428593a1711cbc0c1feb" + }, + { + "id": "1d78dedb35da483aa19ae93ef984728c" + }, + { + "id": "4c66521b076449809cc45c46c834cebf" + }, + { + "id": "4c2c3e9c647a45dda53707d3dc4513a5" + }, + { + "id": "7f2457574fc74af697837b4a285a2d31" + }, + { + "id": "3f4bc6026bc340d7bf286106d2381976" + }, + { + "id": "24d7105838f64c03a776555096791036" + }, + { + "id": "bd838a345ffe4ec9bf4cf009757b432d" + }, + { + "id": "e2fa62b96fda4696b30620f521123361" + }, + { + "id": "ec6eb845b2034cc98fc4c430be032e7f" + }, + { + "id": "23a78c52899941588ac23167674cce6a" + }, + { + "id": "dba14eb45c5b4a02814ad91361e35a36" + }, + { + "id": "75b567bfce2d483788f98ecb7472f47c" + }, + { + "id": "90f714b5b60c45cdaaa5a32865335619" + }, + { + "id": "e13ba717f9e54416bb61bfb665510cdd" + }, + { + "id": "59ec0cf2e15944659c77b96d911ceab0" + }, + { + "id": "7b6534dc66d6458ebaac8ef00c98c7f5" + }, + { + "id": "5e8058a7b16d43f485517f81ee3b4a3a" + }, + { + "id": "d89950b1a4264e0fb3790eea755d2f71" + }, + { + "id": "8bc9e0ba6f914607960f73cd61bdd607" + }, + { + "id": "12ec5c93c19146fdbbeea0c8bbf650cd" + }, + { + "id": "f26047f9818a46b3a4e70a13f0ee2afe" + }, + { + "id": "ee7665e36cc34a67ba02ba0b1d8a7bcc" + }, + { + "id": "3d16c7b6ee554932b816c4da93f147f1" + }, + { + "id": "f1de564a1f2f4a6ca813f99b5b4e8bee" + }, + { + "id": "a51fee9868e746cfb2897294ccbf495b" + }, + { + "id": "208fecbd8c5449c1bd1b0177453cf9b7" + }, + { + "id": "1137531e97ba4657aa07c7a33e864378" + }, + { + "id": "06c9760491af4b67ab944157736a1d37" + }, + { + "id": "1ab28a192f6e4b14a0071a741c9787e1" + }, + { + "id": "d6bb04944a314e1eb2249f019cfac2f6" + }, + { + "id": "e8d79a2a8fe74e84bf6f81043f5e5641" + }, + { + "id": "959bab0d571f4f429d1c1923cb3855d7" + }, + { + "id": "7902980ac783461d8f4d50274667d644" + }, + { + "id": "9f7ab33db32d405c808a2a9479086e06" + }, + { + "id": "1db6e2ef1adf4e80bde4fb0a65b4a94c" + }, + { + "id": "3bc29910565f4107b297185e7313c776" + }, + { + "id": "4e1a0a534af94137bb8bcf832c41339f" + }, + { + "id": "c05d9ada766347b4972e10b6534785f1" + }, + { + "id": "6f5857f474d0451ea88b8d42218c5dc6" + }, + { + "id": "23a9f0f9823f4c6ba66163b1652a6b05" + }, + { + "id": "6fc666ad7c134435b8cf8fcf23f8125d" + }, + { + "id": "1c6179d329394b268eea8f552523b321" + }, + { + "id": "87e2674ce8d84e098f2f768e294d74e3" + }, + { + "id": "35ed2ae3d74d461996decc596476332d" + }, + { + "id": "ba5604cc15714549b1887dda4c79ee1e" + }, + { + "id": "235ada881f2348ed80b6790a9cdf6415" + }, + { + "id": "a487a81f868c4dae9fabcf38497396d1" + }, + { + "id": "9d66eb57ab8a460381f2299472833c31" + }, + { + "id": "682e265ba77543ec884da14042e3ac90" + }, + { + "id": "ba71cadcf9904c98827906d1c71b9c2d" + }, + { + "id": "aa26349086304052b78f80d0bbbffd62" + }, + { + "id": "249f4c256b544aeabfb919547d7f5969" + }, + { + "id": "4e5f40aa1f284f6685174a757e1931ad" + }, + { + "id": "ce8c2511f6a2487995e557418f159efb" + }, + { + "id": "4180c44eaf304a9a956122b8e8523ac6" + }, + { + "id": "5b4ee5101c9847c68cacae79a5ddaf0a" + }, + { + "id": "550a165ae0c647848de379bee1e01e93" + }, + { + "id": "43a169819bd049dab9e207b0422b733f" + }, + { + "id": "135b71ec2d6f4f718913298108dcd76b" + }, + { + "id": "0d267ecf2d3e4b4499a98671ae7e31ff" + }, + { + "id": "1d4b08ca62204b51b18519d5f7ef66f6" + }, + { + "id": "8ead639f93fc46acb2b2761fc7689641" + }, + { + "id": "460bc426e17b4230a01b3c46c9de3b1c" + }, + { + "id": "9a291c44fa1a4d78af5ffdeca1928abd" + }, + { + "id": "1d2807eaaa88446189a53b419b218704" + }, + { + "id": "5eaf2421079b4899a10cbee96d2cee23" + }, + { + "id": "a75a2c793da5402a830e01de7f154cc0" + }, + { + "id": "258ba1332823434980ec88d9ae9e6296" + }, + { + "id": "0def91a119864db7b82f8988d40fa07a" + }, + { + "id": "f6ceae406d594d36865f739600922848" + }, + { + "id": "6dc7354a878e434f999c5f540d943bbb" + }, + { + "id": "7ba7634b50af453eb2d560bf6c4bb791" + }, + { + "id": "d9ea918c55124d12aaa071e29e6c1e28" + }, + { + "id": "39442a5c0c614514b0221df11489fef3" + }, + { + "id": "889a04db874a42f199b597a5f4708bcd" + }, + { + "id": "73f0dd252a6c4ea994872f064611aa27" + }, + { + "id": "b1c0c65c74c24c259b951d6a0d48d561" + }, + { + "id": "626b5316c9e14300b27596c27e1a5c7b" + }, + { + "id": "473fd0c2989541d7818aa50866c02941" + }, + { + "id": "dff359c841e6483e8c5ab98bce20fc29" + }, + { + "id": "d94eac11cc30427fac64da9184a9fa41" + }, + { + "id": "83e227fab836438b9c23a5eabee3f985" + }, + { + "id": "2989bc1f17274269ab803d3035267289" + }, + { + "id": "0c689092f8f44ff995d71ba883c6791e" + }, + { + "id": "e2d39b4356ea455fb10eed85deda8040" + }, + { + "id": "a6ad95f3074e479facf2421462d3f240" + }, + { + "id": "5299ec94dfd7479b83743a56d5f563da" + }, + { + "id": "cc847264086e4567aacc1f8062666dde" + }, + { + "id": "d80763e7226c4f61b747fc7525be2b9e" + }, + { + "id": "582bb618fff14b64bff4fe76b41792ad" + }, + { + "id": "d3b1dec56b024c339380693b31ae4930" + }, + { + "id": "4674a33a917041a3abfa3251b52665b7" + }, + { + "id": "eec2fd38adce4af1a7c28c7ed67a1e1a" + }, + { + "id": "a94000ad11e442f898abfb5d3bd5e1fe" + }, + { + "id": "2f6470a0709b4fd3a29d7a0a2fd89df2" + }, + { + "id": "8a5053cc7aca4cd1a5ffda77ddb5ce9a" + }, + { + "id": "5348edf0674f4896956056dd6ce97675" + }, + { + "id": "8e8417cee60d41d8bcaab2602c2f6808" + }, + { + "id": "d64533a9d99740d3a80b43bf66e34ac2" + }, + { + "id": "c6b75f3d0e3247aca276bac0180e28ad" + }, + { + "id": "8361a9df6ddf4364a8299dc8026e7747" + }, + { + "id": "5a42df77817c483ab33e7e25cd1ed18e" + }, + { + "id": "15c144e4d19d4ccaacecdf41d55cc6cc" + }, + { + "id": "5d88f1541dc44bfbb209a1b38da8f0c5" + }, + { + "id": "676039c9bae441ceb21198523bf6e920" + }, + { + "id": "113ef1dcda894e588b26f9e31a1be728" + }, + { + "id": "29b16038d8d842b8911d0c6cda23ee03" + }, + { + "id": "574cafa1c08f45888c669417d5605c91" + }, + { + "id": "9baa6e227fc843eaa3aae5983ef9424a" + }, + { + "id": "5b6144764a754301808a9375717cad70" + }, + { + "id": "934bccc0b59a4d75b6562ed5649c8985" + }, + { + "id": "6d5a1caf360b4cc4af48f703ac98bea6" + }, + { + "id": "07073c3e2de94a46a90014f980f6eb36" + }, + { + "id": "c2f98462f1334e018aedf02dee8e4313" + }, + { + "id": "bb3d2564665c46e2a30f97a7d5eae18f" + }, + { + "id": "a054243caf0a49b59814afdb8d715755" + }, + { + "id": "10f7141925ef42e2bad2689c5e9c0eda" + }, + { + "id": "65b73d8b6bb447a5aa7a83be192a3caa" + }, + { + "id": "736d549bc5a343c1a16e4b0a95895c72" + }, + { + "id": "22f37f01b06d4413af29385abf55c2d3" + }, + { + "id": "1cde9fe2c8f9435aa48e34c4470bba4a" + }, + { + "id": "d1888a2614b147eeb2957960ac521fe1" + }, + { + "id": "cade297c16cc4381b27843c523b0c43c" + }, + { + "id": "e306bcb045c3479280aae5c54c7fc0a1" + }, + { + "id": "f76941e170a44d948c25ceb88c18557b" + }, + { + "id": "f75e4eca3b14449d915798485cc51c27" + }, + { + "id": "b5449c84277b434398372a0c84965db3" + }, + { + "id": "60b000db8da6444098b74c01cc1eb0e2" + }, + { + "id": "9ad3ddc83fcd44078bdecf5310945776" + }, + { + "id": "f0ce0795002b414387a0a2fdd3e8719a" + }, + { + "id": "98c0e02584b04c36927b5cb2a4dd1677" + }, + { + "id": "7ee3c5d3e0774f0b9a983092d4409d7f" + }, + { + "id": "647e28cc0a244bed847c249d6e8da4f4" + }, + { + "id": "7a21823438ee4da494e7e27783ef2d42" + }, + { + "id": "1a5f20f916df46fdb63bc3469336056e" + }, + { + "id": "7cb802714cea406aa9c35601bb1b2c23" + }, + { + "id": "981894e1f0174f148bfc896cc6c37c38" + }, + { + "id": "fe6e92dd15c0429b9204c0ab231039a9" + }, + { + "id": "b4438c6b6b514aa4a1981bef9d30b21f" + }, + { + "id": "e7f1885a07d94efbbebe04829106830c" + }, + { + "id": "e10044cc3cae45608f6bdaf58005a26f" + }, + { + "id": "96a445e4c32e496baea79636db8eccb9" + }, + { + "id": "ced61a206d8343fb937e03fd2e76a5ba" + }, + { + "id": "8ed57f5c2ea64cf9b3d62d346ca092e4" + }, + { + "id": "8d987ccf45474bd89253c283b0a2afbf" + }, + { + "id": "f6ca20337fba45df808efe56587cd5d5" + }, + { + "id": "00b991d41226417bb77c2825488baac5" + }, + { + "id": "8fc577b2c9a24ab1bce297bad2408a47" + }, + { + "id": "287dfdc1112d49139b699e77a743cdbf" + }, + { + "id": "5d0101c39a8f4b0390277b73d6ca4005" + }, + { + "id": "3b118e16845f4c09a5c0ad28fa40b006" + }, + { + "id": "03a705842c15434283a4f09f89b8b553" + }, + { + "id": "ddb9ebc2b0824145a5a1561d3e303ac0" + }, + { + "id": "9f05e19748384eacbb1a707586954726" + }, + { + "id": "faa49d92b4c0411590608ab3ff7dbd72" + }, + { + "id": "48ae0009527a4ba0842cf64a51233faf" + }, + { + "id": "5f64cb16a5fe4ae59f115abb613a344f" + }, + { + "id": "fc7bfd85bc0249a8a29c7eb20eb47a7e" + }, + { + "id": "7a004527051f4f59ba6f9026620091ed" + }, + { + "id": "6de37e27f88c44f291990360bf815f5a" + }, + { + "id": "4eb0744e8d644b80b94a1344a84859eb" + }, + { + "id": "9a87943f79f1420f80727b05bb2372de" + }, + { + "id": "502fb1bae3ee4cbca89cd9f27ea336ad" + }, + { + "id": "9c037da132884b05be6639477d52a0fc" + }, + { + "id": "fc7d5b5ff10649fb9d504f8c18c074b7" + }, + { + "id": "300801d48f2f4d8088876d6a3061b3aa" + }, + { + "id": "35001a53f50d46cea48df347c4179128" + }, + { + "id": "fd8a452315f149bca027943bf51dc942" + }, + { + "id": "0b57ee37f76e4aedadcf30f3f3f64065" + }, + { + "id": "c2b851b5ff944169af497fcfcc7b5df5" + }, + { + "id": "87885977c8e54c77a529f34c3b03411d" + }, + { + "id": "b10f01cf02e04443a3de620e24f5f86c" + }, + { + "id": "6f4ec63343fb46d0bf9b7a16802285d5" + }, + { + "id": "469ccca2f4df48c5bb1322f6ee1f5ae2" + }, + { + "id": "3ab1af1d409a4e1197e5b6e07b1c71a4" + }, + { + "id": "8d044f3c240847d2862aa7e0d73bd9d8" + }, + { + "id": "e5e77a9058cc4bf7a92c6a911601f701" + }, + { + "id": "ee64d64641884a29968d0d62af120707" + }, + { + "id": "9bb18f3c66b44057947371fa8d185a1e" + }, + { + "id": "0c78abcb5f1a4f029167c156ac56b000" + }, + { + "id": "678a7691bb3e4274b624af40ef488b03" + }, + { + "id": "04ee94462305470b987a01d6405e159a" + }, + { + "id": "c34e279373b949cf8e2c881dacec6171" + }, + { + "id": "86bff10296aa4a27a1fc0b4952c4d0c9" + }, + { + "id": "94289b1fd355402ebb4bea3a9f6c0aaa" + }, + { + "id": "6996fbc6972043ce9f7e0cf90a55b30c" + }, + { + "id": "eda12d2897b048d29d06e154139b5247" + }, + { + "id": "95cf345e73974f0d88a4b8cc672d87f3" + }, + { + "id": "74788324a10444caa373c6c132264ef1" + }, + { + "id": "59a121a02d124bfe84f296faf536ec34" + }, + { + "id": "3706087847e5422f8271b5ef961601f3" + }, + { + "id": "3a649663c9c24c5f990723e9ffce087e" + }, + { + "id": "6b68a6dadf3d4c1fa91721ee57730bdb" + }, + { + "id": "73bbbb3357304ec28774176d9745f8a0" + }, + { + "id": "4edb4e2bad214db8978b50448dc480d2" + }, + { + "id": "5eb5d29e6f8f4ee6bb8c46f30b20bb90" + }, + { + "id": "1e2d37008d46461abe1a480c9e63bb2e" + }, + { + "id": "6abaa808578a4e3fa8eebf0fb4746524" + }, + { + "id": "d95d8625d0334f89915ee105b9e34b4d" + }, + { + "id": "7ec750061ca34bbc927c07fd28f2cefa" + }, + { + "id": "10b4abb550ca406c8d4e33c9cff31fe2" + }, + { + "id": "fa792aca44234b2ba8410edfad4fe8d4" + }, + { + "id": "c687496e98c74437bbc4b8bf606ac072" + }, + { + "id": "c9c3ac27321f436cb35d16796c19d8a6" + }, + { + "id": "428de1f33b694736b48478c9fa3d9c90" + }, + { + "id": "1d8e752954e74f6eb7e718011414d6d7" + }, + { + "id": "86d1ce36f5eb4b5f9d5a2f444e0d3410" + }, + { + "id": "8ce3c7071ee34ee7858f30496106e859" + }, + { + "id": "708b4a8bae5249be828bd88d87df1881" + }, + { + "id": "113920ea62314360be6016cd2157ec4b" + }, + { + "id": "d929253bd8954a849dfc86401d878d1f" + }, + { + "id": "465d7e4fc1b840c7af24faea67493152" + }, + { + "id": "e479559a0b7b47789e1d5293852e6994" + }, + { + "id": "498bd3a0553c45aeaea0c0aff24ffdb6" + }, + { + "id": "7793a82070884d96b11e47e8a1b5a7d7" + }, + { + "id": "8344454542d649fd92f5bf7b046d4bed" + }, + { + "id": "dee7957065c94ee090d39bdf0373ef9a" + }, + { + "id": "4d5e0dea4f574ac187add806d1714a0e" + }, + { + "id": "70202b1963a545e7ba8ebeea2b589c32" + }, + { + "id": "671f3c2e406f4ca7b41cb69fefe72e66" + }, + { + "id": "cff206087af64daeb2f1d2775346610b" + }, + { + "id": "a8648c93b9b0446f8ddeb9c823a713cf" + }, + { + "id": "082c96ededd84d999e08d466a87df5b2" + }, + { + "id": "6ac7b9d73a2b4571b40b686fd84cda6c" + }, + { + "id": "e6cd02bc72bc4037b1c570a31c78e8d2" + }, + { + "id": "328b8c0bf3f041cc8aac9d6e641c6d02" + }, + { + "id": "8e11057543334656906b81118a19a735" + }, + { + "id": "03ff0bfba1e441cb952c5e96e081cd95" + }, + { + "id": "5793a138742e4e3d941703f36e07f1d6" + }, + { + "id": "56ecfbae1dcb45f0bff6b57741549963" + }, + { + "id": "1027da416e6845f7b2dc1ec382291283" + }, + { + "id": "2851247e64e74637a690f20a08320673" + }, + { + "id": "8fe10b939ea84d8abd2b64b409aa6d75" + }, + { + "id": "e57213c97d9541dc97988ec38f544b3c" + }, + { + "id": "5a2b64d258194300b61e72edc4e56190" + }, + { + "id": "3024323c6138434d9f1662affb9fb692" + }, + { + "id": "d4162c20a5414f1c83e112b7d4f484b1" + }, + { + "id": "09fb31b2ca0042ca8fc523353e633341" + }, + { + "id": "7aefedbe5511421bacbdb9d6d5bd94e2" + }, + { + "id": "aced03563b4b43af889b1dfbeea8930c" + }, + { + "id": "56f8ab13e599488db9ea7da6506dd22b" + }, + { + "id": "648a3e22947944589f67270b3bf39e5b" + }, + { + "id": "bf73afe8bee84b3bbe542b2b88e4f8d8" + }, + { + "id": "f8388135f4164e7c9b638736e6dbe2ea" + }, + { + "id": "7a5fb1ab2110493a8d592d19360934d2" + }, + { + "id": "5ff5cbfe8f2b43ac9f83e280fd971805" + }, + { + "id": "08643f2395a442048048c00a198ffff0" + }, + { + "id": "ed13c6c4f99a466abae80756060d24b0" + }, + { + "id": "ebe62a81dd104541971b10c5a5564d20" + }, + { + "id": "e39d19a65521470fa0a20de46fe730c0" + }, + { + "id": "b697e6d84196496d8edf419d5fa91c5c" + }, + { + "id": "8c9fb9585a8a4fea9fcbc4354bb8d103" + }, + { + "id": "81218a8ee1b04b7d879451985591f6fa" + }, + { + "id": "d6ba8697e6694769a0ed824a7df27252" + }, + { + "id": "fbe47ed4a64c4659a4b0ea00be4d94b9" + }, + { + "id": "3be35a549dce4bf090544d80e41457ed" + }, + { + "id": "a5d2e623e6c84312b5fe863872fbce1c" + }, + { + "id": "60f4ea1efa2c412a86b4739a9b046439" + }, + { + "id": "0262ecde89394a1594fe54ed6de67d5d" + }, + { + "id": "4b6b4fd437ea490f9c7064152fd5dcac" + }, + { + "id": "1f75dbd11b634d259b8e74de03011324" + }, + { + "id": "aa3c5dbe762f4e359842e588b8ae47e9" + }, + { + "id": "93c5050360564c80b95382bc27b44fb7" + }, + { + "id": "bae1026af636465dbd8a49333f7c831e" + }, + { + "id": "e7304dd4e7914bdfbb2b912691b84422" + }, + { + "id": "a66e9943b68245029ef11c7874193702" + }, + { + "id": "a382ca2b34114134b632096cc1687285" + }, + { + "id": "2ecea5a42dd645ac8d8c0fb34920a228" + }, + { + "id": "5c63c75dfa8f4d4cad5bccf467ec9f9c" + }, + { + "id": "7a903a18b8e44df0bc6b8af86c7e4bf8" + }, + { + "id": "733fbb4bb2984bb4bf23569014902ff5" + }, + { + "id": "77baa9369e3142f1a7884b426030dac0" + } +] diff --git a/data/searchqa_id_split/train/items.json b/data/searchqa_id_split/train/items.json new file mode 100644 index 0000000..7df7e98 --- /dev/null +++ b/data/searchqa_id_split/train/items.json @@ -0,0 +1,1202 @@ +[ + { + "id": "221c83e6630f4e7983da48fa28da1882" + }, + { + "id": "655e4e5af91f45539ca76f7129656e3e" + }, + { + "id": "a234f47261f14ab9aed867375bddc652" + }, + { + "id": "10031674a6f14bc2b6ffc77f4a8da3a6" + }, + { + "id": "3909c367f524402d8ba5ec2b520ba206" + }, + { + "id": "05234edfaf004e39824dc5e6e24bea97" + }, + { + "id": "9587fe34c9234e2594097a92be3bad43" + }, + { + "id": "a30487960f7c42898d332c6f038686f4" + }, + { + "id": "db15ab3b562142f1aa6e7cbb0a561e83" + }, + { + "id": "b257b0fd2f964b0c89af47ee4df05e16" + }, + { + "id": "1fe1949081d84adfa902563d7f6740d4" + }, + { + "id": "278992dfba6a4d959973da300131fd4a" + }, + { + "id": "12cde56adcdc41d9916c1423da39a156" + }, + { + "id": "69f53eacb33146e7a5bc41be38d39f06" + }, + { + "id": "cd002e0461e8454fb26851ff53c2ba22" + }, + { + "id": "9e4f88422e8c4e47a7f96f7fe0ecb40e" + }, + { + "id": "270f65b074984463b3fe55433c93ff22" + }, + { + "id": "3408f804dc7e4a6bb51891e3da461a54" + }, + { + "id": "3da9ca540d2742c48622caaf07e4b618" + }, + { + "id": "0d46f4a90ab6451486c755fbc4d2525b" + }, + { + "id": "2be20bbbb7fb4e8d8cf61d3863457b3b" + }, + { + "id": "f2b0c18b74224a8290b134bb6c1456be" + }, + { + "id": "ac67fde6994c4862a8307c958e371191" + }, + { + "id": "b86af4b0891045f1a14ed96507512938" + }, + { + "id": "25b184d228f84079a2fe6cde821b9f09" + }, + { + "id": "5d301b6778ff471f9cbda7579ed4e441" + }, + { + "id": "6b12d4b87fb54647b6e35b8ff85336a1" + }, + { + "id": "3c064caf2f694852a0d26e639caefa01" + }, + { + "id": "df224beba4ae45a9b4ed66443174b98b" + }, + { + "id": "31c23742d99d445794d38e6ee9f39e0c" + }, + { + "id": "d5160ee19b8e44eb8e6d5ad2bfb31b4e" + }, + { + "id": "ab32e16d9ebf448282b3352674a23466" + }, + { + "id": "3cf34a52916f4ff9a975008fe9d771d2" + }, + { + "id": "b722cd69f05c4f39bef92c714fd9aaa3" + }, + { + "id": "bcf886e6d73b4fa7be27b32421c5bc49" + }, + { + "id": "7ebc5c81713a4197ac303dec72719c0f" + }, + { + "id": "0ecb623e905e4ef1b6f2b082e3eaf28d" + }, + { + "id": "84ab80ab3c5d47a58ddda464d62c5694" + }, + { + "id": "5b04fd8c679641139047cc40603013c2" + }, + { + "id": "812824d42d7443b5b47c55fc1cb29a47" + }, + { + "id": "a7f79b085bd04f98847589443fd0f61d" + }, + { + "id": "400ef271d312459ebf072bacffd438be" + }, + { + "id": "c07610477e95400dbed310bb792c13f8" + }, + { + "id": "9e8dc4eb9cdb40cda6941e9b0d0fc0ae" + }, + { + "id": "19b5970d418b4eecb5f9efdc2b72e15d" + }, + { + "id": "d0d2a5070eaf43ef8b449dbecdf57817" + }, + { + "id": "d17df63c03b74139a64c2d7833e235d3" + }, + { + "id": "2b3d1cef30794b2a831d7a7343e18ad5" + }, + { + "id": "635d1210762d4f03adaf4fd1657ed943" + }, + { + "id": "236f43d3be1842018933ab6fd06b51c2" + }, + { + "id": "2f64ddfae03e4ad09a994cc70db5661d" + }, + { + "id": "7228c5d4db444c11ad27df32fd1db318" + }, + { + "id": "fdec15afb4cb4e80a6a313aa29cf6c26" + }, + { + "id": "7f74a8a0a880403faff0826dd51aace6" + }, + { + "id": "d38e00263d944bb6ba72bbed58b5a5b3" + }, + { + "id": "64cb77e3c59a40e1ae3a9502bd565867" + }, + { + "id": "4e8f6c14fade4b47b986ae9ff9184234" + }, + { + "id": "58f65c725a344349b9412bddb5ba7c4f" + }, + { + "id": "1f424622ebce46c1b9360177e2864bbb" + }, + { + "id": "8b7616a2a89f4e0abc82562789c71c7d" + }, + { + "id": "d2f25c9e97a6478aa93396deac82f7b3" + }, + { + "id": "37ba164b00db46d4886345dbffb01253" + }, + { + "id": "24df73fb9c2c4cfcacc037761fd38b24" + }, + { + "id": "9ccac9b7c91c4747b36dcae64742e3a1" + }, + { + "id": "d617b9d6ddff4879b8ea242a4132a5f5" + }, + { + "id": "e40d9fd25333471391594f14203c326e" + }, + { + "id": "da56d013f3a14211b608b6ecbc58500b" + }, + { + "id": "c0357a65407d4a28b267b59ed5e02e5f" + }, + { + "id": "96105c04691c409a92a47eeff53b30c6" + }, + { + "id": "3f52dc3e15fd435fa94f015b6a74e09c" + }, + { + "id": "bb078201ee20440680bee6265e6b8da2" + }, + { + "id": "b44cf87107b9474d8e04d8147c895a60" + }, + { + "id": "7c01584f34d540788ea32e0a4054943e" + }, + { + "id": "0d6855b36a0a49229d38e4a376b6ee09" + }, + { + "id": "9a630b9906b74ebfb40abb3b9110fda3" + }, + { + "id": "1eda265b17424078a78594c9c13f2727" + }, + { + "id": "7381db7eeccd4aff8290a391fd87fb28" + }, + { + "id": "d8360bf147c5483a867d45e15cddb1c7" + }, + { + "id": "ef61dcfe0a4e4b4ab3de982f084f5f0d" + }, + { + "id": "2c8296d242ba415da60f642f1d40cdad" + }, + { + "id": "6e240188701842e282d3505b4f16759f" + }, + { + "id": "82823aae4a844d19ac0a56464dfe28c0" + }, + { + "id": "6d89b2af311949b3ab5b6c93f4c85e7e" + }, + { + "id": "d0f792528e814905a9679bfdc38d757e" + }, + { + "id": "03a6254dd0154f0095dc73ff48f2ffb3" + }, + { + "id": "82b3d01ea72847959730b3ef349b2d76" + }, + { + "id": "4fbf8ff98015460dbfeacd831db8b487" + }, + { + "id": "49ce7d167210412c83f5adc12972a8b0" + }, + { + "id": "b3a12a359de04d1e8303487d8abcbcb4" + }, + { + "id": "be33c663172e45daac47333cebdfcb40" + }, + { + "id": "303fa164ac2e4484af9b7a70ebcc9ef5" + }, + { + "id": "849d9ff13e0049a991a9bd486f5ce867" + }, + { + "id": "4f1d8d2bba6a4060b88cd6b0bc22bc1a" + }, + { + "id": "7917f6598ab84bdba8c2e81c3cd99fc3" + }, + { + "id": "3154424fc756402986578589696e0cea" + }, + { + "id": "1319d2f605ee4f138fbb300d204f5b50" + }, + { + "id": "4507bf165d874c68a245b00fbd7fbae7" + }, + { + "id": "11db4151a9f34f9fbe86f521fed062c3" + }, + { + "id": "ec3252ab5a9e4e37967740b172bbec8d" + }, + { + "id": "7b250d069e3945bcba610b9fc43a46f8" + }, + { + "id": "bc720353831844449fce1e565f945ced" + }, + { + "id": "db8cf44331be40b993b053f5f0e4a996" + }, + { + "id": "dcbf2ef2cdf1480797460be29a1ee070" + }, + { + "id": "9ea2737a55994603b6ea7e7ebf88f98d" + }, + { + "id": "58092ebe17e942589c3f820cfdb1b602" + }, + { + "id": "b63e3d061b494f01967242defbed9ad1" + }, + { + "id": "929d024018584364a928895a128d9c74" + }, + { + "id": "358ea6b314a046aeb50bd1de704fca76" + }, + { + "id": "59b6334f5b434d8e81465b14eea5d4a5" + }, + { + "id": "85dd599e005947249d15ab9699c57060" + }, + { + "id": "7e9cc91827e049e89624d8004cb786dd" + }, + { + "id": "af751c75c17f41fb82a03148478f9873" + }, + { + "id": "63787d20e4d5407ba2f041792127069f" + }, + { + "id": "4d75d9ea2bb9452a8ca7f99f548a376b" + }, + { + "id": "df509fcc94dd4f558a8a4408d099cd57" + }, + { + "id": "2651422c5a2043c79aba79b664f8adea" + }, + { + "id": "a6eb760a3753499881a8bbbf9e60749c" + }, + { + "id": "7dd729e3b10e4154a490894f99f6664c" + }, + { + "id": "c66c3e09a4974ac3b4e3ca131991cac9" + }, + { + "id": "4ad21063e6174d03ac6dc1c66f5ed76d" + }, + { + "id": "514ab2eb38724633848138feb7819466" + }, + { + "id": "794dafa0239b4b82993a22737e64e05c" + }, + { + "id": "ef02c2c08ec34a9fbc7a7ce3e4444b13" + }, + { + "id": "0de4e6953a4e46e5ac23de6d0741bc73" + }, + { + "id": "36b0b926869f44d8a987d4d5d7d4db4c" + }, + { + "id": "e244e769dc384b2a8b953c21ba981caf" + }, + { + "id": "fc0f867a93ed4140a5086179a0aa3584" + }, + { + "id": "13af14965cf748a8acecd7f0b9a9b03a" + }, + { + "id": "09bccacc26e94bba8a9378299f46e619" + }, + { + "id": "b55578edb7ba4b5ba9ed09dc7333795b" + }, + { + "id": "87969d82d83a459aa85c0897c717a18c" + }, + { + "id": "2f6c578181d34df796365787acb8ecc1" + }, + { + "id": "d331035c6c7e49f59b661bd058788c02" + }, + { + "id": "1e2d47768a16419688fcffb970f20c45" + }, + { + "id": "3d6d6c6208a5461eb6f3e43db5e56a9c" + }, + { + "id": "6a02ca6d142f4d9da91fb755d9f81d60" + }, + { + "id": "8cefe47651464349bb1633536633fa54" + }, + { + "id": "8a12738fd7e0400090173719016af1e5" + }, + { + "id": "7c4a7fed8d804433bccc2f07fca94686" + }, + { + "id": "bb9262401a7d4fddb761e356ac024c4c" + }, + { + "id": "08daeab29c054c40b06e14e9d48ac440" + }, + { + "id": "24e447c6f2564bfcab303c390cb606dd" + }, + { + "id": "6255c8693ca94f4f893c39421d9469e7" + }, + { + "id": "1771ba8dd1ef499384b613cc5755802b" + }, + { + "id": "3eff11c208b24b9183b109a73fb0fa64" + }, + { + "id": "2bbdbb989efc4ced951e49007006cc17" + }, + { + "id": "a40a4414307640f4a89cc84e00e56ea2" + }, + { + "id": "4e71393e05184d6b84c4538f3d2be58e" + }, + { + "id": "a7e4d6381a1b471c84731033ec54cc0b" + }, + { + "id": "524365710af24abb8fd77f51e4ea9284" + }, + { + "id": "a1ffc8a731aa4fd3af552c1edd681425" + }, + { + "id": "f3c2984ee10e48deb259abb7d0114cf1" + }, + { + "id": "2c2433f52c4f4114b0a7db8d3abe9ab6" + }, + { + "id": "6072e63558b445ad9c01394dfed313b4" + }, + { + "id": "e67fc6bee9a54b9fab439be3c68128cb" + }, + { + "id": "05233ca5688f41e89f651c35808e59f1" + }, + { + "id": "4d45900647c04ab28a883708714bd3b5" + }, + { + "id": "80437ec671f24947ba2ee4c4c40957b7" + }, + { + "id": "1eb4c324645f40958054bbe40249875d" + }, + { + "id": "d7bb76a5be104cc8bd8b709245c32cb9" + }, + { + "id": "0cdf4a84c9084d4d9f8168e530c5d644" + }, + { + "id": "424eb2b3c6f64445a43efa57a60f5477" + }, + { + "id": "4f0262bf1b68426ca3a43d78a4749507" + }, + { + "id": "ad15e2300a1a47fdbfeb72706be74969" + }, + { + "id": "1400ca67c580458d91e3cb319ce25a74" + }, + { + "id": "03cf1cccf4e141c4b305d86909d560a9" + }, + { + "id": "ae7c2c3d320f45868752d8044309d64a" + }, + { + "id": "88d06cbae9ef4d5f827ed4d8ed7e2ee7" + }, + { + "id": "3c93b5a9314a453c98d709447a0d9272" + }, + { + "id": "f11a49f72c5445cea6e370916ec70d93" + }, + { + "id": "4a329ae0cc934d0a8bc1980b894838a5" + }, + { + "id": "a4375645235745baac2b0253713ff424" + }, + { + "id": "553f1b511f044102b9a5a31fb23d7fd4" + }, + { + "id": "404d552206184b09b265f3fa3b4c0939" + }, + { + "id": "9060c712023042b1bb2c449d8a5c392a" + }, + { + "id": "1dd7a02b15e0470d91a8fedfca5357ca" + }, + { + "id": "24c4fc7a5ec64d5bb2758afe030c8709" + }, + { + "id": "5e6de2e5738545d68eb2d7e7ecc86c85" + }, + { + "id": "56ac9a3893af46c9938c0e356255b880" + }, + { + "id": "6252b238d9554d45bc4eb9b9978543c5" + }, + { + "id": "bc1ef2955bcf472196361585d9463538" + }, + { + "id": "456ae8cba98e4a008e46e704118076b4" + }, + { + "id": "ffbdd2a32ba64dd580da86cd611bc5cc" + }, + { + "id": "099a6cecfe654e189ed9bb5b88f07366" + }, + { + "id": "167752253c1c4cff907a6314c5754d12" + }, + { + "id": "7625c352e07f4adda1c443a7f79ab85c" + }, + { + "id": "a408c2691fdb4496ad7361c8b301d60a" + }, + { + "id": "a33ecc8eb40543ac8555dc12059f511e" + }, + { + "id": "dfb0deb117f642bd98d744fdd5aa6867" + }, + { + "id": "0224ec0a999a4e9bba48c1fddaab7a90" + }, + { + "id": "8517d298719f4e7aa9b1acb8c822b27e" + }, + { + "id": "24ca993f0de94e219bf272245186f68e" + }, + { + "id": "8f78cc704744414b85fb4c2337882174" + }, + { + "id": "a2c76bb55b1f4d74b5871c6a1c1fd1c1" + }, + { + "id": "ae698acda1784f6eb6a946e081c43526" + }, + { + "id": "33705396b4a04b1d94ba5c0800174153" + }, + { + "id": "0426c3db91de44cdb04860668f6980db" + }, + { + "id": "4edd282826b84986a11ca56b62daaba6" + }, + { + "id": "1255b79ebe5347e4a596b93faf9a4ef8" + }, + { + "id": "bc63eb4bd9234e4583a6c12cb3851cef" + }, + { + "id": "9503db8da9dd45ed9b51b4ef05ab7833" + }, + { + "id": "81327636d4624edead2294809968062c" + }, + { + "id": "12028e673ec7417ea826d4d7c9e1ca25" + }, + { + "id": "792e4f810ad24433870b2a23fc13f778" + }, + { + "id": "1bfa6eef0be04bf9bac6288883877660" + }, + { + "id": "1828dbc7142641cf8708fd0298158308" + }, + { + "id": "4800beb1d45f4c07b41e608bf9c2b579" + }, + { + "id": "fe04e87911fa4df8a6e1efc58867bc1a" + }, + { + "id": "4abb5ee3b7194c7fa389906773b7c2ce" + }, + { + "id": "699d0ee09cef409cb640078bb31baa00" + }, + { + "id": "f7e5f18e497a4dd1915989f9bc7ec238" + }, + { + "id": "4e8fbabb7956459085d66bc20fae8324" + }, + { + "id": "0ae41d57a6b647969ef065e051dbea43" + }, + { + "id": "fcf2b8b0ba924ce095aa0eb5fa543528" + }, + { + "id": "28ce80cc4e9440b8bae1ec4a2c94c4c4" + }, + { + "id": "1e687cbe048042f390f9448065703dff" + }, + { + "id": "2d8fbdb21ab448f7819e13a06ffd9bbc" + }, + { + "id": "8709f2c5215049e2b2edc482a2d6a332" + }, + { + "id": "d64792de04804b7198aecdc143597e9f" + }, + { + "id": "d86f059053b84a90964e806da32c9302" + }, + { + "id": "af7efd5e34a443a79b86b8e86373c577" + }, + { + "id": "748b37db43cc4ce6a48f8eae5991fd60" + }, + { + "id": "3ca738c4497e48e0a746bcdb81beec97" + }, + { + "id": "d98417a0052c446b9cbec27592b0716b" + }, + { + "id": "6444b0f90cec44c58387b82f6047e411" + }, + { + "id": "1f479608c5434054b6996e4c2dfe8a8d" + }, + { + "id": "c61866c20826409294b00fb21e1ad40f" + }, + { + "id": "05d9f31d54204be3b0ec885879615f07" + }, + { + "id": "3bffa7bffc804719a3c16fcee66dc67d" + }, + { + "id": "5f31efdcec8444b8af01a013da7c1072" + }, + { + "id": "ea88242882794a9ea5705de2e68a3782" + }, + { + "id": "6b497ac583564d389f5e5f3f565e0030" + }, + { + "id": "9a7142e4d14a46e3af6bdd1237f9eadc" + }, + { + "id": "39ae35ba57974073895c183e2c4b0cda" + }, + { + "id": "618aeca497d04e9bab54cf8a95cf2ee5" + }, + { + "id": "eef32f17818e4652b3eeda7b4e0cc430" + }, + { + "id": "da7acff4255640aaaf92c73cf53e1423" + }, + { + "id": "234a1d78bd3449b6ac6c3f4ed367c6b1" + }, + { + "id": "4515ad88ccac4171bff647a836a79771" + }, + { + "id": "13d06d6d989b419ba523462a6e098a22" + }, + { + "id": "781056bddbdf4d9398c75358271035e0" + }, + { + "id": "60f6f1a1734b4ea3ac6130d52c2264ed" + }, + { + "id": "4710790ce91e4a87b5996917a6b1e3bf" + }, + { + "id": "f82dd783e8d14425916cad9fdeb26b6f" + }, + { + "id": "ae33a306677848d684bc3e8f974ac42a" + }, + { + "id": "aa84ea80d64e4233818186bb0285052d" + }, + { + "id": "e702332b0c5d40309a02c987b512dff4" + }, + { + "id": "770470c452174a719260377ebb9ab2f4" + }, + { + "id": "fcb4868f4e8847f79ef5622f2e05ddcd" + }, + { + "id": "2a01dc0307324219a4662f82f260a065" + }, + { + "id": "fe01132758b64f638d75195e73901329" + }, + { + "id": "326f487e62644774bc5d26e906e12039" + }, + { + "id": "1ffcea80a2334a1fba10c96a4162fc0f" + }, + { + "id": "8cde5fd949c8425a8a651fb64249411b" + }, + { + "id": "d5319e0a56c946889232069668b770af" + }, + { + "id": "0d73382067a64ea4a6ed320ae386accb" + }, + { + "id": "4c64770f574c499392b07f182a75c72c" + }, + { + "id": "58d9a51541c94bb3a24e88109329d14c" + }, + { + "id": "4d2b5a81849c458794480c61b07264a2" + }, + { + "id": "691178453c884f98a9205ef3d143bc42" + }, + { + "id": "0463305740994fbea5a10ae8303e66c1" + }, + { + "id": "815a01c49cbf45b1aef3c9ff1844a49f" + }, + { + "id": "d4bbd47856554cb08d00cb00e29d02d3" + }, + { + "id": "55f92cd27974459aa893c131216a075c" + }, + { + "id": "7f568680a90f435ab953e342a7fd57a2" + }, + { + "id": "61edfa3561d44e728f57ab67783cb099" + }, + { + "id": "142bd93393f2485196d7788833e978c2" + }, + { + "id": "cd0482f36ae6406a87338d8f65135eff" + }, + { + "id": "ff0750f494744de299aa839d0cdfcc23" + }, + { + "id": "1c02c69e65764c4680caaa9814ff105c" + }, + { + "id": "7f5514c879eb4fcea5b2c660b4ab0496" + }, + { + "id": "7e5e734bd37c46c8b053387161ab00fb" + }, + { + "id": "b5308cb3eed14303a07973a5fa3532de" + }, + { + "id": "52394ce0966f4ebe8a5ecf7806f447cf" + }, + { + "id": "88ef5630dd7e4d449171aed39f7b78b3" + }, + { + "id": "dc9acf0f01f74baea5aaee4c6bc0d493" + }, + { + "id": "437621201d544d3b8993b6eb1bf040d1" + }, + { + "id": "fa57913ceb924df88a8e7d3968927ae4" + }, + { + "id": "72f6908d092749ee980a42b585143084" + }, + { + "id": "5f096a5da5954ca7b948a59c038e00bd" + }, + { + "id": "df7579f28df544c797dbdd2acf504988" + }, + { + "id": "c3fc79b46f1b49c6aceb10bd477433e6" + }, + { + "id": "c7a84f3899da43b1981bb483288fbae8" + }, + { + "id": "7c3a99ff45e44531ad5af85d04f7d55d" + }, + { + "id": "89fb20b672ea4978a0b394206535cf4b" + }, + { + "id": "f15f9a3314b644fb97db3cccdca78563" + }, + { + "id": "471c570d45144879925eca82ac59f6df" + }, + { + "id": "1ee43fb75a4e4a85a0a91eed1091d738" + }, + { + "id": "7f8fafb69c89438fa2d458434ed4ebe9" + }, + { + "id": "25ac1e9f9a9a44f39f0517e01c9b09e2" + }, + { + "id": "445d4959a0a94d06adfb86d80c5a9dbd" + }, + { + "id": "f58173d98a0249d69e284edc617c72ad" + }, + { + "id": "9d95fe416a564ee4baeed62c62474b99" + }, + { + "id": "783d987accbf45a8b33535139c41fe0a" + }, + { + "id": "8ca657a0c0884c0bbb698f137489c09c" + }, + { + "id": "bc47b1ba0f164881bb3f8997c799ad49" + }, + { + "id": "7f35282de6124a96a21aba7d5f18de50" + }, + { + "id": "46c99666e71e4e0785682433cb5ae53a" + }, + { + "id": "de091de0055049b7aeb94b63ba4f91ea" + }, + { + "id": "7e6a92097b984d0b9bf6307dca097b02" + }, + { + "id": "f07a634f1aec41d0a17e7616e5640bbb" + }, + { + "id": "1368fa832f564bf9b2cd4df9fdb4d356" + }, + { + "id": "c962c464089d40edbfa73b38dcc6e404" + }, + { + "id": "085a07c84b4c4e8ebb5c02c635665606" + }, + { + "id": "ebc6b86a3fa7432a9aadff96c2582451" + }, + { + "id": "2fab3953f8a34a29849e7a69891c514e" + }, + { + "id": "37aa6e63537243169bb27ff4353f3245" + }, + { + "id": "c8bb005b5d4646d28d793113206b4215" + }, + { + "id": "f6f35af39efd4a16ba440993d142349e" + }, + { + "id": "06173fc595e842c6abfbd7d904bba79a" + }, + { + "id": "c7e063e6fba34b3794255784fbc72761" + }, + { + "id": "fd23a7fe7b654680a211f7e1cfb1f90c" + }, + { + "id": "01a52018756346b795de67cabdb907dd" + }, + { + "id": "17a1c01d1e0d42cc9f0fc257e48d1af3" + }, + { + "id": "fd9daac9e3cc4f819c48b05fb6bd843f" + }, + { + "id": "66bb7afcfd5444eea9c7f3487b1371b9" + }, + { + "id": "08f55849c3e743e6aed1d2791072341e" + }, + { + "id": "4af57faa8a2445b0ae9eb303c0f6c5df" + }, + { + "id": "2bb294748127473c895738c5addb9336" + }, + { + "id": "64b954345b54447eae4b0ebbd8d5b530" + }, + { + "id": "6b03e6e855304974afebc7624b2b7d9b" + }, + { + "id": "34f2d76dceb14a998d726e820be90c5e" + }, + { + "id": "483f4ec71103480c8689f0360798c800" + }, + { + "id": "2899eea4efa7475cb3f553968f335d25" + }, + { + "id": "e6f84aaad73243259e2929f51e197d91" + }, + { + "id": "a842c4623d4845f0b0169730dcb1863c" + }, + { + "id": "1ea2851b642c4b15af816e0af9d8aa7e" + }, + { + "id": "d95612fc584b4fb68f15b416ba9924be" + }, + { + "id": "78c03ede8fd84f939e90ee6e2bfde497" + }, + { + "id": "a1861269c436472f825a450f0437975f" + }, + { + "id": "f8a442980c55415b82527160422a2688" + }, + { + "id": "9420eb98f0004e649aa999b6f66fd116" + }, + { + "id": "6b7f872523094f2495e403e37c04eef2" + }, + { + "id": "7e17030f4e8f487e922841322951cc8c" + }, + { + "id": "bcaec7a9860240a187552622a73d38fc" + }, + { + "id": "ade7418b6bd44e3098b3fcac3ca523e1" + }, + { + "id": "2017497b907f4e17988203e7613abef6" + }, + { + "id": "73cce59df8884a53b7444e4a0f091816" + }, + { + "id": "1907684788be4a09b36c6bb24812fe99" + }, + { + "id": "9f7f70c5ce5c4d528fa599828b0db1e4" + }, + { + "id": "1a89c660268e445ea06a830571331573" + }, + { + "id": "4b9a1b40aa0e4e2abdd1ec66ab0c2629" + }, + { + "id": "beec8a8ec05f402eb0a2fe8735165ac7" + }, + { + "id": "e4a35f12d5764e31a01c04652b0d3645" + }, + { + "id": "760a703c4a2e4ffdaf30f3e6d61b4a89" + }, + { + "id": "99ab970df1ad4eb299acf118599d630b" + }, + { + "id": "74e536c9439e48e48a69e8ac58d0f429" + }, + { + "id": "9385f7d320d047ac97f8d15f6169dbfc" + }, + { + "id": "6edbaf5d20644789ad2ee3e43db1e415" + }, + { + "id": "0b7ffa4435c041b58e0517deb8b6303c" + }, + { + "id": "d93529278d1d4fcdb3dff5d9bdefc977" + }, + { + "id": "acb5cbdaf5ec42ea940c459108a43e78" + }, + { + "id": "808a5cedccb34c9b973c223dd6b64407" + }, + { + "id": "1d039c043dde4ca4a740e0ec20e936ea" + }, + { + "id": "3fdfccd5f0a24e57ba350bcf434fa291" + }, + { + "id": "b903e822010b4363837a3d26eb110a7e" + }, + { + "id": "401e8e100ec14ce1abdc673927d835a5" + }, + { + "id": "1c2cc13e0fa745d6b541f6c7eb9f5619" + }, + { + "id": "178c1b2e8450497a8fa6b0d853b885ab" + }, + { + "id": "67186a3da6fe44de968b3dcd38ecc059" + }, + { + "id": "4a01a9f5147b44c89bec8b58e7d82180" + }, + { + "id": "89411c8621e240c494a72549e4b60917" + }, + { + "id": "c1c8058650df4505aa05b186c4baa145" + }, + { + "id": "0fe06452bd2645dc8fabe1ff397d352f" + }, + { + "id": "13d7f98d33d645f5bc6a9d7e392c802d" + }, + { + "id": "5c9102c403f14c0e971318b649cef60c" + }, + { + "id": "49675b915f874f3e8be70b85778d4763" + }, + { + "id": "f9a43b1f4b0045199d978101d84ce464" + }, + { + "id": "2c699e9e4ae44c33a19f35b48d894de7" + }, + { + "id": "da0d8a8163f44bd69f74584c0d3caa0c" + }, + { + "id": "2b3bbf4cb95540c5a5d0e6fa2f4c262c" + }, + { + "id": "c17c5f8d2ef746a7a9a5db0b674b865b" + }, + { + "id": "a74042fa649f4fd28697a8a19aeac620" + }, + { + "id": "ce7deda3129f4ce9ba75e15ab7219cd4" + }, + { + "id": "c69f7fb4be4c4a89bc9644dd8d01ecc2" + }, + { + "id": "276ee199fa1c46fe83e00ca9a3f65b43" + }, + { + "id": "1a02c8a18c464f8eaa996aeeb6781f35" + }, + { + "id": "8c9f6c94c0784cfc81db422bec240984" + }, + { + "id": "08a3439d87854f908e4e2b147f8a109b" + }, + { + "id": "223871d594d44f01802cf260cc94f99e" + }, + { + "id": "1ec91e606a454ca8a248c9f55eed17cf" + }, + { + "id": "224665d37d03421895cf8e7b05863144" + }, + { + "id": "0b550017552f4b8ea569fba16a6516ed" + }, + { + "id": "38f86975e56e4f1196023bc1066d830d" + }, + { + "id": "ef813849b58c4acbae62684d907cebbb" + }, + { + "id": "8b1cc801ee384a50a8a2f85d0057be79" + }, + { + "id": "84d440710b744b89a29347d22493207f" + }, + { + "id": "a6a5643ffbbe4e209ab8b7fc28f12572" + }, + { + "id": "850d851b1f08453982a180d40b9271f7" + }, + { + "id": "5c4fcaf8522d4d69812bf4f813b242b1" + }, + { + "id": "dd1d96c533664c4e98649de8bd67dada" + }, + { + "id": "18515f0c52164134aa75cf7fba6ec4a5" + }, + { + "id": "763c2e8903734a32ad04d027be42eee0" + }, + { + "id": "1a936c9b0fd7482abd4b090e9c16ecf4" + }, + { + "id": "1f6f687c79a642328a742e61154c9d90" + }, + { + "id": "a1673e0607ec48e7972067fab1cc473a" + }, + { + "id": "3d53e8fe018842bf8741c023dbab9a63" + }, + { + "id": "acbc8654d37942cd9f3d2c007167c2d7" + }, + { + "id": "f4afb22a6dd74bf2bb1fc171ef442ee3" + }, + { + "id": "b42aaf6fa0dd4d3195e8ae7e6ca33019" + } +] diff --git a/data/searchqa_id_split/val/items.json b/data/searchqa_id_split/val/items.json new file mode 100644 index 0000000..af86a7f --- /dev/null +++ b/data/searchqa_id_split/val/items.json @@ -0,0 +1,602 @@ +[ + { + "id": "1758dc50625e46ee814e44a6061f091d" + }, + { + "id": "52967b20d9564d5b94756bc0b3d113d6" + }, + { + "id": "e77dbfc0d45e476b9ed5b2204b6d9fc8" + }, + { + "id": "675f01b5343e46599626104968c0f45a" + }, + { + "id": "dccd0f04cd3b41bc8c4182e41a413b3e" + }, + { + "id": "efde5820a29949488feaa18ce7e429f9" + }, + { + "id": "98773558971e490d8de41088e41a3207" + }, + { + "id": "70e3e1e1d41f4795ba45678d5f4b6524" + }, + { + "id": "206a9ffb8ed24686a24db1492c9f10dc" + }, + { + "id": "e91f1225cdb84ee7a2d773b5177945c1" + }, + { + "id": "b68a14a7b43842a0a50a3965d3bbffbb" + }, + { + "id": "e2e679989b544e94bf8dbc3466e49003" + }, + { + "id": "c51c26c07b384bbe865976b81a86736d" + }, + { + "id": "83928d21f3504843aa6294ce9d9c8da1" + }, + { + "id": "f822cde36d864c508b426f3e2b5245b2" + }, + { + "id": "f03757766e034c539bf2595428c03626" + }, + { + "id": "349be67be2e141438287d21cb6a94131" + }, + { + "id": "7f3e18272c6d43dd8c6385c4758b685e" + }, + { + "id": "66dc8aee931142b1834da63363511491" + }, + { + "id": "52ac78645e144b4598958d67920617dc" + }, + { + "id": "32a66c67a8d7490b8572387d8f1d762a" + }, + { + "id": "b028bea99d2641268532cd2a32d4ad2d" + }, + { + "id": "c9f3a971abe64853aca3e0bb1fea6962" + }, + { + "id": "5324041c25d04316a800ccba2d79a2c7" + }, + { + "id": "9960a3f1ac954282a7ea1d7f7c8a86f9" + }, + { + "id": "03bd8fc86a9d465e9370dcdaedb3c422" + }, + { + "id": "a1d0d03fcac54483a1e88542b767afd5" + }, + { + "id": "c21a41745ab9402da8b1deea67055a8d" + }, + { + "id": "d5c48bd600824c8a9328df80a8baf964" + }, + { + "id": "746806ac0b1c48cb80d7ab5ad0a63517" + }, + { + "id": "e3b96fc1b99247ebada09a84fa5fd205" + }, + { + "id": "bbeeec20466a4dab9e4d73d86c964cba" + }, + { + "id": "2b85b097d130404daeb26a3349bdc297" + }, + { + "id": "fbaae95260474ed58ccbb05ac482089b" + }, + { + "id": "28b837ac304845bd961a46cedc9246af" + }, + { + "id": "691e3a24e5994c749ff11c6bc43b2d1c" + }, + { + "id": "12d5b51c793a43b1aa3b7e1bf5f4bc73" + }, + { + "id": "a3a3892296c74fc4b23b2475156901b1" + }, + { + "id": "8065f2e8639848f5b2f16a4f2877b7c2" + }, + { + "id": "b06e0d2be0a5465ea12ff2bf2b27d387" + }, + { + "id": "e50328dc70634863ac39580ef6f8dd5a" + }, + { + "id": "17b441db9b444435951d859e6e081f3b" + }, + { + "id": "0fc37a4bd8424ecf8eede56e956edb3a" + }, + { + "id": "09c769c7262e43eb95a52e874042c02a" + }, + { + "id": "2417ce75564b4b8eab1ddd755b10d962" + }, + { + "id": "8f4beaa8a7704eba93f753a0e7040165" + }, + { + "id": "6d8e99531264435badb67bf6ac209075" + }, + { + "id": "940ddc06648f4c9bac6b3bf5df402e5a" + }, + { + "id": "fa0fcf36e99644d0b51be95869d87adc" + }, + { + "id": "1882f20b777146608474ac0056d54a5e" + }, + { + "id": "439e908aa7c7422daf5632af7e0440ec" + }, + { + "id": "2d90c36ab1dd4724b1c108b2507316e4" + }, + { + "id": "efa5cc2f750c421a807bc4315db0dc1e" + }, + { + "id": "3b1c596e1720499ebe790d14f34a67af" + }, + { + "id": "c87234a3a1074b5ea5500114ad9aa16d" + }, + { + "id": "9d08ada791844519b7a4bb247d50c89c" + }, + { + "id": "74c45e35e5a740798fbd528ff08a70e3" + }, + { + "id": "650b683d39dd45c99ad7435fc7d33fe3" + }, + { + "id": "191ce59c97e143f7b4c33400913d3570" + }, + { + "id": "74441a0f7dae42ca9ee19d8be8c9b318" + }, + { + "id": "baef3cef84f14ed98655ac73293a56db" + }, + { + "id": "099a1ae148184bd5a93c9bc9889306e7" + }, + { + "id": "4fb4332a7a0e44e4b5655de1d7c2e5ac" + }, + { + "id": "cdf86d1982f64cb49d6595680b1c53b8" + }, + { + "id": "fa4cbba4ffed4f4e8ed4c148711cbc78" + }, + { + "id": "41c691d3b7e94ffd993647c7e531e94e" + }, + { + "id": "df43ea868a6d409ea67eb632fc9f285c" + }, + { + "id": "746448dcf441410db997a59f7137a2c8" + }, + { + "id": "3d0cad58c3c145f3b98d51c06a65a0c9" + }, + { + "id": "9cb4317831024d589622e2a3cf544360" + }, + { + "id": "802015dbc9f5406d8df3b468497f6553" + }, + { + "id": "5463f1d3fc124cadb2ad016076a699f5" + }, + { + "id": "8c77419b4e864778b965753f411bb469" + }, + { + "id": "2e112a7c9d97482fa0deb079c6c09172" + }, + { + "id": "f0c26f4f2e9f4d1d84b4f13cb201cbeb" + }, + { + "id": "231d7d9955d54f60a1f1ff4f0bf1c5b6" + }, + { + "id": "9b56a920d67a48e8b33932802d17d374" + }, + { + "id": "092a8aab433c4184bde9c8aa08789464" + }, + { + "id": "266a4337fa5742c8bbcc5aa79e571e85" + }, + { + "id": "d67cecf38f954800a7c651cca70bdc66" + }, + { + "id": "e634fa14bd4941ef96e1d21e796a9cd3" + }, + { + "id": "dbc29aef324e4c4f9f19082dcb1b5cb1" + }, + { + "id": "e688c5081a84488694b111a11a50ef7e" + }, + { + "id": "80b114d03dbb4598b5da648bbe174d0f" + }, + { + "id": "ce42d55ff5874c6380e018896b16e128" + }, + { + "id": "ef2b37212c98453390c9dca445d69de1" + }, + { + "id": "a66639dbb5ed4dfb9e65fe359ed8dc03" + }, + { + "id": "10eacaf72a4e4ad49cc43d3a710ee24a" + }, + { + "id": "b7e3d45f77bd4a4db768505fc4726592" + }, + { + "id": "0d1592f640f04ecb9c7add7164f87c5a" + }, + { + "id": "296a678ada814b09bdd0a53629f7f3f9" + }, + { + "id": "6e6fd6d62ce54031a4e36f88c2d5b63d" + }, + { + "id": "8d0dd71f94cc4abba255f3cdd6c2d940" + }, + { + "id": "7c26258ff99c4adeb4b9b35001005453" + }, + { + "id": "d13c24c99c704e31bbb3aaa585884d56" + }, + { + "id": "dbfb81f100dc474f8267e6f9ab0ae51c" + }, + { + "id": "3b4a5a2f617b43c8aa37b0890f576749" + }, + { + "id": "c49a8bf2d0ad4afd9b0bb0d03c1b9d96" + }, + { + "id": "fbfb4851dd494e44b9a21475d7e6dbaa" + }, + { + "id": "e0ba941f8e2c4fa6a0dea2b562b0e4be" + }, + { + "id": "7c9ec2b1cd8d4a8eaa6e1154e6fb40c4" + }, + { + "id": "4ae645d2721c4751b1e71c83870bbe16" + }, + { + "id": "c7bd32aee95f485d85d0b6749d542c51" + }, + { + "id": "20e140b44e0f4a6484e9cf776b09679b" + }, + { + "id": "5deb6bfcdd8c486d9f5f630309968763" + }, + { + "id": "18d1fcc3ed394b23ada799aad3beca90" + }, + { + "id": "74a655bcaead4857b92a49ec5f93ee23" + }, + { + "id": "bfec094ae9da4aaebd91713b39d3008e" + }, + { + "id": "80ebe13e36eb413a9acaace84648b36d" + }, + { + "id": "982e414b371248788c72732975949957" + }, + { + "id": "963ad06f3e314d3b99e0b6b9eaeaf2d8" + }, + { + "id": "90b6478bd7ae4145976c474dc42da80f" + }, + { + "id": "ea475dcd08d54da6896f56e330fa2a1c" + }, + { + "id": "64899cac91e5423fa3617bf576e6df4d" + }, + { + "id": "6231d57d040143e3bc56cafaabaa0975" + }, + { + "id": "3fdb66ab2b0a49318b053edda6633237" + }, + { + "id": "7dff080db1b8438e9ed737df1d7c32e3" + }, + { + "id": "878108ec7e9742a090175bd33390e2c5" + }, + { + "id": "3b2acfd62bdf4361aa4e51895d12df8b" + }, + { + "id": "0335847f4a44451db5a1b775cc019765" + }, + { + "id": "b6bfa34339c649398ae3b4540ed95fcf" + }, + { + "id": "82fd2c61b9a64b5f8971b28a1cfd8c1b" + }, + { + "id": "a9385b35a3a5492283aa461b93fdb361" + }, + { + "id": "847f117d4638487d92539d52eb07a999" + }, + { + "id": "de103586389245e7858b26b9b9a1ba57" + }, + { + "id": "82d95f6b3df64bb48668f7e98ddf5b9c" + }, + { + "id": "b1f6907b8f0f41ec82dc2eae7f84666b" + }, + { + "id": "1f232b1de1f34793b6557c8688bf4d9c" + }, + { + "id": "c16f718bd6554e998a95fe1d07623a13" + }, + { + "id": "dc2add9004fd40e69fcbd2eea80c513c" + }, + { + "id": "464e204e258340b69eca2b90451f06b8" + }, + { + "id": "c12fd75ea3db450bbfcabeb4f62f7e8f" + }, + { + "id": "5dbcadd728fa4956a4f739c1168fbf18" + }, + { + "id": "1730220302104d54ba2bb781ca219e52" + }, + { + "id": "ffcf25fe8cce4198bd483f2faaa17889" + }, + { + "id": "98ed5d627d0746e0aecb42eae39c14dc" + }, + { + "id": "30718db44c7f44a5b4b0c68842b1053b" + }, + { + "id": "4e90a93f4548458bb5d35ac9c767233b" + }, + { + "id": "dabccc32402a43a7ab9c82408f3603a1" + }, + { + "id": "80a8a1ea275944228e71fd0dfc74fb74" + }, + { + "id": "687434d75186430896ea636fec156e6d" + }, + { + "id": "28e5233352ba44ea9f7a8d4cf7f6b86a" + }, + { + "id": "b11499ee2a1f4a2b8fa8c9f561136430" + }, + { + "id": "6efcba5974464dcbaf5d9c5e9322bd9c" + }, + { + "id": "a1d460152c3840d98ed3bc45bf80d5c1" + }, + { + "id": "0d482f0e297544dcb2ce1e87e4c3b4b4" + }, + { + "id": "1a5baeb7a51d453d892ba70a9de2146d" + }, + { + "id": "ab84169c66b2415180563269295f53be" + }, + { + "id": "08c0e9678f1548aebeada27ab7b954e1" + }, + { + "id": "efd2d367aeec4cc4ae72cd3eaa040a0f" + }, + { + "id": "ae54626131ed42eaa55755f10ee64a76" + }, + { + "id": "1787cbe3e160427ca4e61931daf66e51" + }, + { + "id": "24143f01e7dd455e86c34a4a6f4ed3b9" + }, + { + "id": "5e92b869dd0b4ecbaf8694041891eb75" + }, + { + "id": "8d130f744b214055ad0c4cdce459a4b9" + }, + { + "id": "87913f6be2004e938ba60cda550bc675" + }, + { + "id": "68c7f892cbd94fc59ebe4ef1e1ad0c89" + }, + { + "id": "f7bf36628c7540da907dda84b48aac82" + }, + { + "id": "357af030b55046fc856e03fdac36032a" + }, + { + "id": "3a90f09909344dfdbb1a73f80c983a87" + }, + { + "id": "44fd76ce5f864cc2aef99c75cfd4904e" + }, + { + "id": "95e54b962ce1420d82c716ff428d84fe" + }, + { + "id": "8e22ba3b6fc34886a86d60dd1a1db025" + }, + { + "id": "162917ea12b24d0e874540feecf9b445" + }, + { + "id": "c8e87f5da41c47c7a3116bd1ed0cc34d" + }, + { + "id": "a3458bd73564491585df515468475a66" + }, + { + "id": "e27a4c1fb25c48809feb3cc438b5eb3c" + }, + { + "id": "f36338c39e2846959491ea48cbce07f6" + }, + { + "id": "eae01f064523415c95e57ff1e8dadc43" + }, + { + "id": "8b845d50f998467a898367107377c64b" + }, + { + "id": "a50ae9e7cb6a4f1e8dad07a9bbace810" + }, + { + "id": "5c2009e4f1384b27b4b4fe9422f048ed" + }, + { + "id": "0baaa0fe4fde4bae99bc6d2be230d525" + }, + { + "id": "d2563a0ca702457497995a6959ff1c61" + }, + { + "id": "b07871d2991d47e899cfa68a91f16b52" + }, + { + "id": "b745738e5b3c482aa2ae94ad5688246b" + }, + { + "id": "d66f1f0158054f638ff1b8acf3e18a8a" + }, + { + "id": "ede0705d606d4435a300e06beab2322f" + }, + { + "id": "e9d415f928644b59b5ed19b7011f627d" + }, + { + "id": "1a78a8cf476c4a0dbf551558da1d879b" + }, + { + "id": "ea7c66d5dae241eda155763541749cd1" + }, + { + "id": "78ae271549144680a42139693d1117e9" + }, + { + "id": "7eb9a69a23b8469b9f20f1915abff3be" + }, + { + "id": "db73ca2786684b9e92deb1fcc796effd" + }, + { + "id": "a876fb1accf64f4eb1b20b4a0ee089db" + }, + { + "id": "71d6dabd553c471b98debe941341a4c0" + }, + { + "id": "1c6add1cbd464f34b6e90485780f1c3e" + }, + { + "id": "5e9b0b44825e44d89ba0501c3b819685" + }, + { + "id": "4cfc841cb81e4c90822cfb7c0ec9f86e" + }, + { + "id": "2933df89dd6146c791eac1a85f0d02c8" + }, + { + "id": "5af06dca120a46889964109e94fff0a0" + }, + { + "id": "77970683cb6444388e5cfdd743784db6" + }, + { + "id": "3e40d5d91615427d976a56932be3c955" + }, + { + "id": "70bc65e0c17041729a4eabe86ff36d53" + }, + { + "id": "b913e55670474b69a70736d314822155" + }, + { + "id": "ada9cbe2dd8e48deb55171645e47a24a" + }, + { + "id": "fba813b8837249cbb39a1996dc657ef0" + }, + { + "id": "f7ecc32a212e4c3b8471ce26c32f3135" + }, + { + "id": "015641bf588b43aebba1aee07a8afde0" + }, + { + "id": "bcecb67a14f34fc6896bc5d65c58405a" + } +] diff --git a/data/spreadsheetbench_id_split/split_manifest.json b/data/spreadsheetbench_id_split/split_manifest.json new file mode 100644 index 0000000..03a558c --- /dev/null +++ b/data/spreadsheetbench_id_split/split_manifest.json @@ -0,0 +1,24 @@ +{ + "benchmark": "SpreadsheetBench", + "manifest_type": "id_split", + "source_repo": "KAKA22/SpreadsheetBench", + "source_repo_type": "dataset", + "source_url": "https://huggingface.co/datasets/KAKA22/SpreadsheetBench", + "source_revision": "ab0b742b0fc95b946f212d80ac7771b5531272e4", + "source_file": "spreadsheetbench_verified_400.tar.gz", + "source_split_name": "spreadsheetbench_split", + "counts": { + "train": 80, + "val": 40, + "test": 280 + }, + "item_fields": [ + "id", + "spreadsheet_path", + "instruction_type" + ], + "notes": [ + "This is a split manifest, not the full SpreadsheetBench payload.", + "Materialize full task JSON rows plus spreadsheet files from SpreadsheetBench Verified 400 before evaluation." + ] +} diff --git a/data/spreadsheetbench_id_split/test/items.json b/data/spreadsheetbench_id_split/test/items.json new file mode 100644 index 0000000..08c78ee --- /dev/null +++ b/data/spreadsheetbench_id_split/test/items.json @@ -0,0 +1,1402 @@ +[ + { + "id": "52532", + "spreadsheet_path": "spreadsheet/52532", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "41-47", + "spreadsheet_path": "spreadsheet/41-47", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "59794", + "spreadsheet_path": "spreadsheet/59794", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "42515", + "spreadsheet_path": "spreadsheet/42515", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "493-5", + "spreadsheet_path": "spreadsheet/493-5", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "41969", + "spreadsheet_path": "spreadsheet/41969", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "49237", + "spreadsheet_path": "spreadsheet/49237", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "516-46", + "spreadsheet_path": "spreadsheet/516-46", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "290-27", + "spreadsheet_path": "spreadsheet/290-27", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "32562", + "spreadsheet_path": "spreadsheet/32562", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "545-35", + "spreadsheet_path": "spreadsheet/545-35", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "32293", + "spreadsheet_path": "spreadsheet/32293", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "42181", + "spreadsheet_path": "spreadsheet/42181", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "46240", + "spreadsheet_path": "spreadsheet/46240", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "55817", + "spreadsheet_path": "spreadsheet/55817", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "59358", + "spreadsheet_path": "spreadsheet/59358", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "44389", + "spreadsheet_path": "spreadsheet/44389", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "58701", + "spreadsheet_path": "spreadsheet/58701", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "50193", + "spreadsheet_path": "spreadsheet/50193", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "55976", + "spreadsheet_path": "spreadsheet/55976", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "56378", + "spreadsheet_path": "spreadsheet/56378", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "438-18", + "spreadsheet_path": "spreadsheet/438-18", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "50952", + "spreadsheet_path": "spreadsheet/50952", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "408-5", + "spreadsheet_path": "spreadsheet/408-5", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "56953", + "spreadsheet_path": "spreadsheet/56953", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "54925", + "spreadsheet_path": "spreadsheet/54925", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "448-11", + "spreadsheet_path": "spreadsheet/448-11", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "52917", + "spreadsheet_path": "spreadsheet/52917", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "382-29", + "spreadsheet_path": "spreadsheet/382-29", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "45707", + "spreadsheet_path": "spreadsheet/45707", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "42526", + "spreadsheet_path": "spreadsheet/42526", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "41978", + "spreadsheet_path": "spreadsheet/41978", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "146-49", + "spreadsheet_path": "spreadsheet/146-49", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "440-24", + "spreadsheet_path": "spreadsheet/440-24", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "46167", + "spreadsheet_path": "spreadsheet/46167", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "230-16", + "spreadsheet_path": "spreadsheet/230-16", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "54675", + "spreadsheet_path": "spreadsheet/54675", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "41601", + "spreadsheet_path": "spreadsheet/41601", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "41410", + "spreadsheet_path": "spreadsheet/41410", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "50521", + "spreadsheet_path": "spreadsheet/50521", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "18645", + "spreadsheet_path": "spreadsheet/18645", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "52807", + "spreadsheet_path": "spreadsheet/52807", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "52964", + "spreadsheet_path": "spreadsheet/52964", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "50250", + "spreadsheet_path": "spreadsheet/50250", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "203-15", + "spreadsheet_path": "spreadsheet/203-15", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "37900", + "spreadsheet_path": "spreadsheet/37900", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "48975", + "spreadsheet_path": "spreadsheet/48975", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "367-23", + "spreadsheet_path": "spreadsheet/367-23", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "60-7", + "spreadsheet_path": "spreadsheet/60-7", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "55708", + "spreadsheet_path": "spreadsheet/55708", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "49196", + "spreadsheet_path": "spreadsheet/49196", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "57693", + "spreadsheet_path": "spreadsheet/57693", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "50051", + "spreadsheet_path": "spreadsheet/50051", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "290-1", + "spreadsheet_path": "spreadsheet/290-1", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "57262", + "spreadsheet_path": "spreadsheet/57262", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "41589", + "spreadsheet_path": "spreadsheet/41589", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "49300", + "spreadsheet_path": "spreadsheet/49300", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "37554", + "spreadsheet_path": "spreadsheet/37554", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "39515", + "spreadsheet_path": "spreadsheet/39515", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "37086", + "spreadsheet_path": "spreadsheet/37086", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "36277", + "spreadsheet_path": "spreadsheet/36277", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "374-18", + "spreadsheet_path": "spreadsheet/374-18", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "50971", + "spreadsheet_path": "spreadsheet/50971", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "37229", + "spreadsheet_path": "spreadsheet/37229", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "57033", + "spreadsheet_path": "spreadsheet/57033", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "477-45", + "spreadsheet_path": "spreadsheet/477-45", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "531-18", + "spreadsheet_path": "spreadsheet/531-18", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "38074", + "spreadsheet_path": "spreadsheet/38074", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "56599", + "spreadsheet_path": "spreadsheet/56599", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "31746", + "spreadsheet_path": "spreadsheet/31746", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "51556", + "spreadsheet_path": "spreadsheet/51556", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "48924", + "spreadsheet_path": "spreadsheet/48924", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "423-16", + "spreadsheet_path": "spreadsheet/423-16", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "43213", + "spreadsheet_path": "spreadsheet/43213", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "49801", + "spreadsheet_path": "spreadsheet/49801", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "54717", + "spreadsheet_path": "spreadsheet/54717", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "58147", + "spreadsheet_path": "spreadsheet/58147", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "54474", + "spreadsheet_path": "spreadsheet/54474", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "56921", + "spreadsheet_path": "spreadsheet/56921", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "39190", + "spreadsheet_path": "spreadsheet/39190", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "168-17", + "spreadsheet_path": "spreadsheet/168-17", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "49945", + "spreadsheet_path": "spreadsheet/49945", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "32093", + "spreadsheet_path": "spreadsheet/32093", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "17111", + "spreadsheet_path": "spreadsheet/17111", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "120-24", + "spreadsheet_path": "spreadsheet/120-24", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "97-36", + "spreadsheet_path": "spreadsheet/97-36", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "59639", + "spreadsheet_path": "spreadsheet/59639", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "567-21", + "spreadsheet_path": "spreadsheet/567-21", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "48643", + "spreadsheet_path": "spreadsheet/48643", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "1925", + "spreadsheet_path": "spreadsheet/1925", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "40959", + "spreadsheet_path": "spreadsheet/40959", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "51431", + "spreadsheet_path": "spreadsheet/51431", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "37462", + "spreadsheet_path": "spreadsheet/37462", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "53161", + "spreadsheet_path": "spreadsheet/53161", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "283-32", + "spreadsheet_path": "spreadsheet/283-32", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "1563", + "spreadsheet_path": "spreadsheet/1563", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "599-9", + "spreadsheet_path": "spreadsheet/599-9", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "50682", + "spreadsheet_path": "spreadsheet/50682", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "304-35", + "spreadsheet_path": "spreadsheet/304-35", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "297-42", + "spreadsheet_path": "spreadsheet/297-42", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "40478", + "spreadsheet_path": "spreadsheet/40478", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "9391", + "spreadsheet_path": "spreadsheet/9391", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "269-44", + "spreadsheet_path": "spreadsheet/269-44", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "208-20", + "spreadsheet_path": "spreadsheet/208-20", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "45063", + "spreadsheet_path": "spreadsheet/45063", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "54590", + "spreadsheet_path": "spreadsheet/54590", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "3002", + "spreadsheet_path": "spreadsheet/3002", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "9448", + "spreadsheet_path": "spreadsheet/9448", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "388-47", + "spreadsheet_path": "spreadsheet/388-47", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "51354", + "spreadsheet_path": "spreadsheet/51354", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "24-23", + "spreadsheet_path": "spreadsheet/24-23", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "52541", + "spreadsheet_path": "spreadsheet/52541", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "353-29", + "spreadsheet_path": "spreadsheet/353-29", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "45300", + "spreadsheet_path": "spreadsheet/45300", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "13-1", + "spreadsheet_path": "spreadsheet/13-1", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "58942", + "spreadsheet_path": "spreadsheet/58942", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "45944", + "spreadsheet_path": "spreadsheet/45944", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "52305", + "spreadsheet_path": "spreadsheet/52305", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "52575", + "spreadsheet_path": "spreadsheet/52575", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "52220", + "spreadsheet_path": "spreadsheet/52220", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "91-34", + "spreadsheet_path": "spreadsheet/91-34", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "40757", + "spreadsheet_path": "spreadsheet/40757", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "399-14", + "spreadsheet_path": "spreadsheet/399-14", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "82-38", + "spreadsheet_path": "spreadsheet/82-38", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "156-14", + "spreadsheet_path": "spreadsheet/156-14", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "45896", + "spreadsheet_path": "spreadsheet/45896", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "33157", + "spreadsheet_path": "spreadsheet/33157", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "48257", + "spreadsheet_path": "spreadsheet/48257", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "4714", + "spreadsheet_path": "spreadsheet/4714", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "302-1", + "spreadsheet_path": "spreadsheet/302-1", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "47842", + "spreadsheet_path": "spreadsheet/47842", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "36764", + "spreadsheet_path": "spreadsheet/36764", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "455-35", + "spreadsheet_path": "spreadsheet/455-35", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "14240", + "spreadsheet_path": "spreadsheet/14240", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "33722", + "spreadsheet_path": "spreadsheet/33722", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "387-16", + "spreadsheet_path": "spreadsheet/387-16", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "35739", + "spreadsheet_path": "spreadsheet/35739", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "44296", + "spreadsheet_path": "spreadsheet/44296", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "17-35", + "spreadsheet_path": "spreadsheet/17-35", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "54513", + "spreadsheet_path": "spreadsheet/54513", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "343-20", + "spreadsheet_path": "spreadsheet/343-20", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "9111", + "spreadsheet_path": "spreadsheet/9111", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "56419", + "spreadsheet_path": "spreadsheet/56419", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "59884", + "spreadsheet_path": "spreadsheet/59884", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "44017", + "spreadsheet_path": "spreadsheet/44017", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "53449", + "spreadsheet_path": "spreadsheet/53449", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "191-40", + "spreadsheet_path": "spreadsheet/191-40", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "28-7", + "spreadsheet_path": "spreadsheet/28-7", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "31202", + "spreadsheet_path": "spreadsheet/31202", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "18935", + "spreadsheet_path": "spreadsheet/18935", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "51359", + "spreadsheet_path": "spreadsheet/51359", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "51586", + "spreadsheet_path": "spreadsheet/51586", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "547-18", + "spreadsheet_path": "spreadsheet/547-18", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "44913", + "spreadsheet_path": "spreadsheet/44913", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "45937", + "spreadsheet_path": "spreadsheet/45937", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "334-11", + "spreadsheet_path": "spreadsheet/334-11", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "177-6", + "spreadsheet_path": "spreadsheet/177-6", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "384-4", + "spreadsheet_path": "spreadsheet/384-4", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "130-9", + "spreadsheet_path": "spreadsheet/130-9", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "32789", + "spreadsheet_path": "spreadsheet/32789", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "36191", + "spreadsheet_path": "spreadsheet/36191", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "183-8", + "spreadsheet_path": "spreadsheet/183-8", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "59160", + "spreadsheet_path": "spreadsheet/59160", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "56225", + "spreadsheet_path": "spreadsheet/56225", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "54196", + "spreadsheet_path": "spreadsheet/54196", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "7665", + "spreadsheet_path": "spreadsheet/7665", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "38537", + "spreadsheet_path": "spreadsheet/38537", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "41420", + "spreadsheet_path": "spreadsheet/41420", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "56915", + "spreadsheet_path": "spreadsheet/56915", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "15380", + "spreadsheet_path": "spreadsheet/15380", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "59224", + "spreadsheet_path": "spreadsheet/59224", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "57590", + "spreadsheet_path": "spreadsheet/57590", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "209-30", + "spreadsheet_path": "spreadsheet/209-30", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "15387", + "spreadsheet_path": "spreadsheet/15387", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "53167", + "spreadsheet_path": "spreadsheet/53167", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "34210", + "spreadsheet_path": "spreadsheet/34210", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "50486", + "spreadsheet_path": "spreadsheet/50486", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "50811", + "spreadsheet_path": "spreadsheet/50811", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "45738", + "spreadsheet_path": "spreadsheet/45738", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "488-14", + "spreadsheet_path": "spreadsheet/488-14", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "61-4", + "spreadsheet_path": "spreadsheet/61-4", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "38969", + "spreadsheet_path": "spreadsheet/38969", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "54242", + "spreadsheet_path": "spreadsheet/54242", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "47933", + "spreadsheet_path": "spreadsheet/47933", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "57989", + "spreadsheet_path": "spreadsheet/57989", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "535-20", + "spreadsheet_path": "spreadsheet/535-20", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "469-9", + "spreadsheet_path": "spreadsheet/469-9", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "48608", + "spreadsheet_path": "spreadsheet/48608", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "55468", + "spreadsheet_path": "spreadsheet/55468", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "44266", + "spreadsheet_path": "spreadsheet/44266", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "38985", + "spreadsheet_path": "spreadsheet/38985", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "59902", + "spreadsheet_path": "spreadsheet/59902", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "16511", + "spreadsheet_path": "spreadsheet/16511", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "54274", + "spreadsheet_path": "spreadsheet/54274", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "59511", + "spreadsheet_path": "spreadsheet/59511", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "41265", + "spreadsheet_path": "spreadsheet/41265", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "374-31", + "spreadsheet_path": "spreadsheet/374-31", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "52233", + "spreadsheet_path": "spreadsheet/52233", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "79-7", + "spreadsheet_path": "spreadsheet/79-7", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "38823", + "spreadsheet_path": "spreadsheet/38823", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "486-17", + "spreadsheet_path": "spreadsheet/486-17", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "49667", + "spreadsheet_path": "spreadsheet/49667", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "50796", + "spreadsheet_path": "spreadsheet/50796", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "359-21", + "spreadsheet_path": "spreadsheet/359-21", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "50526", + "spreadsheet_path": "spreadsheet/50526", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "13284", + "spreadsheet_path": "spreadsheet/13284", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "55977", + "spreadsheet_path": "spreadsheet/55977", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "58723", + "spreadsheet_path": "spreadsheet/58723", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "157-4", + "spreadsheet_path": "spreadsheet/157-4", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "58904", + "spreadsheet_path": "spreadsheet/58904", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "56920", + "spreadsheet_path": "spreadsheet/56920", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "41691", + "spreadsheet_path": "spreadsheet/41691", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "534-26", + "spreadsheet_path": "spreadsheet/534-26", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "341-40", + "spreadsheet_path": "spreadsheet/341-40", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "73-45", + "spreadsheet_path": "spreadsheet/73-45", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "409-45", + "spreadsheet_path": "spreadsheet/409-45", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "49857", + "spreadsheet_path": "spreadsheet/49857", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "50324", + "spreadsheet_path": "spreadsheet/50324", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "53117", + "spreadsheet_path": "spreadsheet/53117", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "53994", + "spreadsheet_path": "spreadsheet/53994", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "42198", + "spreadsheet_path": "spreadsheet/42198", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "42930", + "spreadsheet_path": "spreadsheet/42930", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "84-40", + "spreadsheet_path": "spreadsheet/84-40", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "51262", + "spreadsheet_path": "spreadsheet/51262", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "236-22", + "spreadsheet_path": "spreadsheet/236-22", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "41348", + "spreadsheet_path": "spreadsheet/41348", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "15671", + "spreadsheet_path": "spreadsheet/15671", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "333-29", + "spreadsheet_path": "spreadsheet/333-29", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "38703", + "spreadsheet_path": "spreadsheet/38703", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "52050", + "spreadsheet_path": "spreadsheet/52050", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "49036", + "spreadsheet_path": "spreadsheet/49036", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "50088", + "spreadsheet_path": "spreadsheet/50088", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "147-48", + "spreadsheet_path": "spreadsheet/147-48", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "51289", + "spreadsheet_path": "spreadsheet/51289", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "57612", + "spreadsheet_path": "spreadsheet/57612", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "59129", + "spreadsheet_path": "spreadsheet/59129", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "48969", + "spreadsheet_path": "spreadsheet/48969", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "496-15", + "spreadsheet_path": "spreadsheet/496-15", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "341-14", + "spreadsheet_path": "spreadsheet/341-14", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "47827", + "spreadsheet_path": "spreadsheet/47827", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "48982", + "spreadsheet_path": "spreadsheet/48982", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "58032", + "spreadsheet_path": "spreadsheet/58032", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "42902", + "spreadsheet_path": "spreadsheet/42902", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "353-6", + "spreadsheet_path": "spreadsheet/353-6", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "23-24", + "spreadsheet_path": "spreadsheet/23-24", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "39667", + "spreadsheet_path": "spreadsheet/39667", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "50631", + "spreadsheet_path": "spreadsheet/50631", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "11276", + "spreadsheet_path": "spreadsheet/11276", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "510-3", + "spreadsheet_path": "spreadsheet/510-3", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "3911", + "spreadsheet_path": "spreadsheet/3911", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "58687", + "spreadsheet_path": "spreadsheet/58687", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "43657", + "spreadsheet_path": "spreadsheet/43657", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "55085", + "spreadsheet_path": "spreadsheet/55085", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "55260", + "spreadsheet_path": "spreadsheet/55260", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "416-27", + "spreadsheet_path": "spreadsheet/416-27", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "44628", + "spreadsheet_path": "spreadsheet/44628", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "269-43", + "spreadsheet_path": "spreadsheet/269-43", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "51680", + "spreadsheet_path": "spreadsheet/51680", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "35747", + "spreadsheet_path": "spreadsheet/35747", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "547-43", + "spreadsheet_path": "spreadsheet/547-43", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "496-34", + "spreadsheet_path": "spreadsheet/496-34", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "142-19", + "spreadsheet_path": "spreadsheet/142-19", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "279-23", + "spreadsheet_path": "spreadsheet/279-23", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "280-17", + "spreadsheet_path": "spreadsheet/280-17", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "5835", + "spreadsheet_path": "spreadsheet/5835", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "50683", + "spreadsheet_path": "spreadsheet/50683", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "109-21", + "spreadsheet_path": "spreadsheet/109-21", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "42216", + "spreadsheet_path": "spreadsheet/42216", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "59734", + "spreadsheet_path": "spreadsheet/59734", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "57232", + "spreadsheet_path": "spreadsheet/57232", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "160-6", + "spreadsheet_path": "spreadsheet/160-6", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "57117", + "spreadsheet_path": "spreadsheet/57117", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "188-39", + "spreadsheet_path": "spreadsheet/188-39", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "524-31", + "spreadsheet_path": "spreadsheet/524-31", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "12307", + "spreadsheet_path": "spreadsheet/12307", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "39432", + "spreadsheet_path": "spreadsheet/39432", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "57354", + "spreadsheet_path": "spreadsheet/57354", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "267-21", + "spreadsheet_path": "spreadsheet/267-21", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "178-22", + "spreadsheet_path": "spreadsheet/178-22", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "54667", + "spreadsheet_path": "spreadsheet/54667", + "instruction_type": "Cell-Level Manipulation" + } +] diff --git a/data/spreadsheetbench_id_split/train/items.json b/data/spreadsheetbench_id_split/train/items.json new file mode 100644 index 0000000..bcc3376 --- /dev/null +++ b/data/spreadsheetbench_id_split/train/items.json @@ -0,0 +1,402 @@ +[ + { + "id": "32438", + "spreadsheet_path": "spreadsheet/32438", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "398-14", + "spreadsheet_path": "spreadsheet/398-14", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "47766", + "spreadsheet_path": "spreadsheet/47766", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "48365", + "spreadsheet_path": "spreadsheet/48365", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "32255", + "spreadsheet_path": "spreadsheet/32255", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "10747", + "spreadsheet_path": "spreadsheet/10747", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "50916", + "spreadsheet_path": "spreadsheet/50916", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "577-40", + "spreadsheet_path": "spreadsheet/577-40", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "35742", + "spreadsheet_path": "spreadsheet/35742", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "46121", + "spreadsheet_path": "spreadsheet/46121", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "51090", + "spreadsheet_path": "spreadsheet/51090", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "51249", + "spreadsheet_path": "spreadsheet/51249", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "82-30", + "spreadsheet_path": "spreadsheet/82-30", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "56274", + "spreadsheet_path": "spreadsheet/56274", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "57445", + "spreadsheet_path": "spreadsheet/57445", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "46646", + "spreadsheet_path": "spreadsheet/46646", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "105-24", + "spreadsheet_path": "spreadsheet/105-24", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "6239", + "spreadsheet_path": "spreadsheet/6239", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "414-20", + "spreadsheet_path": "spreadsheet/414-20", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "165-23", + "spreadsheet_path": "spreadsheet/165-23", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "40892", + "spreadsheet_path": "spreadsheet/40892", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "48745", + "spreadsheet_path": "spreadsheet/48745", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "32612", + "spreadsheet_path": "spreadsheet/32612", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "325-44", + "spreadsheet_path": "spreadsheet/325-44", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "262-17", + "spreadsheet_path": "spreadsheet/262-17", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "141-20", + "spreadsheet_path": "spreadsheet/141-20", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "52216", + "spreadsheet_path": "spreadsheet/52216", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "22-47", + "spreadsheet_path": "spreadsheet/22-47", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "55421", + "spreadsheet_path": "spreadsheet/55421", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "56427", + "spreadsheet_path": "spreadsheet/56427", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "36097", + "spreadsheet_path": "spreadsheet/36097", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "32902", + "spreadsheet_path": "spreadsheet/32902", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "32023", + "spreadsheet_path": "spreadsheet/32023", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "1818", + "spreadsheet_path": "spreadsheet/1818", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "170-13", + "spreadsheet_path": "spreadsheet/170-13", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "66-24", + "spreadsheet_path": "spreadsheet/66-24", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "58949", + "spreadsheet_path": "spreadsheet/58949", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "42354", + "spreadsheet_path": "spreadsheet/42354", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "194-19", + "spreadsheet_path": "spreadsheet/194-19", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "31915", + "spreadsheet_path": "spreadsheet/31915", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "58499", + "spreadsheet_path": "spreadsheet/58499", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "45372", + "spreadsheet_path": "spreadsheet/45372", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "11842", + "spreadsheet_path": "spreadsheet/11842", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "57558", + "spreadsheet_path": "spreadsheet/57558", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "472-15", + "spreadsheet_path": "spreadsheet/472-15", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "55060", + "spreadsheet_path": "spreadsheet/55060", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "31011", + "spreadsheet_path": "spreadsheet/31011", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "408-39", + "spreadsheet_path": "spreadsheet/408-39", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "54085", + "spreadsheet_path": "spreadsheet/54085", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "39903", + "spreadsheet_path": "spreadsheet/39903", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "48983", + "spreadsheet_path": "spreadsheet/48983", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "108-24", + "spreadsheet_path": "spreadsheet/108-24", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "58484", + "spreadsheet_path": "spreadsheet/58484", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "118-50", + "spreadsheet_path": "spreadsheet/118-50", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "10452", + "spreadsheet_path": "spreadsheet/10452", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "39931", + "spreadsheet_path": "spreadsheet/39931", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "3413", + "spreadsheet_path": "spreadsheet/3413", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "247-24", + "spreadsheet_path": "spreadsheet/247-24", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "56786", + "spreadsheet_path": "spreadsheet/56786", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "55965", + "spreadsheet_path": "spreadsheet/55965", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "379-36", + "spreadsheet_path": "spreadsheet/379-36", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "58109", + "spreadsheet_path": "spreadsheet/58109", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "433-47", + "spreadsheet_path": "spreadsheet/433-47", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "192-22", + "spreadsheet_path": "spreadsheet/192-22", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "49333", + "spreadsheet_path": "spreadsheet/49333", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "493-18", + "spreadsheet_path": "spreadsheet/493-18", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "54638", + "spreadsheet_path": "spreadsheet/54638", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "34033", + "spreadsheet_path": "spreadsheet/34033", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "30930", + "spreadsheet_path": "spreadsheet/30930", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "585-41", + "spreadsheet_path": "spreadsheet/585-41", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "32337", + "spreadsheet_path": "spreadsheet/32337", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "55427", + "spreadsheet_path": "spreadsheet/55427", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "263-1", + "spreadsheet_path": "spreadsheet/263-1", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "254-34", + "spreadsheet_path": "spreadsheet/254-34", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "57113", + "spreadsheet_path": "spreadsheet/57113", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "57743", + "spreadsheet_path": "spreadsheet/57743", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "43589", + "spreadsheet_path": "spreadsheet/43589", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "250-20", + "spreadsheet_path": "spreadsheet/250-20", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "48080", + "spreadsheet_path": "spreadsheet/48080", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "370-43", + "spreadsheet_path": "spreadsheet/370-43", + "instruction_type": "Sheet-Level Manipulation" + } +] diff --git a/data/spreadsheetbench_id_split/val/items.json b/data/spreadsheetbench_id_split/val/items.json new file mode 100644 index 0000000..a918756 --- /dev/null +++ b/data/spreadsheetbench_id_split/val/items.json @@ -0,0 +1,202 @@ +[ + { + "id": "45635", + "spreadsheet_path": "spreadsheet/45635", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "560-12", + "spreadsheet_path": "spreadsheet/560-12", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "55049", + "spreadsheet_path": "spreadsheet/55049", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "9569", + "spreadsheet_path": "spreadsheet/9569", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "7902", + "spreadsheet_path": "spreadsheet/7902", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "227-40", + "spreadsheet_path": "spreadsheet/227-40", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "463-17", + "spreadsheet_path": "spreadsheet/463-17", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "54144", + "spreadsheet_path": "spreadsheet/54144", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "80-42", + "spreadsheet_path": "spreadsheet/80-42", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "2768", + "spreadsheet_path": "spreadsheet/2768", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "37456", + "spreadsheet_path": "spreadsheet/37456", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "12864", + "spreadsheet_path": "spreadsheet/12864", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "55979", + "spreadsheet_path": "spreadsheet/55979", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "48620", + "spreadsheet_path": "spreadsheet/48620", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "48588", + "spreadsheet_path": "spreadsheet/48588", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "395-36", + "spreadsheet_path": "spreadsheet/395-36", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "382-10", + "spreadsheet_path": "spreadsheet/382-10", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "59595", + "spreadsheet_path": "spreadsheet/59595", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "53383", + "spreadsheet_path": "spreadsheet/53383", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "48921", + "spreadsheet_path": "spreadsheet/48921", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "416-15", + "spreadsheet_path": "spreadsheet/416-15", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "47798", + "spreadsheet_path": "spreadsheet/47798", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "56563", + "spreadsheet_path": "spreadsheet/56563", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "46897", + "spreadsheet_path": "spreadsheet/46897", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "9726", + "spreadsheet_path": "spreadsheet/9726", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "50768", + "spreadsheet_path": "spreadsheet/50768", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "51-12", + "spreadsheet_path": "spreadsheet/51-12", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "31628", + "spreadsheet_path": "spreadsheet/31628", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "39046", + "spreadsheet_path": "spreadsheet/39046", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "8942", + "spreadsheet_path": "spreadsheet/8942", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "48527", + "spreadsheet_path": "spreadsheet/48527", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "59196", + "spreadsheet_path": "spreadsheet/59196", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "6698", + "spreadsheet_path": "spreadsheet/6698", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "43436", + "spreadsheet_path": "spreadsheet/43436", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "38462", + "spreadsheet_path": "spreadsheet/38462", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "402-43", + "spreadsheet_path": "spreadsheet/402-43", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "267-18", + "spreadsheet_path": "spreadsheet/267-18", + "instruction_type": "Sheet-Level Manipulation" + }, + { + "id": "37378", + "spreadsheet_path": "spreadsheet/37378", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "53647", + "spreadsheet_path": "spreadsheet/53647", + "instruction_type": "Cell-Level Manipulation" + }, + { + "id": "142-12", + "spreadsheet_path": "spreadsheet/142-12", + "instruction_type": "Sheet-Level Manipulation" + } +] diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 0000000..818a67e --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1,69 @@ +# Contributing to SkillOpt + +Thank you for your interest in contributing to SkillOpt! This guide covers how to get started. + +## Development Setup + +```bash +git clone https://github.com/microsoft/SkillOpt.git +cd SkillOpt +pip install -e ".[dev]" +``` + +## Ways to Contribute + +### 🐛 Bug Reports + +Open an issue with: +- Steps to reproduce +- Expected vs actual behavior +- Config file used (sanitize API keys) +- Python version and OS + +### 🔧 New Benchmark + +See [Add a New Benchmark](guide/new-benchmark.md) for the implementation guide. + +**Checklist:** +- [ ] Data loader in `skillopt/envs//loader.py` +- [ ] Environment adapter in `skillopt/envs//env.py` +- [ ] Config file in `configs//default.yaml` +- [ ] Registration in `skillopt/envs/__init__.py` +- [ ] Documentation page in `docs/` + +### 🤖 New Model Backend + +See [Add a New Model Backend](guide/new-backend.md) for the implementation guide. + +**Checklist:** +- [ ] Backend in `skillopt/model/.py` +- [ ] Registration in `skillopt/model/__init__.py` +- [ ] API key entry in `.env.example` +- [ ] Documentation update + +### 📝 Documentation + +Documentation is built with MkDocs Material: + +```bash +pip install -e ".[docs]" +mkdocs serve # Preview at http://localhost:8000 +``` + +## Code Style + +- Follow existing patterns in the codebase +- Use type hints for function signatures +- Keep docstrings concise + +## Pull Request Process + +1. Fork the repository +2. Create a feature branch: `git checkout -b feature/my-benchmark` +3. Make your changes +4. Test with an existing benchmark config +5. Submit a PR with a clear description + +## License + +By contributing, you agree that your contributions will be licensed under the MIT License. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md new file mode 100644 index 0000000..57fffae --- /dev/null +++ b/docs/guide/configuration.md @@ -0,0 +1,139 @@ +# Configuration Guide + +SkillOpt uses YAML configuration files with a hierarchical override system. + +## Config Structure + +``` +configs/ +├── _base_/ +│ └── default.yaml # Global defaults +├── searchqa/ +│ └── default.yaml # SearchQA overrides +├── docvqa/ +│ └── default.yaml # DocVQA overrides +└── alfworld/ + └── default.yaml # ALFWorld overrides +``` + +Benchmark configs inherit from `_base_/default.yaml` and override specific values. + +## Key Parameters + +### Model + +```yaml +model: + backend: azure_openai # azure_openai | openai_chat | claude_code_exec | qwen + optimizer: gpt-5.5 # Optimizer model (for reflection) + target: gpt-5.5 # Target model (for rollout) +``` + +### Training + +```yaml +train: + num_epochs: 4 # Number of training epochs + batch_size: 40 # Tasks per step (batch size) + accumulation: 1 # Gradient accumulation + seed: 42 +``` + +### Gradient (Reflection) + +```yaml +gradient: + minibatch_size: 8 # Reflect minibatch size + analyst_workers: 16 # Parallel reflection workers + max_analyst_rounds: 3 # Max rounds of analyst reflection + failure_only: false # Only reflect on failures +``` + +### Optimizer + +```yaml +optimizer: + learning_rate: 4 # Max edits per step (edit budget) + min_learning_rate: 2 # Min edits for decay schedulers + lr_scheduler: cosine # constant | linear | cosine | autonomous + use_slow_update: true # Momentum-like blending at epoch boundary + slow_update_samples: 20 # Samples for slow update evaluation + use_meta_skill: true # Cross-epoch strategy memory +``` + +### Skill-Aware Reflection (optional, off by default) + +EmbodiSkill-style failure routing: the failure analyst classifies each +failure pattern as **SKILL_DEFECT** (the rule is wrong or missing → normal +gated body edit) or **EXECUTION_LAPSE** (a valid rule exists but was not +followed → a short reminder appended to a protected appendix region inside +the skill that step-level edits can never modify). + +```yaml +optimizer: + use_skill_aware_reflection: false # Master switch (default off = baseline-identical) + skill_aware_appendix_source: both # both | failure_only (paper-faithful S_app) + skill_aware_consolidate_threshold: 0 # >0: LLM-compact the appendix past N notes (experimental) +``` + +Notes: + +- The switch is resolved process-wide from the config + (`configure_skill_aware_reflection`), so it applies to every benchmark + with no per-adapter wiring. +- `failure_only` restricts appendix notes to the failure analyst, matching + the original S_app formulation; `both` additionally lets the success + analyst re-emphasize existing rules. +- Appendix notes bypass the validation gate by design and accumulate with + order-preserving dedup; lapse-only steps (no body edits) still flush + their notes. +- Not supported together with `skill_update_mode=rewrite_from_suggestions` + or the full-rewrite modes: whole-document rewrites can drop the appendix + region. + +### Evaluation + +```yaml +evaluation: + use_gate: true # Validation gating (accept/reject updates) + eval_test: true # Run test evaluation after training +``` + +### Environment (Data) + +```yaml +env: + name: searchqa # Benchmark name + split_mode: ratio # ratio | split_dir + split_ratio: "2:1:7" # train:val:test ratio + data_path: "" # Path to dataset + exec_timeout: 120 # Per-task timeout (seconds) +``` + +## CLI Overrides + +Override any config value from the command line: + +```bash +python scripts/train.py \ + --config configs/searchqa/default.yaml \ + optimizer.learning_rate=16 \ + optimizer.lr_scheduler=linear \ + gradient.analyst_workers=8 +``` + +## Environment Variables + +Model credentials are loaded from environment variables: + +| Variable | Backend | Description | +|---|---|---| +| `AZURE_OPENAI_ENDPOINT` | azure_openai | Azure resource endpoint | +| `AZURE_OPENAI_API_KEY` | azure_openai | Azure API key | +| `OPENAI_API_KEY` | openai | OpenAI API key | +| `ANTHROPIC_API_KEY` | claude | Anthropic API key | +| `QWEN_API_BASE` | qwen | Local Qwen vLLM endpoint | + +## Full Reference + +See [Configuration Reference](../reference/config.md) for the complete parameter list. diff --git a/docs/guide/dl-analogy.md b/docs/guide/dl-analogy.md new file mode 100644 index 0000000..758566f --- /dev/null +++ b/docs/guide/dl-analogy.md @@ -0,0 +1,51 @@ +# Deep Learning ↔ SkillOpt Analogy + +SkillOpt is designed around a core insight: **optimizing natural-language prompts follows the same structure as training neural networks**. This page maps every DL concept to its SkillOpt counterpart. + +## Complete Mapping + +| Deep Learning | SkillOpt | Description | +|---|---|---| +| **Model weights** | Skill document (Markdown) | The thing being optimized | +| **Forward pass** | Rollout | Target executes tasks using current skill | +| **Loss function** | Task evaluator | Scores task execution quality | +| **Backpropagation** | Reflect | Optimizer analyzes failures → edit patches | +| **Gradients** | Edit patches | Proposed changes to the skill | +| **Gradient aggregation** | Patch aggregation | Merge similar edits | +| **Gradient clipping** | Edit selection | Cap max edits per step | +| **Learning rate** | `learning_rate` | Max number of edits applied per step | +| **LR scheduler** | `lr_scheduler` | Decay schedule: cosine, linear, constant | +| **SGD step** | Skill update | Apply selected patches to document | +| **Validation set** | Selection split | Gate checks improvement before accepting | +| **Early stopping** | Gate patience | Reject updates that don't improve | +| **Training step** | Step | One rollout → reflect → update cycle | +| **Epoch** | Epoch | Full pass with slow update + meta memory | +| **Momentum** | Slow update | Longitudinal comparison at epoch boundary | +| **Meta-learning** | Meta skill | Cross-epoch optimizer strategy memory | +| **Batch size** | `batch_size` | Tasks sampled per rollout | +| **Data parallelism** | `analyst_workers` | Parallel reflection workers | +| **Training set** | Train split | Items used for rollout | +| **Test set** | Test split | Held-out final evaluation | +| **Warm-up** | (implicit) | High LR early steps explore broadly | +| **Checkpointing** | Skill snapshots | Saved after each accepted step | +| **Transfer learning** | Seed skill / cross-benchmark init | Start from pre-trained skill | + +## Why This Analogy Matters + +1. **Familiar mental model**: ML practitioners immediately understand how to tune SkillOpt +2. **Principled hyperparameter search**: Grid search over `learning_rate` × `lr_scheduler` works just like in DL +3. **Proven mechanisms**: Gating ≈ validation-based selection, patience ≈ early stopping, slow update ≈ momentum — all with strong theoretical motivation + +## Hyperparameter Transfer Rules + +From our experiments, these DL intuitions transfer well: + +!!! success "What transfers" + - **Cosine schedule > constant** — same as in DL, cosine annealing helps convergence + - **Moderate LR (4-16) > very high/low** — too few edits = slow learning, too many = noisy + - **Slow update helps** — longitudinal comparison prevents catastrophic forgetting across epochs + - **Meta skill memory improves reflection** — optimizer benefits from cross-epoch strategy notes + +!!! warning "What doesn't transfer" + - **Batch size ≠ better** — larger rollout batches have diminishing returns due to API costs + - **More epochs ≠ better** — skills converge faster than neural networks (2-4 epochs usually enough) diff --git a/docs/guide/first-experiment.md b/docs/guide/first-experiment.md new file mode 100644 index 0000000..2a65589 --- /dev/null +++ b/docs/guide/first-experiment.md @@ -0,0 +1,110 @@ +# Your First Experiment + +This guide walks through running a complete SkillOpt training on SearchQA. + +## 1. Choose a Benchmark + +SkillOpt includes ready-to-use configs for several benchmarks: + +| Benchmark | Difficulty | Typical Runtime | +|---|---|---| +| SearchQA | ⭐ Easy | ~30 min | +| DocVQA | ⭐⭐ Medium | ~2 hours | +| ALFWorld | ⭐⭐⭐ Hard | ~3 hours | + +We'll use **SearchQA** as it's the fastest to complete. + +## 2. Configure + +Review the config file: + +```bash +cat configs/searchqa/default.yaml +``` + +Key parameters (deep learning analogy in parentheses): + +```yaml +train: + num_epochs: 4 # (epochs) + batch_size: 40 # (batch size) + +optimizer: + learning_rate: 4 # (max edits per step) + lr_scheduler: cosine # (learning rate schedule) + use_slow_update: true # (momentum at epoch boundary) + use_meta_skill: true # (cross-epoch optimizer memory) + +gradient: + analyst_workers: 16 # (parallel reflection workers) + +evaluation: + use_gate: true # (validation gating) +``` + +## 3. Train + +```bash +python scripts/train.py --config configs/searchqa/default.yaml +``` + +You'll see output like: + +``` +[Step 1/8] Rollout: 20 items, 4 workers... +[Step 1/8] Score: 0.65 → Reflect... +[Step 1/8] 6 edit patches generated +[Step 1/8] Selected 4 edits (lr=8, cosine → 7.7) +[Step 1/8] Gate: val score 0.68 > 0.65 ✓ ACCEPT +[Step 2/8] ... +``` + +## 4. Monitor + +Training outputs are saved to `outputs///`: + +``` +outputs/searchqa/2024-01-15_10-30-00/ +├── steps/ +│ ├── step_0001/ +│ │ ├── candidate_skill.md +│ │ ├── step_record.json +│ │ └── trajectory_digest.json +│ └── step_0002/ +├── slow_update/ +│ └── epoch_02/ +├── meta_skill/ +│ └── epoch_02/ +├── skills/ +│ └── step_0001.md +├── best_skill.md +├── history.json +└── config.yaml +``` + +## 5. Evaluate + +Evaluate the best skill on the test split: + +```bash +python scripts/eval_only.py \ + --config configs/searchqa/default.yaml \ + --skill outputs/searchqa//skills/best_skill.md +``` + +## WebUI + +Prefer a graphical interface? Launch the WebUI: + +```bash +pip install -e ".[webui]" +python -m skillopt_webui.app +``` + +Then open `http://localhost:7860` in your browser to configure parameters and launch training. + +## Next Steps + +- [Understand the training loop](training-loop.md) +- [Configuration reference](../reference/config.md) +- [Add a new benchmark](new-benchmark.md) diff --git a/docs/guide/installation.md b/docs/guide/installation.md new file mode 100644 index 0000000..0fd390e --- /dev/null +++ b/docs/guide/installation.md @@ -0,0 +1,89 @@ +# Installation + +## Requirements + +- Python ≥ 3.10 +- At least one model API key (Azure OpenAI, OpenAI, Anthropic, or local Qwen) + +## Quick Install + +```bash +git clone https://github.com/microsoft/SkillOpt.git +cd SkillOpt +pip install -e . +``` + +## Optional Dependencies + +Install extras for specific benchmarks or backends: + +=== "ALFWorld" + + ```bash + pip install -e ".[alfworld]" + ``` + +=== "Claude Backend" + + ```bash + pip install -e ".[claude]" + ``` + +=== "Qwen (Local)" + + ```bash + pip install -e ".[qwen]" + ``` + +=== "WebUI" + + ```bash + pip install -e ".[webui]" + ``` + +=== "Development" + + ```bash + pip install -e ".[dev]" + ``` + +=== "All" + + ```bash + pip install -e ".[alfworld,claude,qwen,webui,dev]" + ``` + +## Environment Variables + +Copy the example `.env` file and fill in your credentials: + +```bash +cp .env.example .env +``` + +Edit `.env` with your API keys: + +```ini +# Azure OpenAI (default backend) +AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/ +AZURE_OPENAI_API_KEY=your-key + +# Or use OpenAI directly +OPENAI_API_KEY=sk-... + +# Or Anthropic Claude +ANTHROPIC_API_KEY=sk-ant-... +``` + +!!! tip + You only need credentials for the backend you plan to use. Azure OpenAI is the default. + +## Verify Installation + +```bash +python -c "import skillopt; print('SkillOpt ready!')" +``` + +## Next Steps + +→ [Run your first experiment](first-experiment.md) diff --git a/docs/guide/local-env-smoke.md b/docs/guide/local-env-smoke.md new file mode 100644 index 0000000..f1af13e --- /dev/null +++ b/docs/guide/local-env-smoke.md @@ -0,0 +1,143 @@ +# Local Environment Smoke Tests + +This guide describes a lightweight pattern for testing a custom SkillOpt environment before connecting it to expensive model calls or a full benchmark dataset. + +The goal is to validate the training loop plumbing first: + +- config loading +- adapter construction +- dataloader splits +- rollout output shape +- reflection patch shape +- merge/rank/update control flow +- artifact creation under `out_root` + +Once those are stable, you can switch the same environment to real model calls and larger evaluation splits. + +## 1. Add a tiny fixture split + +Start with a handful of deterministic examples that cover the expected pass/fail cases for your environment. Keep them small enough that a single training step can run locally. + +A minimal fixture item usually needs: + +```json +{ + "id": "example-1", + "split": "train", + "question": "...", + "expected": "..." +} +``` + +Use the split names your adapter maps to SkillOpt phases: + +- `train` for optimization rollouts +- `val` or `valid_seen` for selection/gating +- `test` or `valid_unseen` for final evaluation + +## 2. Support an offline mock mode + +Add a configuration flag such as `mock: true` to your adapter. In mock mode, `rollout()` should return deterministic responses without calling external model APIs. + +This lets you verify the SkillOpt loop with a fast command such as: + +```bash +python scripts/train.py \ + --config configs/myenv/tiny_mock.yaml +``` + +Mock mode should still write the same artifacts as a real run, for example: + +- `responses.json` +- `rollout_results.json` +- `ranked_edits.json` +- `candidate_skill.md` +- `summary.json` + +## 3. Keep the smoke config tiny + +A CI-friendly smoke config should run a single small step: + +```yaml +train: + num_epochs: 1 + train_size: 3 + batch_size: 3 + +gradient: + minibatch_size: 1 + merge_batch_size: 2 + analyst_workers: 1 + max_analyst_rounds: 1 + +optimizer: + learning_rate: 1 + min_learning_rate: 1 + lr_scheduler: constant + skill_update_mode: patch + use_slow_update: false + +evaluation: + use_gate: true + sel_env_num: 2 + test_env_num: 2 + eval_test: false + +env: + name: myenv + out_root: outputs/myenv_tiny_mock + mock: true +``` + +Prefer a mock config that runs without credentials. That makes it useful for contributors and CI. + +## 4. Validate optimizer JSON before returning it + +If your environment or extension asks an LLM to merge or rank skill edits, validate the returned JSON before passing it back into SkillOpt. This avoids silent fallbacks from empty, malformed, or out-of-range responses. + +Useful checks for edit payloads: + +- response is a JSON object +- `edits` is a non-empty list +- every edit is an object +- every edit has an allowed operation +- required fields such as `content` or `target` are present for that operation + +Useful checks for ranking payloads: + +- `selected_indices` exists +- indices are integers +- indices are unique +- indices are within the candidate edit range +- selected count does not exceed the edit budget + +On failure, retry with a compact prompt that includes the schema error. If retries fail, raise an explicit error instead of silently accepting malformed output. + +## 5. Run progressively stronger checks + +A good development sequence is: + +```bash +python -m py_compile scripts/train.py skillopt/envs/myenv/adapter.py +python scripts/train.py --config configs/myenv/tiny_mock.yaml +python scripts/train.py --config configs/myenv/tiny.yaml +``` + +For the real tiny run, verify that: + +- the run completes +- `summary.json` is written +- `ranked_edits.json` contains the expected ranking metadata +- any optimizer bridge log marks the response schema as valid +- no generated files are written outside `out_root` + +## 6. Keep custom environments isolated + +When adding a custom environment to the registry, avoid side effects for existing benchmarks: + +- lazy-import optional dependencies +- install environment-specific hooks only when `cfg["env"]` matches your environment +- keep mock behavior behind an explicit config flag +- write generated artifacts only under `out_root` + +This makes it easier to review and test a custom integration without affecting the built-in benchmarks. diff --git a/docs/guide/new-backend.md b/docs/guide/new-backend.md new file mode 100644 index 0000000..03fca9e --- /dev/null +++ b/docs/guide/new-backend.md @@ -0,0 +1,130 @@ +# Add a New Model Backend + +SkillOpt supports multiple LLM backends. This guide shows how to add your own. + +## Backend Architecture + +``` +skillopt/model/ +├── base.py # Abstract base class +├── azure_openai.py # Azure OpenAI backend +├── openai_model.py # Direct OpenAI backend +├── claude.py # Anthropic Claude backend +├── qwen.py # Local Qwen (vLLM) backend +└── your_backend.py # Your new backend +``` + +## Step 1: Create the Backend + +Create `skillopt/model/your_backend.py`: + +```python +from skillopt.model.base import ModelBackend, ModelResponse + +class YourBackend(ModelBackend): + """Your custom model backend.""" + + def __init__(self, cfg: dict): + super().__init__(cfg) + self.model_name = cfg.get('model_name', 'your-default-model') + self.api_key = os.environ.get('YOUR_API_KEY', '') + self.client = self._init_client() + + def _init_client(self): + """Initialize API client.""" + # TODO: Set up your API client + pass + + async def generate( + self, + messages: list[dict], + temperature: float = 0.7, + max_tokens: int = 4096, + **kwargs + ) -> ModelResponse: + """ + Generate a completion. + + Args: + messages: Chat messages [{"role": "...", "content": "..."}] + temperature: Sampling temperature + max_tokens: Maximum tokens in response + + Returns: + ModelResponse with content, usage, and metadata + """ + response = await self.client.chat( + model=self.model_name, + messages=messages, + temperature=temperature, + max_tokens=max_tokens, + ) + + return ModelResponse( + content=response.text, + usage={ + 'prompt_tokens': response.usage.input, + 'completion_tokens': response.usage.output, + }, + model=self.model_name, + ) + + async def generate_with_tools( + self, + messages: list[dict], + tools: list[dict], + **kwargs + ) -> ModelResponse: + """Generate with tool/function calling support.""" + # Optional: implement if your model supports tool use + raise NotImplementedError("Tool use not supported") +``` + +## Step 2: Register the Backend + +Add to `skillopt/model/__init__.py`: + +```python +from .your_backend import YourBackend + +BACKEND_REGISTRY = { + # ... existing backends ... + 'your_backend': YourBackend, +} +``` + +## Step 3: Configure + +Use your backend in any config: + +```yaml +model: + backend: your_backend + model_name: your-model-id + temperature: 0.7 + max_tokens: 4096 +``` + +Set credentials via environment variable: + +```bash +export YOUR_API_KEY="your-key" +``` + +## Required Interface + +Your backend must implement these methods: + +| Method | Required | Description | +|---|---|---| +| `generate()` | ✅ | Basic text generation | +| `generate_with_tools()` | Optional | Tool/function calling | +| `count_tokens()` | Optional | Token counting for context management | + +## Tips + +!!! tip + - Test your backend with `python -c "from skillopt.model.your_backend import YourBackend"` first + - Use `async` methods for all API calls — SkillOpt uses asyncio throughout + - Implement retry logic with exponential backoff for production use + - Add your API key to `.env.example` when submitting a PR diff --git a/docs/guide/new-benchmark.md b/docs/guide/new-benchmark.md new file mode 100644 index 0000000..6dae9a1 --- /dev/null +++ b/docs/guide/new-benchmark.md @@ -0,0 +1,371 @@ +# Add a New Benchmark + +Extend SkillOpt with your own benchmark in ~200 lines of code. We will use +a tiny worked example, `docfaithful`, that scores a target model on +how faithfully it answers questions grounded in a small reference doc. + +> **Working reference.** The easiest way to copy-cargo-cult a new env is +> to read [`skillopt/envs/officeqa/`](https://github.com/microsoft/SkillOpt/tree/main/skillopt/envs/officeqa). +> Everything below is the same shape, simplified. + +## What you need to build + +To add a benchmark you implement four things: + +1. **A `SplitDataLoader` subclass** — knows how to load train / val / test + item dicts from disk. +2. **A rollout helper** — runs the target model on a batch of items + under the current skill and scores each prediction. +3. **An `EnvAdapter` subclass** — wires the loader + rollout helper into + SkillOpt's lifecycle (`build_*_env`, `rollout`, `reflect`, + `get_task_types`). +4. **A YAML config** — references your env name plus the standard + train / optimizer / gradient knobs. + +Then one line in `scripts/train.py`'s `_register_builtins()` makes it +discoverable. + +--- + +## Step 1 — Create the package + +```bash +mkdir -p skillopt/envs/docfaithful +touch skillopt/envs/docfaithful/__init__.py +``` + +## Step 2 — Implement the data loader + +`skillopt/envs/docfaithful/dataloader.py`: + +```python +from __future__ import annotations + +import json +from pathlib import Path + +from skillopt.datasets.base import SplitDataLoader + + +def _normalize(raw: dict) -> dict: + """Make sure every item has an ``id``. Other keys are env-specific.""" + return { + "id": str(raw["uid"]), + "question": raw["question"], + "ground_truth": raw["answer"], + "reference_text": raw.get("reference", ""), + "task_type": raw.get("category", "docfaithful"), + } + + +class DocFaithfulDataLoader(SplitDataLoader): + """Load DocFaithful items from JSON files inside each split dir.""" + + def load_split_items(self, split_path: str) -> list[dict]: + # split_path is e.g. data/docfaithful_split/train/ + json_files = sorted(Path(split_path).glob("*.json")) + if not json_files: + raise FileNotFoundError(f"No .json file found in {split_path}") + with json_files[0].open(encoding="utf-8") as f: + raw = json.load(f) + return [_normalize(item) for item in raw] +``` + +Only `load_split_items()` is mandatory. If you also want to support +`split_mode="ratio"` (auto-split a single raw file into train/val/test), +override `load_raw_items(data_path)` as well — see +`skillopt/datasets/base.py` docstrings. + +## Step 3 — Write the rollout helper + +`skillopt/envs/docfaithful/rollout.py`: + +```python +from __future__ import annotations + +import json +import os +from pathlib import Path + +from skillopt.model import chat_target + + +def _score(prediction: str, ground_truth: str) -> tuple[int, float]: + """Trivial exact-match scorer. Replace with F1 / ROUGE / LLM-judge.""" + p = (prediction or "").strip().lower() + g = (ground_truth or "").strip().lower() + hard = int(p == g and bool(g)) + soft = 1.0 if hard else 0.0 + return hard, soft + + +def _rollout_one(item: dict, skill_content: str, + *, max_completion_tokens: int) -> dict: + system = skill_content + user = ( + f"Question: {item['question']}\n\n" + f"Reference:\n{item.get('reference_text', '')}\n\n" + "Answer:" + ) + prediction, _usage = chat_target( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + ) + hard, soft = _score(prediction, item.get("ground_truth", "")) + return { + "id": str(item["id"]), + "hard": hard, + "soft": soft, + "predicted_answer": prediction, + "question": item.get("question", ""), + "reference_text": item.get("reference_text", ""), + "task_type": item.get("task_type", "docfaithful"), + } + + +def run_batch(*, items: list[dict], skill_content: str, out_root: str, + workers: int = 4, max_completion_tokens: int = 4096) -> list[dict]: + """Run a batch of episodes sequentially or with a thread pool.""" + os.makedirs(out_root, exist_ok=True) + # For brevity we go sequentially — swap in concurrent.futures.ThreadPoolExecutor + # when network / model latency dominates. + results = [ + _rollout_one(item, skill_content, + max_completion_tokens=max_completion_tokens) + for item in items + ] + Path(out_root, "rollouts.json").write_text( + json.dumps(results, ensure_ascii=False, indent=2) + ) + return results +``` + +Two design points worth flagging: + +- **Scoring lives here, not in `EnvAdapter`.** There is no `evaluate()` + method on the ABC. Whatever signal you put in `hard` (0/1, or a float + in [0, 1] for smoothed reward) and `soft` (float in [0, 1]) is what + the optimizer reads. +- **Use `skillopt.model.chat_target`**, not raw OpenAI/Claude calls. + That routes through whichever **chat** target backend the user + configured (`openai_chat` / `claude_chat` / `qwen_chat` / + `minimax_chat`) without your adapter caring. Exec-style backends + (`codex_exec`, `claude_code_exec`) need env-specific rollout code — + see `skillopt/envs/swebench/` for an example. + +## Step 4 — Implement the environment adapter + +`skillopt/envs/docfaithful/adapter.py`: + +```python +from __future__ import annotations + +from skillopt.datasets.base import BatchSpec +from skillopt.envs.base import EnvAdapter +from skillopt.envs.docfaithful.dataloader import DocFaithfulDataLoader +from skillopt.envs.docfaithful.rollout import run_batch + + +class DocFaithfulAdapter(EnvAdapter): + """SkillOpt adapter for the DocFaithful benchmark.""" + + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "split_dir", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + workers: int = 4, + analyst_workers: int = 4, + failure_only: bool = False, + minibatch_size: int = 8, + edit_budget: int = 4, + seed: int = 42, + limit: int = 0, + max_completion_tokens: int = 4096, + ) -> None: + self.workers = workers + self.analyst_workers = analyst_workers + self.failure_only = failure_only + self.minibatch_size = minibatch_size + self.edit_budget = edit_budget + self.max_completion_tokens = int(max_completion_tokens) + self.dataloader = DocFaithfulDataLoader( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + seed=seed, + limit=limit, + ) + + # ── Lifecycle ─────────────────────────────────────────────────────── + + def setup(self, cfg: dict) -> None: + super().setup(cfg) + self.dataloader.setup(cfg) + + def get_dataloader(self): + return self.dataloader + + # ── Env construction ──────────────────────────────────────────────── + + def build_env_from_batch(self, batch: BatchSpec, **kwargs): + # For dataset-backed envs the "manager" is just the items list. + return list(batch.payload or []) + + def build_train_env(self, batch_size: int, seed: int, **kwargs): + batch = self.dataloader.build_train_batch( + batch_size=batch_size, seed=seed, **kwargs + ) + return self.build_env_from_batch(batch, **kwargs) + + def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs): + batch = self.dataloader.build_eval_batch( + env_num=env_num, split=split, seed=seed, **kwargs + ) + return self.build_env_from_batch(batch, **kwargs) + + # ── The rollout method (reflect is inherited) ─────────────────────── + + def rollout(self, env_manager, skill_content: str, + out_dir: str, **kwargs) -> list[dict]: + items: list[dict] = env_manager + return run_batch( + items=items, + skill_content=skill_content, + out_root=out_dir, + workers=self.workers, + max_completion_tokens=self.max_completion_tokens, + ) + + # reflect() is inherited from EnvAdapter — it delegates to + # run_minibatch_reflect with your analyst_error_* / analyst_success_* + # prompts. Override it only if you need custom reflection logic. + + def get_task_types(self) -> list[str]: + seen: list[str] = [] + for item in ( + self.dataloader.train_items + + self.dataloader.val_items + + self.dataloader.test_items + ): + tt = str(item.get("task_type") or "docfaithful") + if tt not in seen: + seen.append(tt) + return seen or ["docfaithful"] +``` + +### What the rollout actually does + +Look back at `run_batch` from Step 3 — it sends each `item["question"]` +to the target model with `skill_content` as the system prompt, scores +the answer against `item["ground_truth"]`, and returns a list of dicts: + +```python +[ + {"id": "ex_001", "hard": 1, "soft": 0.92, + "predicted_answer": "...", "question": "...", + "reference_text": item["reference_text"]}, + {"id": "ex_002", "hard": 0, "soft": 0.13, "fail_reason": "...", ...}, + ... +] +``` + +The trainer only requires `id`, `hard`, `soft`. The rest is preserved on +`RolloutResult.extras` (see `skillopt/types.py`) and is what your +`reflect()` consumes via `run_minibatch_reflect`. + +## Step 5 — Register the adapter + +Edit [`scripts/train.py`](https://github.com/microsoft/SkillOpt/blob/main/scripts/train.py) +and add to `_register_builtins()`: + +```python + try: + from skillopt.envs.docfaithful.adapter import DocFaithfulAdapter + _ENV_REGISTRY["docfaithful"] = DocFaithfulAdapter + except ImportError: + pass # docfaithful deps not installed — skip +``` + +There is **no `BENCHMARK_REGISTRY` dict in `skillopt/envs/__init__.py`** — +the registry lives in `scripts/train.py` and is populated lazily so that +optional deps don't break `--help`. + +## Step 6 — Create the YAML config + +`configs/docfaithful/default.yaml`: + +```yaml +_base_: ../_base_/default.yaml # NOTE: string, not list + +model: + reasoning_effort: medium + +train: + batch_size: 16 + accumulation: 1 + num_epochs: 4 + +gradient: + minibatch_size: 8 + merge_batch_size: 8 + +optimizer: + learning_rate: 4 + +env: + name: docfaithful + # Optional: a seed skill document. Create this file (or any markdown + # file) yourself before the first run, or omit the key to let SkillOpt + # start from an empty skill. + skill_init: skillopt/envs/docfaithful/skills/initial.md + split_mode: split_dir + split_dir: data/docfaithful_split + workers: 4 + max_completion_tokens: 4096 + limit: 0 +``` + +> ⚠️ `_base_` is currently parsed as a **string path**, not a list. Write +> `_base_: ../_base_/default.yaml`, not `_base_: ['../_base_/default.yaml']`. +> See [`skillopt/config.py`](https://github.com/microsoft/SkillOpt/blob/main/skillopt/config.py) +> if you want to add list-form inheritance. + +## Step 7 — Run + +```bash +# If you set skill_init above, create the seed skill first: +# mkdir -p skillopt/envs/docfaithful/skills +# echo "# DocFaithful initial skill" > skillopt/envs/docfaithful/skills/initial.md + +python scripts/train.py --config configs/docfaithful/default.yaml +``` + +If you get `ValueError: Unknown environment 'docfaithful'. Available: [...]`, +you forgot Step 5. + +If you get `TypeError: Can't instantiate abstract class DocFaithfulAdapter`, +you forgot to implement one of the four abstract methods on `EnvAdapter`: +`build_train_env`, `build_eval_env`, `rollout`, `get_task_types`. + +## Tips + +- Start with `train.batch_size: 4` and `limit: 10` while debugging. +- The `evaluate` half lives **inside your `rollout`**, not as a separate + method — there is no `evaluate()` in the `EnvAdapter` ABC. Score the + prediction in `run_batch` and put the score on each result dict's + `hard` / `soft`. +- Noisy scoring kills the optimizer. Spend time on `run_batch`'s scoring + before you spend time on prompts. +- If your benchmark needs heavy optional deps (selenium, vllm, ...), + wrap the registration block with `try / except ImportError` (Step 5) + so people without those deps can still `--help`. +- Copy `skillopt/envs/_template/` as a starting skeleton — it now + implements the real abstract methods. diff --git a/docs/guide/skill-document.md b/docs/guide/skill-document.md new file mode 100644 index 0000000..62d1a34 --- /dev/null +++ b/docs/guide/skill-document.md @@ -0,0 +1,78 @@ +# Skill Document + +A **skill document** is a Markdown file that serves as the "prompt weights" of your agent. SkillOpt trains this document through iterative optimization. + +## What is a Skill Document? + +A skill document is a structured set of instructions that tells a language model **how** to approach a specific type of task. It's analogous to learned weights in a neural network — encoding task-specific knowledge in natural language rather than floating-point parameters. + +## Structure + +A typical skill document contains: + +```markdown +# Task Strategy + +## General Approach +- Break complex problems into sub-steps +- Always verify intermediate results + +## Common Patterns +- When you see X, try approach Y +- Avoid Z because it leads to errors + +## Edge Cases +- If the input contains A, handle it specially by... +- Watch out for B — it requires C + +## Output Format +- Always include reasoning before the answer +- Format numbers with proper units +``` + +## How It Evolves + +During training, the skill document is modified by **edit patches**: + +1. **Additions**: New rules or strategies discovered from failed trajectories +2. **Modifications**: Refining existing rules that are partially correct +3. **Deletions**: Removing rules that consistently lead to errors + +Each edit is validated through the **gate** mechanism before being permanently accepted. + +## Initial Skill + +You can start training with: + +- **Empty skill**: The system learns everything from scratch +- **Seed skill**: Provide initial instructions to bootstrap training +- **Pre-trained skill**: Transfer a skill from a related benchmark + +Configure the initial skill in your YAML: + +```yaml +train: + init_skill: "path/to/initial_skill.md" # or omit for empty +``` + +## Skill Quality Metrics + +Track your skill's evolution through: + +- **Validation score**: Primary metric on the selection split +- **Test score**: Final metric on held-out test data +- **Skill length**: Total tokens in the document +- **Edit acceptance rate**: Fraction of proposed edits that pass gating + +## Best Practices + +!!! tip "Tips for better skills" + 1. **Start with a seed skill** (`env.skill_init`) if you have domain knowledge — it converges faster + 2. **Use cosine LR schedule** — aggressive early exploration + careful late refinement + 3. **Enable slow update** (`use_slow_update: true`) to prevent forgetting across epochs + 4. **Enable meta skill** (`use_meta_skill: true`) so the optimizer accumulates strategy memory + +## Next Steps + +- [Deep Learning Analogy](dl-analogy.md) +- [Configuration Reference](../reference/config.md) diff --git a/docs/guide/training-loop.md b/docs/guide/training-loop.md new file mode 100644 index 0000000..7922305 --- /dev/null +++ b/docs/guide/training-loop.md @@ -0,0 +1,92 @@ +# The Training Loop + +SkillOpt's core insight: **optimizing natural-language skill documents follows the same structure as training neural networks**. + +## Overview + +``` +┌─────────────────────────────────────────────────────────┐ +│ Training Loop │ +│ │ +│ for epoch in epochs: │ +│ for step in steps: │ +│ 1. Rollout — Target executes tasks │ +│ 2. Reflect — Optimizer analyzes trajectories │ +│ 3. Aggregate — Hierarchical merge of patches │ +│ 4. Select — Rank & clip edits (learning rate) │ +│ 5. Update — Apply patches to skill doc │ +│ 6. Gate — Validate & accept/reject │ +│ │ +│ Epoch Boundary: │ +│ • Slow Update (longitudinal comparison & guidance) │ +│ • Meta Skill (cross-epoch strategy memory) │ +└─────────────────────────────────────────────────────────┘ +``` + +## Stage Details + +### 1. Rollout (Forward Pass) + +The **target** model executes tasks using the current skill document as its prompt. Each task produces a trajectory and a score. + +```python +# Analogy: forward pass through the network +predictions = model(input, skill_document) +scores = evaluate(predictions, ground_truth) +``` + +### 2. Reflect (Backward Pass) + +The **optimizer** model analyzes failed trajectories and produces **edit patches** — structured suggestions for improving the skill document. + +Two modes: + +- **Shallow**: Analyze each trajectory independently +- **Deep**: Cross-reference multiple failures to find systemic issues + +```python +# Analogy: computing gradients +gradients = loss.backward() # → edit patches +``` + +### 3. Aggregate + +Semantically similar edit patches are merged to avoid redundant edits. + +### 4. Select (Gradient Clipping) + +Edits are ranked by relevance score. The `learning_rate` parameter caps how many edits are applied per step — just like gradient clipping prevents overshooting. + +```python +# Analogy: gradient clipping + optimizer step size +selected = top_k(edits, k=learning_rate) +``` + +The `lr_scheduler` adjusts this over training: + +- **cosine**: Start aggressive, taper smoothly +- **linear**: Linear decay +- **constant**: Fixed rate + +### 5. Update (Parameter Update) + +Selected edits are applied to the skill document, producing a new version. + +### 6. Gate (Validation) + +The updated skill is evaluated on a **selection split** (analogous to a validation set). The update is only accepted if performance improves. + +## Epoch Boundary Mechanisms + +### Slow Update + +At the end of each epoch (starting from epoch 2), the system performs a **longitudinal comparison**: it rolls out both the previous epoch's skill and the current skill on the same samples, categorizes items as improved/regressed/persistent_fail/stable_success, then generates high-level **guidance** that is injected into the skill document. This prevents catastrophic forgetting of earlier improvements. + +### Meta Skill + +A **meta-skill memory** accumulates high-level strategy notes across the entire training run. At the end of each epoch, the optimizer reflects on what changed between epochs and produces a compact memory that is provided as additional context during future reflection steps. + +## Next Steps + +- [Understand Skill Documents](skill-document.md) +- [DL ↔ SkillOpt analogy table](dl-analogy.md) diff --git a/docs/guideline.html b/docs/guideline.html new file mode 100644 index 0000000..8712012 --- /dev/null +++ b/docs/guideline.html @@ -0,0 +1,1042 @@ + + + + + +SkillOpt — Documentation & Reproduction Guide + + + + + + +
+ + + SkillOpt + Documentation & Reproduction Guide + + GitHub ↗ + Paper ↗ +
+ +
+ + + + + +
+ + Microsoft Research +

SkillOpt Documentation & Reproduction Guide

+

Train agent skills like you train neural networks — with epochs, (mini-)batch size, learning rates, and validation gates — but without touching any model weights.

+

This guide walks you from a clean checkout to a reproduced result and a full reference for every configuration knob and core function. It is generated from, and kept consistent with, the current state of the codebase.

+ + +
+

1.1 What is SkillOpt #

+

SkillOpt is a text-space optimizer that improves a frozen language agent by iteratively editing a natural-language skill document — never the model weights. The skill document is a Markdown file that conditions a target model as it executes tasks. SkillOpt treats this document as the "weights" and runs a training loop that mirrors deep-learning training: rollout (forward pass), reflect (backward pass / gradients), select & apply edits (optimizer step), and a validation gate (accept/reject).

+

Two roles split every model call:

+
    +
  • Target — executes tasks using the current skill document (the agent being improved).
    • +
  • Optimizer — analyzes the target's trajectories and proposes edits to the skill document.
    • +
+

The same loop drives six benchmarks out of the box (QA, document QA, embodied agents, math, spreadsheet code generation, and tool-augmented QA).

+
+ +
+

1.2 Deep-Learning ↔ SkillOpt Analogy #

+

Every concept below maps to a concrete code construct, so deep-learning intuitions transfer directly to hyperparameter tuning.

+
+ + + + + + + + + + + + + + + + + + + +
Deep learningSkillOptWhere it lives
Model weightsSkill document (Markdown)skillopt/optimizer/skill.py
Forward passRollout — target runs tasksenvs/<bench>/rollout.py
Loss / scoreTask evaluatorenvs/<bench>/evaluator.py
Backprop / gradientsReflect → edit patchesgradient/reflect.py
Gradient aggregationHierarchical patch mergegradient/aggregate.py
Gradient clippingRank & select top-k editsoptimizer/clip.py
Learning rateoptimizer.learning_rate (edits/step)optimizer/scheduler.py
LR schedulerlr_scheduler (cosine/linear/…)optimizer/scheduler.py
Optimizer stepApply patches to the documentoptimizer/skill.py
Validation setSelection split (valid_seen)evaluation/gate.py
Early stopping / acceptValidation gateevaluation/gate.py
MomentumSlow update (epoch boundary)optimizer/slow_update.py
Meta-learningMeta skill (cross-epoch memory)optimizer/meta_skill.py
Batch / minibatchbatch_size / minibatch_sizeengine/trainer.py
EpochEpoch (+ slow update & meta skill)engine/trainer.py
+
+
What transfers from DL +

Cosine schedule tends to beat constant; moderate learning rates (≈4–16 edits/step) beat very high/low; slow update curbs cross-epoch forgetting; meta-skill memory improves reflection quality. Conversely, bigger rollout batches and many epochs show diminishing returns — skills converge in ~2–4 epochs.

+
+
+ +
+

1.3 Key Features #

+
+

Validation gating

Every candidate skill is scored on a held-out selection split and only accepted if it beats the current/best skill.

+

Slow update

Epoch-boundary longitudinal comparison writes guidance into a protected region — momentum against forgetting. Force-injected or selection-gated.

+

Meta skill

Optimizer-side memory that reflects on what worked across epochs and feeds back into reflection.

+

Pluggable backends

OpenAI / Azure OpenAI, Anthropic Claude, local Qwen (vLLM), plus Codex/Claude-Code exec backends for the target.

+

Six benchmarks

SearchQA, DocVQA, ALFWorld, LiveMathematicianBench, SpreadsheetBench, OfficeQA — each a self-contained env module.

+

Auto-resume

Every run is checkpointed step-by-step; re-running the same command continues from the last completed step.

+
+
+ +
+

1.4 Repository Layout #

+
# top level
+configs/            # YAML configs (_base_ + per-benchmark)
+scripts/            # train.py, eval_only.py CLIs
+ckpt/               # packaged reference skills (e.g. gpt5.5_skill.md)
+docs/               # this guide + mkdocs sources
+skillopt/           # the package
+ ├─ config.py        # YAML loading, _base_ inheritance, flatten
+ ├─ engine/trainer.py# the training loop (ReflACTTrainer)
+ ├─ gradient/        # reflect.py (analyst), aggregate.py (merge)
+ ├─ optimizer/       # skill edits, scheduler, clip, slow_update, meta_skill
+ ├─ evaluation/      # gate.py (accept/reject logic)
+ ├─ model/           # backend clients + routing
+ └─ envs/<benchmark>/ # adapter, dataloader, rollout, evaluator, reflect
+
+ + +
+

2.1 Requirements #

+
    +
  • Python ≥ 3.10
    • +
  • Credentials for at least one model backend (Azure OpenAI, OpenAI-compatible, Anthropic, or a local Qwen server)
    • +
  • Benchmark datasets are not bundled — prepare your own splits (see §4)
    • +
+
+ +
+

2.2 Install the Package #

+

Option A — from PyPI:

+
pip install skillopt
+
+# Optional extras:
+pip install skillopt[alfworld]   # ALFWorld benchmark
+pip install skillopt[webui]      # Gradio monitoring dashboard
+pip install skillopt[claude]     # Claude model backend
+
+

Option B — from source (for development):

+
git clone https://github.com/microsoft/SkillOpt.git
+cd SkillOpt
+pip install -e .
+
+# Optional extras (install only what you need):
+pip install -e ".[alfworld]"   # ALFWorld benchmark
+pip install -e ".[claude]"     # Anthropic Claude backend
+pip install -e ".[qwen]"       # local Qwen backend
+pip install -e ".[webui]"      # monitoring dashboard
+
+# ALFWorld also needs its data assets:
+alfworld-download
+
+ +
+

2.3 Configure Credentials #

+

Copy the template and fill in whichever backend you will use:

+
cp .env.example .env
+# edit .env, then:
+set -a; source .env; set +a
+
One env-var family for all OpenAI modes +

SkillOpt reuses the AZURE_OPENAI_* variable names even for plain OpenAI — there is no separate OPENAI_API_KEY knob. AZURE_OPENAI_ENDPOINT is required for every OpenAI auth mode.

+
+

Azure OpenAI (default)

+
export AZURE_OPENAI_ENDPOINT="https://your-resource.openai.azure.com/"
+export AZURE_OPENAI_API_VERSION="2024-12-01-preview"
+# Auth option 1 — API key:
+export AZURE_OPENAI_API_KEY="your-key"
+# Auth option 2 — Azure CLI (no key; recommended on Azure VMs):
+export AZURE_OPENAI_AUTH_MODE=azure_cli
+# Auth option 3 — Managed Identity:
+export AZURE_OPENAI_AUTH_MODE=managed_identity
+export AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID="your-client-id"
+

OpenAI-compatible endpoint

+
export AZURE_OPENAI_ENDPOINT="https://api.openai.com/v1"
+export AZURE_OPENAI_API_KEY="sk-..."
+export AZURE_OPENAI_AUTH_MODE=openai_compatible
+

Anthropic Claude / local Qwen

+
export ANTHROPIC_API_KEY="sk-ant-..."          # claude_chat backend
+
+export QWEN_CHAT_BASE_URL="http://localhost:8000/v1" # local vLLM
+export QWEN_CHAT_MODEL="Qwen/Qwen3.5-4B"
+
+ +
+

2.4 Verify Installation #

+
python -c "import skillopt; print('SkillOpt ready!')"
+
+ + +
+

3.1 Your First Demo #

+

What ships in this repo: ready-to-use configs and + pretrained skills (ckpt/) for six benchmarks, plus + lightweight ID manifests under data/. The manifests + pin exactly which examples each split uses but do not + contain the example contents — so you materialize the data once before + the first run.

+

Step 1 — materialize the SearchQA splits (one-time; downloads the ~6.5 GB source dataset). The manifest IDs match the key field of the + lucadiliello/searchqa + dataset:

+
pip install datasets
+python - <<'PY'
+import json, os
+from datasets import load_dataset
+
+ds = load_dataset("lucadiliello/searchqa")
+by_key = {r["key"]: r for split in ds.values() for r in split}
+
+for split in ["train", "val", "test"]:
+    ids = json.load(open(f"data/searchqa_id_split/{split}/items.json"))
+    items = []
+    for x in ids:
+        r = by_key[x["id"]]
+        items.append({"id": r["key"], "question": r["question"],
+                      "context": r["context"], "answers": r["answers"]})
+    os.makedirs(f"data/searchqa_split/{split}", exist_ok=True)
+    json.dump(items, open(f"data/searchqa_split/{split}/items.json", "w"))
+    print(split, len(items))
+PY
+

Step 2 — train (4 epochs × batch 40; see §3.2 + for the CLI reference):

+
python scripts/train.py \
+    --config configs/searchqa/default.yaml \
+    --split_dir data/searchqa_split \
+    --azure_openai_endpoint https://your-resource.openai.azure.com/ \
+    --optimizer_model gpt-5.5 \
+    --target_model gpt-5.5
+

Other benchmarks follow the same pattern — materialize from the raw + source listed in + data/README.md + (it documents the lookup key per benchmark), then point + --split_dir at the result. The one exception is + ALFWorld, whose bundled + data/alfworld_path_split works directly: just + pip install -e ".[alfworld]" && alfworld-download and + set $ALFWORLD_DATA.

+

To sanity-check your setup without training, evaluate a + packaged pretrained skill instead (§3.3 uses + ckpt/searchqa/gpt5.5_skill.md), or launch the monitoring + WebUI (§8.4).

+
+ +
+

3.2 Train a Skill #

+
# Minimal SearchQA run
+python scripts/train.py \
+    --config configs/searchqa/default.yaml \
+    --split_dir /path/to/your/searchqa_split \
+    --azure_openai_endpoint https://your-resource.openai.azure.com/ \
+    --optimizer_model gpt-5.5 \
+    --target_model gpt-5.5
+

Swap the config for another benchmark (e.g. configs/livemathematicianbench/default.yaml, configs/alfworld/default.yaml). Common CLI arguments:

+
+ + + + + + + + + + +
ArgumentDescription
--configBenchmark config YAML (required)
--split_dirPath to the data split directory
--azure_openai_endpointAzure OpenAI endpoint URL
--optimizer_model / --target_modelDeployment names for optimizer / target
--num_epochs / --batch_sizeEpochs and rollout batch size
--out_rootOutput directory
--cfg-options k=v ...Override any config key (see §6.1)
+
+ +
+

3.3 Evaluate a Skill #

+

Evaluate any skill document (a packaged reference skill, or a trained run's best_skill.md) without training:

+
# Evaluate the packaged GPT-5.5 SearchQA skill on the test split
+python scripts/eval_only.py \
+  --config configs/searchqa/default.yaml \
+  --skill ckpt/searchqa/gpt5.5_skill.md \
+  --split valid_unseen \
+  --split_dir /path/to/searchqa_split \
+  --azure_openai_endpoint https://your-resource.openai.azure.com/
+
+ + + + + + + +
--splitMeaning
valid_unseenTest set (held-out)
valid_seenValidation / selection set
trainTraining set
allAll splits combined (default)
+
+ +
+

3.4 Output Structure #

+
outputs/<run_name>/
+ ├─ config.json          # flattened runtime config
+ ├─ history.json         # per-step training history
+ ├─ runtime_state.json   # resume checkpoint
+ ├─ best_skill.md        # best validated skill document
+ ├─ skills/skill_vXXXX.md# skill snapshot per step
+ ├─ steps/step_XXXX/     # per-step artifacts (patches, evals)
+ ├─ slow_update/epoch_XX/# slow-update logs & rollouts
+ └─ meta_skill/epoch_XX/ # meta-skill logs
+
+ +
+

3.5 Auto-Resume #

+

Each completed step persists its state to runtime_state.json and a steps/step_XXXX/ directory. Re-running the same command against the same out_root detects finished work and continues from the last completed step — including epoch-boundary slow-update and meta-skill stages.

+
+ + +
+

4.1 Split Directory Format #

+

Bringing your own dataset takes three steps: + (1) create a split directory with train/ val/ test/ item + files in the format below; (2) make sure each item carries the fields + the closest existing benchmark adapter expects (§4.2); (3) point + --split_dir at it and train with that benchmark's config. + If no existing adapter matches your task shape (different rollout or + scoring logic), write a new benchmark adapter instead — see §7.2.

+ +

With env.split_mode: split_dir (the recommended, deterministic mode), SkillOpt reads a directory containing train/, val/, and test/ subfolders, each holding a JSON array of task items:

+
data/my_split/
+ ├─ train/items.json   # used for rollout (the "train split")
+ ├─ val/items.json     # selection split → validation gate (valid_seen)
+ └─ test/items.json    # held-out final eval (valid_unseen)
+
Split naming +

Internally the splits are referred to as train, valid_seen (validation/selection), and valid_unseen (test). The --split flag of eval_only.py uses these names.

+
+
+ +
+

4.2 Item JSON Schema #

+

Required fields depend on the benchmark; consult skillopt/envs/<benchmark>/dataloader.py for the exact contract. A SearchQA item, for example:

+
[
+  {
+    "id":       "unique_item_id",
+    "question": "Who wrote the novel ...",
+    "context":  "[DOC] relevant passage text ...",
+    "answers":  ["expected answer"]
+  }
+]
+
Datasets not included +

This repository ships no benchmark data. Prepare your own splits in the format above before training.

+
+
+ +
+

4.3 Split Modes #

+
+ + + + + +
env.split_modeBehavior
split_dirUse a pre-built directory with explicit train/val/test folders (set env.split_dir). Deterministic and reproducible.
ratioBuild a deterministic split on the fly from a single env.data_path, using split_seed (and a train:val:test ratio). Convenient for quick experiments.
+
+ + +
+

5.1 The Training Loop #

+

The loop lives in ReflACTTrainer (skillopt/engine/trainer.py). Each epoch runs a series of optimization steps over rollout batches, then performs two epoch-boundary stages.

+
for epoch in epochs:
+    for step in steps:
+        1. Rollout    # target executes a batch of tasks
+        2. Reflect    # optimizer analyzes trajectories → edit patches
+        3. Aggregate  # hierarchically merge similar patches
+        4. Select     # rank & clip edits to the learning rate
+        5. Update     # apply patches → candidate skill
+        6. Gate       # score on selection split → accept / reject
+
+    # epoch boundary (from epoch 2 onward)
+    Slow update   # longitudinal comparison → protected guidance
+    Meta skill    # cross-epoch optimizer memory
+
+ +
+

5.2 The Six Per-Step Stages #

+
+ + + + + + + + + +
StageWhat happensSource
1. RolloutThe target model runs each task in the batch with the current skill as context, producing trajectories and scores.envs/<b>/rollout.py
2. ReflectThe optimizer runs an error analyst (and optional success analyst) over minibatches of trajectories, emitting structured edit patches. Runs in parallel across analyst_workers.gradient/reflect.py
3. AggregateSemantically similar patches are merged hierarchically to remove redundancy.gradient/aggregate.py → merge_patches
4. SelectPatches are ranked and clipped to the current learning rate (max edits this step), set by the scheduler.optimizer/clip.py → rank_and_select
5. UpdateSelected edits are applied to the skill document, producing a candidate skill (patch / rewrite modes).optimizer/skill.py, update_modes.py
6. GateThe candidate is scored on the selection split and accepted only if it improves (see §5.3).evaluation/gate.py → evaluate_gate
+
+ +
+

5.3 Validation Gate #

+

evaluate_gate is a pure decision function. It compares the candidate's selection-set score against the current and best skills:

+
    +
  • accept_new_best — candidate > current and candidate > best → becomes both current and best.
    • +
  • accept — candidate > current but ≤ best → becomes current only.
    • +
  • reject — candidate ≤ current → discarded; current/best unchanged.
    • +
+

The comparison metric is configurable via evaluation.gate_metric:

+
+ + + + + + +
MetricScore used
hard defaultExact-match / discrete score
softPartial-credit / continuous score
mixedWeighted blend, controlled by gate_mixed_weight
+
When to use soft/mixed +

The soft/mixed metrics (contributed config configs/examples/soft_gate.yaml) help when the selection split is small and rewards are continuous, where a discrete hard gate may reject every candidate and stall training. Paper numbers use the default hard gate.

+
+
+ +
+

5.4 Slow Update (Momentum) #

+

At each epoch boundary (from epoch 2), the slow update rolls out both the previous epoch's skill and the current skill on the same sampled tasks, categorizes items (improved / regressed / persistent-fail / stable-success), and asks the optimizer to write a free-form guidance block. This guidance lands in a protected region of the skill that step-level edits cannot touch — only the slow update overwrites it. It is SkillOpt's analogue of momentum, countering cross-epoch forgetting.

+

Acceptance has two modes, selected by optimizer.slow_update_gate_with_selection:

+
+ + + + + +
ModeBehavior
false default — force-injectedGuidance is injected into both current and best skills unconditionally. The longitudinal guidance always persists; it is not gated by step-level selection scores.
true — gatedThe slow-update candidate is scored on the selection split and accepted/rejected through the same validation gate as step-level updates.
+
+ +
+

5.5 Meta Skill (Optimizer Memory) #

+

The meta skill is optimizer-side memory — it never modifies the target skill document. At the end of each epoch (skipped for epoch 1), the optimizer compares the previous and current epoch's last-step skills on the same sampled tasks and writes a compact, evidence-based reflection on what kind of edits helped or hurt. That memory is then injected as extra context into the next epoch's reflect / merge / learning-rate / ranking stages, so the optimizer accumulates strategy across the run.

+
+ +
+

5.6 Skill Document Anatomy #

+

A skill document is plain Markdown. Initial skills can be empty (learn from scratch) or seeded with domain knowledge via env.skill_init. During training the document accrues rules, patterns, and edge-case handling through accepted edit patches. A dedicated protected region holds the slow-update guidance, delimited by HTML-comment markers:

+
# Question Answering Skill
+
+## Learned rules ...
+- When the context contains multiple candidates, prefer ...
+
+<!-- SLOW_UPDATE_START -->
+# (epoch-level longitudinal guidance — only the slow update writes here)
+<!-- SLOW_UPDATE_END -->
+

Helpers in optimizer/slow_update.py manage this region: inject_empty_slow_update_field (placeholder at epoch 1), extract_slow_update_field (read), and replace_slow_update_field (overwrite). Step-level edits are blocked from modifying anything inside the markers.

+
+ + +
+

6.1 Configuration System #

+

Configs are structured YAML with section blocks (model, train, gradient, optimizer, evaluation, env) and _base_ inheritance. A benchmark config inherits the shared defaults and overrides only what differs:

+
# configs/searchqa/default.yaml
+_base_: ../_base_/default.yaml
+train:
+  train_size: 400
+  batch_size: 40
+optimizer:
+  learning_rate: 4
+env:
+  name: searchqa
+  split_dir: data/searchqa_split
+

Override any key at the command line without editing files:

+
python scripts/train.py --config configs/searchqa/default.yaml \
+  --cfg-options optimizer.learning_rate=16 optimizer.lr_scheduler=linear
+
Reading the tables below +

Each section lists the key (relative to its YAML block), type, default (from configs/_base_/default.yaml), allowed values, and meaning. Defaults shown are the shipped base defaults.

+
+
+ +
+

6.2 model.* #

+
+ + + + + + + + + + + + + + +
KeyTypeDefaultDescription / options
backendstrazure_openaiHigh-level backend label for the run.
optimizerstrgpt-5.5Optimizer model deployment (writes skill edits).
targetstrgpt-5.5Target model deployment (executes tasks).
optimizer_backendstropenai_chatClient path for the optimizer: openai_chat or claude_chat.
target_backendstropenai_chatClient path for the target: openai_chat / claude_chat / qwen_chat / codex_exec / claude_code_exec.
reasoning_effortstrmediumlow / medium / high / xhigh / max (or empty).
rewrite_reasoning_effortstr""Override effort for full-rewrite calls (empty = inherit).
rewrite_max_completion_tokensint64000Token cap for full-rewrite optimizer calls.
azure_openai_endpointstr""Azure resource URL (or via AZURE_OPENAI_ENDPOINT).
azure_openai_api_versionstr2024-12-01-previewAzure API version header.
azure_openai_auth_modestr""api_key / azure_cli / managed_identity / openai_compatible (empty → env default).
+
Separate optimizer / target endpoints +

Every azure_openai_* key also has optimizer_azure_openai_* and target_azure_openai_* variants, letting you point the optimizer and target at different Azure resources. Exec backends (codex_exec, claude_code_exec) add their own codex_exec_* / claude_code_exec_* knobs (sandbox, reasoning effort, SDK mode, etc.).

+
+
+ +
+

6.3 train.* #

+
+ + + + + + + + +
KeyTypeDefaultDL analogyDescription
num_epochsint4EpochsNumber of training epochs.
train_sizeint0Train-set size0 = derive from the dataset split. (Fixed by split size when using split_dir.)
batch_sizeint40Batch sizeTasks rolled out per optimization step.
accumulationint1Grad accumulationAccumulation rounds per step.
seedint42Random seedReproducibility seed.
+
+ +
+

6.4 gradient.* #

+
+ + + + + + + + +
KeyTypeDefaultDescription
minibatch_sizeint8Trajectories per reflect minibatch.
merge_batch_sizeint8Patches per merge batch during aggregation.
analyst_workersint16Parallel reflection workers (data parallelism).
max_analyst_roundsint3Max rounds of analyst reflection per step.
failure_onlyboolfalseReflect only on failed trajectories when true.
+
+ +
+

6.5 optimizer.* #

+
+ + + + + + + + + + + + + + + + +
KeyTypeDefaultDL analogyDescription / options
learning_rateint4Learning rateMax edit patches applied per step (the "edit budget").
min_learning_rateint2Min LRFloor edit budget for decaying schedulers.
lr_schedulerstrcosineLR scheduleconstant / linear / cosine / autonomous.
lr_control_modestrfixed—fixed / autonomous / none.
skill_update_modestrpatch—patch / rewrite_from_suggestions / full_rewrite_minibatch.
use_slow_updatebooltrueMomentumEnable epoch-boundary slow update.
slow_update_samplesint20—Tasks sampled for the longitudinal comparison.
slow_update_gate_with_selectionboolfalse—false = force-inject guidance; true = gate it on the selection split (see §5.4).
longitudinal_pair_policystrmixed—mixed / changed / unchanged — which comparison pairs to keep.
use_meta_skillbooltrueMeta-learningEnable cross-epoch optimizer memory.
use_skill_aware_reflectionboolfalse—EmbodiSkill-style failure routing: SKILL_DEFECT (rule wrong/missing → gated body edit) vs EXECUTION_LAPSE (valid rule not followed → reminder appended to a protected appendix region that step-level edits never modify). Off = baseline-identical; resolved process-wide, works on every benchmark. Not supported with rewrite_from_suggestions / full-rewrite modes.
skill_aware_appendix_sourcestrboth—both (success analyst may also re-emphasize rules) / failure_only (paper-faithful S_app: failure side only).
skill_aware_consolidate_thresholdint0—>0: LLM-compact the appendix once it exceeds N notes (experimental); 0 = off.
+
+ +
+

6.6 evaluation.* #

+
+ + + + + + + + + +
KeyTypeDefaultDescription / options
use_gatebooltrueValidation gating is mandatory in this branch (must remain true).
gate_metricstrhardhard / soft / mixed — score used by the gate (see §5.3).
gate_mixed_weightfloat0.5Weight on the soft score when gate_metric = mixed.
sel_env_numint0Selection-split eval size (0 = use full split).
test_env_numint0Test-split eval size (0 = use full split).
eval_testbooltrueRun a final test evaluation after training.
+
Gate is required +

Setting evaluation.use_gate: false raises an error — validation gating cannot be disabled in this branch.

+
+
+ +
+

6.7 env.* #

+
+ + + + + + + + + + + +
KeyTypeDefaultDescription
namestr""Benchmark name (searchqa, docvqa, alfworld, …). Selects the env module.
skill_initstr""Path to a seed skill (empty = start from scratch).
split_modestrratioratio or split_dir (see §4.3).
split_dirstr""Pre-split directory (when split_mode = split_dir).
data_pathstr""Single dataset path (when split_mode = ratio).
split_seedint42Seed for deterministic ratio splitting.
exec_timeoutint120Per-task target/code-agent timeout (seconds).
out_rootstr""Output directory for the run.
+
Benchmark-specific env keys +

Env blocks may carry extra benchmark-specific keys (e.g. max_turns, workers, max_completion_tokens, limit). Unmapped env keys are passed straight through to the benchmark adapter — check the relevant configs/<benchmark>/default.yaml.

+
+
+ + +
+

7.1 Supported Benchmarks #

+
+ + + + + + + + + +
BenchmarkTypeConfig
SearchQAQuestion answeringconfigs/searchqa/default.yaml
DocVQADocument QAconfigs/docvqa/default.yaml
ALFWorldEmbodied agentconfigs/alfworld/default.yaml
LiveMathematicianBenchMath reasoningconfigs/livemathematicianbench/default.yaml
SpreadsheetBenchSpreadsheet code generationconfigs/spreadsheetbench/default.yaml
OfficeQATool-augmented QAconfigs/officeqa/default.yaml
+

Each benchmark is a self-contained module under skillopt/envs/<benchmark>/ with an adapter.py, dataloader.py, rollout.py, and evaluator.py (some add a custom reflect.py). Packaged reference skills live in ckpt/<benchmark>/.

+
+ +
+

7.2 Add a New Benchmark #

+

Use skillopt/envs/_template/ as a starting point. At minimum, implement:

+
    +
  1. Dataloader — read your item JSON into the framework's item dicts (dataloader.py).
    2. +
  2. Rollout — run the target on one item with the current skill and return a trajectory + score (rollout.py).
    3. +
  3. Evaluator — score predictions against ground truth (evaluator.py).
    4. +
  4. Adapter — wire the above into the trainer's expected interface and register the env name (adapter.py).
    5. +
+

Then add a configs/<name>/default.yaml inheriting _base_/default.yaml and set env.name to your new benchmark.

+
+ + +
+

8.1 Module Map #

+
+ + + + + + + + + + +
ModuleResponsibility
skillopt/config.pyLoad structured YAML, resolve _base_ inheritance, flatten to the trainer's flat dict, apply CLI overrides.
skillopt/engine/trainer.pyReflACTTrainer — orchestrates the whole loop, gating, slow update, meta skill, resume, and artifact writing.
skillopt/gradient/Reflection ("backward pass"): reflect.py analysts, aggregate.py patch merging.
skillopt/optimizer/The "optimizer": edit application, learning-rate scheduling, edit selection, slow update, meta skill, rewrite modes.
skillopt/evaluation/gate.pyPure accept/reject decision and metric selection.
skillopt/model/Backend clients (OpenAI/Azure, Claude, Qwen, Codex/Claude-Code exec) and routing.
skillopt/envs/<b>/Per-benchmark dataloader, rollout, evaluator, adapter.
+
+ +
+

8.2 Core Functions #

+
+ + + + + + + + + + + + + + + +
FunctionFilePurpose
load_config / flatten_config / apply_overridesconfig.pyLoad YAML with inheritance; flatten sections; apply key=value overrides.
run_minibatch_reflectgradient/reflect.pyRun error/success analysts over trajectory minibatches → edit patches.
merge_patchesgradient/aggregate.pyHierarchically merge semantically similar patches.
rank_and_selectoptimizer/clip.pyRank edits and clip to the learning-rate budget.
build_scheduleroptimizer/scheduler.pyConstruct the LR (edit-budget) scheduler: constant/linear/cosine/autonomous.
decide_autonomous_learning_rateoptimizer/lr_autonomous.pyLet the optimizer pick the next learning rate (autonomous mode).
apply_patch / apply_editoptimizer/skill.pyApply edits to the skill document (respecting the protected region).
rewrite_skill_from_suggestionsoptimizer/rewrite.pyFull-rewrite update mode from accumulated suggestions.
evaluate_gate / select_gate_scoreevaluation/gate.pyAccept/reject decision; compute hard/soft/mixed score.
run_slow_updateoptimizer/slow_update.pyProduce epoch-boundary longitudinal guidance.
replace_slow_update_field / extract_slow_update_fieldoptimizer/slow_update.pyRead/overwrite the protected guidance region.
run_meta_skill / format_meta_skill_contextoptimizer/meta_skill.pyGenerate cross-epoch optimizer memory and render it into reflection context.
+
+ +
+

8.3 CLI Scripts #

+

scripts/train.py

+

Runs a full training loop. Required: --config. Override config via --cfg-options section.key=value … or legacy flat flags (--num_epochs, --batch_size, --optimizer_model, --target_model, --lr_scheduler, --edit_budget, --split_dir, …).

+

scripts/eval_only.py

+

Evaluates a skill document without training. Required: --config and --skill. Use --split to choose train / valid_seen / valid_unseen / all.

+
python scripts/eval_only.py \
+  --config configs/searchqa/default.yaml \
+  --skill outputs/my_run/best_skill.md \
+  --split valid_unseen
+
+ +
+

8.4 WebUI #

+

An optional Gradio dashboard to configure parameters and monitor runs:

+
pip install -e ".[webui]"
+python -m skillopt_webui.app          # http://localhost:7860
+python -m skillopt_webui.app --share  # public share link
+
+ + + + + + +
FlagDefaultDescription
--port7860Server port.
--host0.0.0.0Bind address.
--shareoffCreate a public Gradio share link.
+
+ +
+

9.1 SkillOpt-Sleep — the deployment-time companion (preview) #

+

SkillOpt-Sleep applies SkillOpt's discipline to your own daily usage. It gives a + local coding agent a nightly sleep cycle that reviews your past sessions, replays your + recurring tasks on your own API budget, and consolidates what it learns into validated + long-term memory and skills — behind a held-out gate, staged for your review. The agent gets better + the more you use it, with no weight training and zero inference-time overhead. It is an early + preview we are actively iterating on; interfaces and defaults may change.

+

One "night":

+
harvest Claude Code / Codex transcripts → mine recurring tasks → replay offline
+   → consolidate (reflect → bounded edit → GATE on real held-out tasks)
+   → stage proposal → (you) adopt
+

The engine lives in the top-level skillopt_sleep/ package with zero dependency + on the paper's skillopt/ experiment code (the validation gate is vendored). Deterministic + proof, no API key required: + python -m skillopt_sleep.experiments.run_experiment --persona researcher --assert-improves.

+ +

9.2 Plugins (three agents) #

+

One engine, thin per-agent shells (see plugins/):

+
+ + + + + + +
PlatformFolderInstall
Claude Codeplugins/claude-code/plugin marketplace add ./plugins/claude-code → /skillopt-sleep
Codexplugins/codexbash plugins/codex/install.sh → skillopt-sleep skill
Copilotplugins/copilotregister plugins/copilot/mcp_server.py as an MCP server
+

Transcript source and replay backend are separate knobs: --source claude for Claude Code + transcripts, --source codex for Codex Desktop archived sessions under + ~/.codex/archived_sessions, and --backend codex only when you want the + replay/optimizer to spend Codex budget.

+ +

9.3 Experience replay & dream rollouts (opt-in) #

+

Two consolidation mechanisms, both default off (so behavior is unchanged unless + enabled). They strengthen the nightly update when your tasks have a clean correctness signal; the + validation gate still governs what ships.

+
+ + + + + + +
Config knobDefaultEffect
dream_rollouts1Run each task K times and learn from the good-vs-bad contrast (contrastive reflection).
recall_k0Associative recall — pull the K most-similar past tasks (from a persisted archive) into tonight's dream.
dream_factor0Add N lightweight synthetic variants of each task.
+

On a clean-signal benchmark the gain scales with recall depth (deployment protocol: 5 nights × + 10 new real tasks/night, full held-out test, GPT-5.5, gated): recall_k=10 → +3.1 pts, + recall_k=20 → +4.5 pts, full-history replay reference → +5.6 pts; a second benchmark + (SpreadsheetBench, GPT-5.4-nano, gate-free) gives +3.6 pts. On saturated or noisy tasks the effect is + flat within run-to-run noise (±1–2 pts). Keep the gate on; it bounds the downside.

+ +
+ SkillOpt — Executive Strategy for Self-Evolving Agent Skills · + github.com/microsoft/SkillOpt · + arXiv:2605.23904
+ This guide reflects the current configuration defaults in configs/_base_/default.yaml. When in doubt, the code is the source of truth. +
+
+ +
+ + + + +
+ + + + diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..2dfe6da --- /dev/null +++ b/docs/index.md @@ -0,0 +1,170 @@ +--- +hide: + - navigation +--- + +
+ +# SkillOpt + +### Train Agent Skills Like Neural Networks + +*Optimize natural-language skill documents through iterative rollout, reflection, and gated validation — with epochs, learning rates, and validation gates — without touching model weights.* + +[Get Started :material-rocket-launch:](guide/installation.md){ .md-button .md-button--primary } +[View on GitHub :material-github:](https://github.com/microsoft/SkillOpt){ .md-button } + +
+ +--- + +## How It Works + +
+
+ +
+
🎯
+
Rollout
+
Target executes tasks
+
+ +
+ +
+
🔍
+
Reflect
+
Optimizer analyzes trajectories
+
+ +
+ +
+
🔗
+
Aggregate
+
Merge edit patches
+
+ +
+ +
+
✂️
+
Select
+
Rank & clip edits
+
+ +
+ +
+
📝
+
Update
+
Apply to skill doc
+
+ +
+ +
+
🚦
+
Gate
+
Validate & accept
+
+ +
+ +
+
🔄 Slow Update
+
🧠 Meta Skill
+
Epoch Boundary
+
+ +
+ +--- + +## Deep Learning Analogy + +SkillOpt brings the familiar deep-learning training paradigm to agentic prompt optimization: + +| Deep Learning | SkillOpt | +|---|---| +| Model weights | Skill document (Markdown) | +| Forward pass | Rollout (target executes tasks) | +| Loss / gradient | Reflect (optimizer produces edit patches) | +| Gradient clipping | Edit selection (`learning_rate` = max edits) | +| SGD step | Patch application to skill | +| Validation set | Gated evaluation on selection split | +| LR schedule | `lr_scheduler`: cosine, linear, constant | +| Epochs | Multi-epoch with slow update & meta skill memory | + +--- + +## Supported Benchmarks + +| Benchmark | Type | Config | +|---|---|---| +| **DocVQA** | Document QA | `configs/docvqa/` | +| **ALFWorld** | Embodied AI | `configs/alfworld/` | +| **OfficeQA** | Enterprise QA | `configs/officeqa/` | +| **SearchQA** | Open-domain QA | `configs/searchqa/` | +| **LiveMathBench** | Math reasoning | `configs/livemathematicianbench/` | +| **SWEBench** | Software Engineering | `configs/swebench/` | +| + 5 more | Various | See [docs](guide/first-experiment.md) | + +--- + +## Quick Example + +```bash +# Install +pip install -e . + +# Configure credentials +export AZURE_OPENAI_ENDPOINT="https://your-resource.openai.azure.com/" +export AZURE_OPENAI_API_KEY="your-key" + +# Train on SearchQA +python scripts/train.py --config configs/searchqa/default.yaml + +# Evaluate best skill +python scripts/eval_only.py \ + --config configs/searchqa/default.yaml \ + --skill outputs/best_skill.md +``` + +--- + +
+ +- :material-book-open-variant:{ .lg .middle } **Getting Started** + + --- + + Install SkillOpt, configure your API keys, and run your first experiment in 5 minutes. + + [:octicons-arrow-right-24: Installation](guide/installation.md) + +- :material-puzzle:{ .lg .middle } **Add a Benchmark** + + --- + + Extend SkillOpt with your own benchmark in ~100 lines of code. + + [:octicons-arrow-right-24: Extension Guide](guide/new-benchmark.md) + +- :material-cog:{ .lg .middle } **Configuration** + + --- + + Full reference for all hyperparameters with deep learning analogies. + + [:octicons-arrow-right-24: Config Reference](reference/config.md) + +- :material-monitor-dashboard:{ .lg .middle } **WebUI** + + --- + + Configure, launch, and monitor training from your browser. + + [:octicons-arrow-right-24: WebUI Guide](guide/first-experiment.md#webui) + +
diff --git a/docs/reference/api.md b/docs/reference/api.md new file mode 100644 index 0000000..8e364c7 --- /dev/null +++ b/docs/reference/api.md @@ -0,0 +1,195 @@ +# API Reference + +This page documents the public Python API SkillOpt exposes for **extending the +framework** with new environments / benchmarks. For ready-made adapters, +browse [`skillopt/envs/`](https://github.com/microsoft/SkillOpt/tree/main/skillopt/envs). + +> **Source of truth.** The classes below are real Python ABCs defined in +> `skillopt/envs/base.py`, `skillopt/datasets/base.py`, `skillopt/types.py`, +> and `skillopt/evaluation/gate.py`. If this page ever drifts, the code +> wins — please open an issue. + +--- + +## Core Classes + +### `EnvAdapter` + +`skillopt/envs/base.py` — abstract adapter that connects the SkillOpt +trainer to an environment (benchmark, simulator, REST API, ...). +Subclasses **must** implement the five abstract methods below. + +```python +from abc import ABC, abstractmethod +from skillopt.datasets.base import BaseDataLoader, BatchSpec + +class EnvAdapter(ABC): + + # ── Lifecycle hooks (have defaults; override only if needed) ──────── + + def setup(self, cfg: dict) -> None: ... + def get_dataloader(self) -> BaseDataLoader | None: ... + def requires_ray(self) -> bool: ... # default False + + # ── Abstract methods (subclasses MUST implement) ──────────────────── + + @abstractmethod + def build_train_env(self, batch_size: int, seed: int, **kwargs): + """Return an environment-manager object to be passed to rollout().""" + + @abstractmethod + def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs): + """Like build_train_env() but for a fixed eval split.""" + + @abstractmethod + def rollout(self, env_manager, skill_content: str, + out_dir: str, **kwargs) -> list[dict]: + """Run a batch of episodes with the current skill. + + Each returned dict MUST contain: + - "id": str episode/task identifier + - "hard": int (0|1) pass/fail (may be float 0.0-1.0 if smoothed) + - "soft": float partial-credit score in [0.0, 1.0] + It MAY contain env-specific extra keys (parsed into RolloutResult.extras). + """ + + @abstractmethod + def reflect(self, results: list[dict], skill_content: str, + out_dir: str, **kwargs) -> list[dict | None]: + """Turn rollout results into a list of raw patch dicts. + + Each dict (or None to drop the slot) MUST contain: + - "patch": {"edits": [...]} a Patch.to_dict() payload + - "source_type": "failure" | "success" + """ + + @abstractmethod + def get_task_types(self) -> list[str]: + """Distinct task-type strings used for stratified sampling.""" +``` + +The trainer also calls a few default-implemented helpers on every adapter: +`build_reference_text`, `get_reference_metadata`, `attach_reference_context`, +`select_representative_items`, and `build_env_from_batch`. Read the docstrings +in `skillopt/envs/base.py` if you need to override any of these — most +benchmarks don't. + +### `BaseDataLoader` / `SplitDataLoader` + +`skillopt/datasets/base.py` — episode-planning loaders. + +```python +class BaseDataLoader(ABC): + def setup(self, cfg: dict) -> None: ... + @abstractmethod + def build_train_batch(self, batch_size: int, seed: int, **kwargs) -> BatchSpec: ... + @abstractmethod + def build_eval_batch(self, env_num: int, split: str, seed: int, **kwargs) -> BatchSpec: ... + +class SplitDataLoader(BaseDataLoader): + """Concrete base for dataset-backed envs with on-disk train/val/test splits. + + Subclasses only need to implement load_split_items() (and optionally + load_raw_items() if you also want ``split_mode='ratio'``). + """ + def load_split_items(self, split_path: str) -> list[dict]: ... + def load_raw_items(self, data_path: str) -> list[dict]: ... # optional +``` + +`SplitDataLoader` handles two layout modes: + +| `split_mode` | What it expects | +|---|---| +| `"split_dir"` | A directory with `train/`, `val/`, `test/` subdirs already split. | +| `"ratio"` | A raw dataset path + `split_ratio: "2:1:7"` style string. | + +In either case the items returned by `load_split_items()` are plain +`dict` objects with at minimum an `"id"` key. + +### `BatchSpec` + +`skillopt/datasets/base.py` — a slotted dataclass describing one batch +request the trainer hands to the adapter. + +```python +@dataclass(slots=True) +class BatchSpec: + phase: str # "train" | "eval" + split: str # "train" | "val" | "test" | "valid_seen" | ... + seed: int + batch_size: int + payload: object | None = None # what the loader produced (e.g. list[dict]) + metadata: dict = field(default_factory=dict) +``` + +### `Edit` / `Patch` + +`skillopt/types.py` — the I/O types Reflect / Aggregate / Update produce +and consume. + +```python +EditOp = Literal["append", "insert_after", "replace", "delete"] + +@dataclass +class Edit: + op: EditOp + content: str = "" + target: str = "" + support_count: int | None = None + source_type: Literal["failure", "success"] | None = None + merge_level: int | None = None + update_origin: str = "" + update_target: str = "" + +@dataclass +class Patch: + edits: list[Edit] = field(default_factory=list) + reasoning: str = "" + ranking_details: dict[str, Any] | None = None +``` + +Both types support `to_dict()` / `from_dict()` for serialization. + +### `RolloutResult` + +`skillopt/types.py` — the normalised rollout return type. The trainer +calls `RolloutResult.from_dict(...)` on each dict returned from +`EnvAdapter.rollout()`, so the only **hard** requirement on those dicts is +the three keys above (`id`, `hard`, `soft`). Extra fields are preserved +into `RolloutResult.extras`. + +### `GateResult` / `GateAction` + +`skillopt/evaluation/gate.py` — the validation-gate decision types +returned each epoch. + +--- + +## Registering an environment + +Environments are not registered via decorators or a `BENCHMARK_REGISTRY` +dict. The trainer keeps a lazy registry inside `scripts/train.py` — +`_ENV_REGISTRY` — populated by `_register_builtins()`. To add a new env +you append a `try / except ImportError` block there. See +[Add a New Benchmark](../guide/new-benchmark.md) for the full step-by-step. + +--- + +## Backends (model layer) + +The model layer lives under `skillopt.model.*`. Backends are selected +via `model.optimizer_backend` and `model.target_backend` in the config — +not via a base class subclass. Supported values (as of this writing): + +| Backend | Optimizer? | Target? | +|---|---|---| +| `openai_chat` | ✓ | ✓ | +| `claude_chat` | ✓ | ✓ | +| `qwen_chat` | ✓ | ✓ | +| `minimax_chat` | ✓ | ✓ | +| `codex_exec` | — | ✓ | +| `claude_code_exec` | — | ✓ | + +See `skillopt/model/backend_config.py` for the live whitelist and +[`docs/reference/config.md`](./config.md) for the per-backend +configuration keys. diff --git a/docs/reference/cli.md b/docs/reference/cli.md new file mode 100644 index 0000000..24b5325 --- /dev/null +++ b/docs/reference/cli.md @@ -0,0 +1,71 @@ +# CLI Reference + +## Training + +```bash +python scripts/train.py --config [overrides...] +``` + +### Arguments + +| Argument | Description | +|---|---| +| `--config` | Path to YAML config file (required) | +| `key=value` | Override any config parameter | + +### Examples + +```bash +# Basic training +python scripts/train.py --config configs/searchqa/default.yaml + +# With overrides +python scripts/train.py \ + --config configs/searchqa/default.yaml \ + --cfg-options optimizer.learning_rate=16 optimizer.lr_scheduler=linear + +# With custom initial skill +python scripts/train.py \ + --config configs/searchqa/default.yaml \ + --cfg-options env.skill_init=skills/my_seed.md +``` + +## Evaluation + +```bash +python scripts/eval_only.py --config --skill +``` + +### Arguments + +| Argument | Description | +|---|---| +| `--config` | Path to YAML config file (required) | +| `--skill` | Path to skill document to evaluate (required) | +| `--split` | Evaluation split: `test` (default), `valid`, `train` | + +### Examples + +```bash +# Evaluate best skill on test set +python scripts/eval_only.py \ + --config configs/searchqa/default.yaml \ + --skill outputs/searchqa/run_001/skills/best_skill.md + +# Evaluate on validation set +python scripts/eval_only.py \ + --config configs/searchqa/default.yaml \ + --skill outputs/searchqa/run_001/skills/best_skill.md \ + --split valid +``` + +## WebUI + +```bash +python -m skillopt_webui.app [--port PORT] [--share] +``` + +| Argument | Default | Description | +|---|---|---| +| `--port` | 7860 | Port number | +| `--share` | false | Create public Gradio link | diff --git a/docs/reference/config.md b/docs/reference/config.md new file mode 100644 index 0000000..0b39bd0 --- /dev/null +++ b/docs/reference/config.md @@ -0,0 +1,85 @@ +# Configuration Reference + +Complete reference for all SkillOpt configuration parameters. + +## Model + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `model.backend` | str | `azure_openai` | Backend: `azure_openai` / `openai_chat` / `claude_code_exec` / `qwen` | +| `model.optimizer` | str | `gpt-5.5` | Optimizer model (for reflection & slow update) | +| `model.target` | str | `gpt-5.5` | Target model (for rollout execution) | +| `model.reasoning_effort` | str | `medium` | Reasoning effort level | +| `model.optimizer_backend` | str | `openai_chat` | Optimizer backend: `openai_chat` / `claude_chat` / `qwen_chat` / `minimax_chat` | +| `model.target_backend` | str | `openai_chat` | Target backend: chat backends plus execution harnesses | +| `model.qwen_chat_base_url` | str | `http://localhost:8000/v1` | Shared Qwen/vLLM OpenAI-compatible endpoint | +| `model.qwen_chat_enable_thinking` | bool | `false` | Shared Qwen thinking flag | +| `model.optimizer_qwen_chat_base_url` | str | — | Optimizer-specific Qwen/vLLM endpoint; overrides shared `qwen_chat_base_url` | +| `model.target_qwen_chat_base_url` | str | — | Target-specific Qwen/vLLM endpoint; overrides shared `qwen_chat_base_url` | + +## Training (`train`) + +| Parameter | Type | Default | DL Analogy | Description | +|---|---|---|---|---| +| `train.num_epochs` | int | 4 | Epochs | Number of training epochs | +| `train.batch_size` | int | 40 | Batch size | Tasks sampled per step | +| `train.accumulation` | int | 1 | Gradient accumulation | Accumulation rounds per step | +| `train.seed` | int | 42 | Random seed | Reproducibility seed | + +## Gradient / Reflection (`gradient`) + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `gradient.minibatch_size` | int | 8 | Reflect minibatch size | +| `gradient.merge_batch_size` | int | 8 | Patch merge batch size | +| `gradient.analyst_workers` | int | 16 | Parallel reflection workers | +| `gradient.max_analyst_rounds` | int | 3 | Max rounds of analyst reflection | +| `gradient.failure_only` | bool | `false` | Only reflect on failures | + +## Optimizer (`optimizer`) + +| Parameter | Type | Default | DL Analogy | Description | +|---|---|---|---|---| +| `optimizer.learning_rate` | int | 4 | Learning rate | Max edit patches per step (edit budget) | +| `optimizer.min_learning_rate` | int | 2 | Min LR | Min edits for decay schedulers | +| `optimizer.lr_scheduler` | str | `cosine` | LR schedule | `constant` / `linear` / `cosine` / `autonomous` | +| `optimizer.skill_update_mode` | str | `patch` | — | `patch` / `rewrite_from_suggestions` / `full_rewrite_minibatch` | +| `optimizer.use_slow_update` | bool | `true` | Momentum | Epoch-boundary longitudinal comparison & guidance | +| `optimizer.slow_update_samples` | int | 20 | — | Samples for slow update evaluation | +| `optimizer.use_meta_skill` | bool | `true` | Meta-learning | Cross-epoch optimizer-side strategy memory | +| `optimizer.longitudinal_pair_policy` | str | `mixed` | — | `mixed` / `changed` / `unchanged` | + +## Evaluation (`evaluation`) + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `evaluation.use_gate` | bool | `true` | Enable validation gating (accept/reject updates) | +| `evaluation.eval_test` | bool | `true` | Run test evaluation after training | + +## Environment (`env`) + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `env.name` | str | — | Benchmark name (e.g., `searchqa`, `docvqa`) | +| `env.data_path` | str | — | Path to dataset | +| `env.skill_init` | str | — | Path to initial seed skill (optional) | +| `env.split_mode` | str | `ratio` | `ratio` or `split_dir` | +| `env.split_ratio` | str | `2:1:7` | Train:val:test ratio | +| `env.exec_timeout` | int | 120 | Per-task timeout in seconds | +| `env.out_root` | str | — | Output directory | + +## Azure OpenAI Credentials + +| Variable | Description | +|---|---| +| `AZURE_OPENAI_ENDPOINT` / `model.azure_openai_endpoint` | Azure resource endpoint | +| `AZURE_OPENAI_API_KEY` / `model.azure_openai_api_key` | Azure API key | +| `OPENAI_API_KEY` | OpenAI API key (for `openai_chat` backend) | +| `ANTHROPIC_API_KEY` | Anthropic API key (for `claude_code_exec` backend) | +| `QWEN_CHAT_BASE_URL` | Shared local vLLM endpoint for `qwen_chat` | +| `QWEN_CHAT_MODEL` | Shared served model name for `qwen_chat` | +| `QWEN_CHAT_API_KEY` | Optional API key for the shared Qwen endpoint | +| `OPTIMIZER_QWEN_CHAT_BASE_URL` | Optimizer-specific local vLLM endpoint | +| `OPTIMIZER_QWEN_CHAT_MODEL` | Optimizer-specific served model name | +| `TARGET_QWEN_CHAT_BASE_URL` | Target-specific local vLLM endpoint | +| `TARGET_QWEN_CHAT_MODEL` | Target-specific served model name | diff --git a/docs/sleep/README.md b/docs/sleep/README.md new file mode 100644 index 0000000..b4fd45b --- /dev/null +++ b/docs/sleep/README.md @@ -0,0 +1,110 @@ +# SkillOpt-Sleep 😴 — deployment-time companion (preview) + +**SkillOpt-Sleep** applies SkillOpt's discipline to your *own daily usage*. It gives a +local coding agent a nightly **sleep cycle** that reviews your past sessions, replays +your recurring tasks on your own API budget, and consolidates what it learns into +**validated** long-term memory and skills — behind a held-out gate, staged for your +review. The agent gets better the more you use it, with **no weight training** and +**zero inference-time overhead**. + +> **Preview.** This is an early preview we are actively iterating on; interfaces and +> defaults may change. The engine lives in the top-level [`skillopt_sleep/`](../../skillopt_sleep) +> package with **zero dependency** on the paper's `skillopt/` code (the validation gate +> is vendored). + +## How it works + +One "night": + +``` +harvest Claude Code / Codex transcripts → mine recurring tasks → replay offline + → consolidate (reflect → bounded edit → GATE on real held-out tasks) + → stage proposal → (you) adopt +``` + +It synthesizes **SkillOpt** (validation-gated bounded text edits), **Claude Dreams** +(offline consolidation; review-then-adopt), and the **agent-sleep** idea (short-term +experience → long-term competence). + +## How to use it + +### Quickest path: the `skillopt-sleep` CLI (pip) + +```bash +pip install skillopt # installs the engine + the `skillopt-sleep` command +skillopt-sleep dry-run # harvest + mine + replay, report only (changes nothing) +skillopt-sleep run # a full nightly cycle; the proposal is staged for review +skillopt-sleep status # show state + the latest staged proposal +skillopt-sleep adopt # apply the latest staged proposal +skillopt-sleep schedule # install a nightly cron entry for this project +``` + +The per-agent plugin shells below (Claude Code / Codex / Copilot) still come from the +repo; the CLI above is the standalone, pip-only way to run a cycle. + +One engine, thin per-agent shells (see [`plugins/`](../../plugins)): + +| Platform | Folder | Install | +|---|---|---| +| **Claude Code** | [`plugins/claude-code`](../../plugins/claude-code) | `/plugin marketplace add ./plugins/claude-code` → `/skillopt-sleep` | +| **Codex** | [`plugins/codex`](../../plugins/codex) | `bash plugins/codex/install.sh` → `skillopt-sleep` skill | +| **Copilot** | [`plugins/copilot`](../../plugins/copilot) | register `plugins/copilot/mcp_server.py` as an MCP server | + +Deterministic proof (no API key): +`python -m skillopt_sleep.experiments.run_experiment --persona researcher --assert-improves`. + +### Opt-in: experience replay & dream rollouts + +Two consolidation mechanisms, both default **off** (behavior is unchanged unless you +enable them). They strengthen the nightly update when your tasks have a clean +correctness signal; the validation gate still governs what ships. + +| Config knob | Default | Effect | +|---|---|---| +| `dream_rollouts` | `1` | Run each task K times → learn from the good-vs-bad contrast (contrastive reflection). | +| `recall_k` | `0` | Associative recall — pull the K most-similar past tasks (from a persisted archive) into tonight's dream. | +| `dream_factor` | `0` | Add N lightweight synthetic variants of each task. | + +## Results + +> 📊 **More results & analysis — the gate-safety stress test, experience-replay +> scaling, and the dream-diversity ablation — are in +> [`docs/sleep/RESULTS.md`](RESULTS.md).** The highlights: + +**Protocol (identical for every row below).** 5 nights × 10 new real "today" tasks +per night; the full held-out **test** split is scored before night 1 (baseline) and +after night 5 (after); optimizer = GPT-5.5; single seed (42); run through the exact +shipped engine (`skillopt_sleep.dream.dream_consolidate`). Numbers are absolute +held-out accuracy; **Δ** = `after − baseline` in percentage points. + +**(a) End-to-end on real agents — [gbrain-evals](https://github.com/garrytan/gbrain-evals) `skillopt-v1`.** +Deficient seed skills go **0.00 → 1.00** on the held-out set with **both Claude Code +and Codex** as the target agent (all 4 seeds, including a real tool-use loop). + +**(b) Experience replay scales the gain — SearchQA** (1,400-item held-out test, +SQuAD exact-match; target = GPT-5.5; **validation-gated**): + +| Replay config (`dream_rollouts=5`) | Baseline → After | Δ (pts) | +|---|---|---| +| `recall_k=10` | 0.802 → 0.834 | +3.1 | +| `recall_k=20` | 0.803 → 0.848 | **+4.5** | +| full-history replay *(reference, not a shipping default)* | 0.796 → 0.851 | +5.6 | +| `recall_k=10`, `dream_rollouts=8` *(more dreaming, same recall)* | 0.798 → 0.835 | +3.7 | + +The gain rises monotonically with how much relevant past experience is recalled. The +same SearchQA cell **without** the gate (`recall_k=10`) is 0.808 → 0.839 (+3.1). + +**(c) Second benchmark — SpreadsheetBench** (280-item held-out test; the agent's +generated openpyxl code is executed and compared cell-by-cell to a golden workbook; +target = GPT-5.4-nano; gate-free + the output-contract guardrail): 0.279 → 0.314 (**+3.6**). + +**(d) Honest scope.** These gains hold where tasks recur and have a checkable +correctness signal. On saturated or noisy benchmarks (e.g. a strong model already +near ceiling) the effect is **flat within run-to-run noise** — single-seed baseline +variance here is ±1–2 pts, so treat sub-~1.5 pt differences as noise. The validation +gate keeps the worst case bounded; keep it **on** by default. + +## Learn more + +Full reference (pipeline, the three plugins, the experience-replay knobs) is in the +**[Documentation & Reproduction Guide](https://microsoft.github.io/SkillOpt/docs/guideline.html#sleep)**. diff --git a/docs/sleep/RESULTS.md b/docs/sleep/RESULTS.md new file mode 100644 index 0000000..c43d686 --- /dev/null +++ b/docs/sleep/RESULTS.md @@ -0,0 +1,185 @@ +# SkillOpt-Sleep — results & analysis + +This is the evidence behind SkillOpt-Sleep: does a nightly, offline sleep cycle +actually make a *deployed* agent better, and is it safe to run unattended? We +answer with a controlled deployment-scale study — the same protocol the plugin +runs in production, scored on full held-out test sets. + +## Setup + +**Protocol (identical for every cell unless stated).** 5 nights; each night adds +**10 new real "today" tasks**; the skill carries over and is refined night to +night. The full held-out **test** split is scored before night 1 (*baseline*) and +after night 5 (*after*); **Δ = after − baseline** in percentage points. Optimizer +model = **GPT-5.5**; single seed (42); every number is produced by the exact +shipped engine `skillopt_sleep.dream.dream_consolidate` (the experiment harness and +the plugin cycle call the same function). + +**Benchmarks** (real evaluators, not format heuristics): + +| Benchmark | Held-out test | Scoring | +|---|---|---| +| SearchQA | 1,400 items | SQuAD exact-match vs gold | +| LiveMathematicianBench | 124 items | multiple-choice label (choices shuffled per item) | +| SpreadsheetBench | 280 items | the agent's generated openpyxl code is **executed**, output workbook compared cell-by-cell to a golden file | + +**Targets:** GPT-5.5, GPT-5.4-mini, GPT-5.4-nano. **Modes:** validation-gated +(default) and gate-free. + +--- + +## 1. The headline — the validation gate is what makes nightly self-evolution *safe* + +Self-evolution is easy to build and easy to ruin: an optimizer that accepts its +own "lessons" unconditionally can adopt a plausible-but-wrong rule and an obedient +model will follow it off a cliff. We reproduced exactly that failure, then showed +the gate prevents it. + +Stress case — **GPT-5.4-nano on SearchQA**, weak model on a single-sample (degraded) +reflection signal, same nights, same candidate edits, gate **off** vs **on**: + +| | Night 0 → Night 5 | Δ | +|---|---|---| +| **no gate** | 0.554 → **0.026** | **−52.8** | +| **with gate (default)** | 0.570 → 0.570 | 0.0 | + +Ungated, the optimizer learned "answer with the document-title string, verbatim"; +the model complied and accuracy collapsed night after night +(0.554 → 0.490 → 0.325 → 0.031 → 0.034 → 0.026). The gated twin **rejected every one +of those edits** and never lost a point. This single experiment is the core +argument for SkillOpt-Sleep's design, and why the gate ships **on by default**. + +--- + +## 2. Cross-model scaling — bigger gains where there's headroom + +The same protocol on a weaker target model (**GPT-5.4-nano**, optimizer = GPT-5.5) +produces substantially larger gains — because the weaker model has more room to +learn. This is the realistic "cheap deployed agent, strong overnight optimizer" +scenario: + +| Config (SearchQA, nano, gated) | Baseline → After | Δ | Night-by-night | +|---|---|---|---| +| **cumulative replay, nights=5** | 0.560 → **0.679** | **+11.9** | 0.560 → 0.626 → 0.665 → 0.665 → 0.665 → 0.679 | +| recall_k=20, nights=5 | 0.566 → 0.681 | +11.5 | 0.566 → 0.659 → 0.685 → 0.685 → 0.681 → 0.681 | +| cumulative, nights=8 | 0.562 → 0.657 | +9.5 | saturates after night 5 | + +Both replay strategies (cumulative and recall) agree within 0.4 pt — the gain is +robust across configurations. + +**Compared to GPT-5.5 on the same benchmark (SearchQA, gated):** + +| Target model | Best Δ | Baseline | Headroom | +|---|---|---|---| +| GPT-5.4-nano | **+11.9** | 0.560 | 44 pt | +| GPT-5.5 | +6.0 | 0.798 | 20 pt | + +The story: **SkillOpt-Sleep helps most where there's the most to learn** — weaker +deployed models benefit ~2× as much from the same nightly optimization. This is +also the economical deployment pattern (cheap inference model + one strong +overnight optimizer call). + +--- + +## 3. Experience replay turns a one-time bump into a climb + +The plugin's two opt-in knobs (`recall_k`, `dream_rollouts`) are what produce the +gains. On **SearchQA, GPT-5.5, gated** — the gain rises monotonically with how +much relevant past experience is recalled: + +| Replay (`dream_rollouts=5`) | Baseline → After | Δ | +|---|---|---| +| `recall_k=10` | 0.802 → 0.834 | +3.1 | +| `recall_k=20` | 0.803 → 0.848 | **+4.5** | +| full-history (reference, not a default) | 0.796 → 0.851 | +5.6 | + +And the curve genuinely **climbs across nights** rather than jumping once and +plateauing — full-history replay, gated, night by night: + +``` +0.798 → 0.814 → 0.854 → 0.854 → 0.854 → 0.858 +``` + +The gate accepts a new, better skill as late as **night 5** (0.854 → 0.858). +Replay-policy ablation (SearchQA, GPT-5.5): + +| Replay policy | Gate-free Δ | Gated Δ | +|---|---|---| +| none (tonight's tasks only) | +3.9 | +2.0 | +| **recall k=10 (shipped default-able)** | +5.1 | +4.4 | +| cumulative (full history) | +4.8 | +6.0 | + +Recall captures most of cumulative's benefit at a fraction of the per-night cost. + +--- + +## 4. Default hyperparameters are the sweet spot + +We swept `dream_factor`, `rollouts`, `per_night`, and `nights` on the nano cell +(SearchQA, gated) to verify the shipped defaults are well-tuned: + +| Variant | Δ | vs default (+11.9) | +|---|---|---| +| dream_factor=4 (default 2) | +8.8 | −3.1 | +| rollouts=10 (default 5) | +9.5 | −2.4 | +| per_night=15 (default 10) | +2.7 | −9.2 | +| nights=8 (default 5) | +9.5 | −2.4 | + +Every direction away from the default hurts. This means users get the best result +**out of the box** without tuning — the recipe is robust by design. + +--- + +## 5. Why these gains exist — the dream-diversity fix (and a rigor note) + +Reflection learns from the **contrast** between good and bad rollouts of the same +task, which requires the K dream rollouts to be *independent samples*. An early +version of the engine collapsed them to one cached sample, so contrastive +reflection never fired. Fixing that, then adding recall, is what produces the +gains in Sections 1–2. Measured across an 18-cell deployment sweep (3 benchmarks × +3 targets × 2 modes), under three engine configurations: + +| Engine configuration | mean Δ | worst-cell Δ | cells > +0.5 | cells < −0.5 | +|---|---|---|---|---| +| single-sample reflection (degraded) | −2.66 | **−52.8** | 7 / 18 | 5 / 18 | +| diverse rollouts (K=5), no recall | +0.24 | −4.0 | 6 / 18 | 7 / 18 | +| **diverse rollouts + recall (shipped)** | **+0.53** | **−2.4** | 7 / 18 | 7 / 18 | + +The catastrophic −52.8 is removed **at its source** by diverse rollouts: the same +gate-free nano-SearchQA cell goes 0.554 → **0.586 (+2.7)** with no gate at all once +the dream is fixed. Recall then lifts the grid mean and tightens the worst case. +This is **defense in depth, each layer measured**: diverse rollouts propose better +edits, recall remembers relevant experience, and the gate catches whatever still +slips through. + +--- + +## 6. End-to-end on real agents + +On the public [gbrain-evals](https://github.com/garrytan/gbrain-evals) `skillopt-v1` +benchmark — designed for exactly this learnable-gap setting — deficient seed skills +go **0.00 → 1.00** on the held-out set with **both Claude Code and Codex** as the +target agent (all 4 seeds, including a real tool-use loop), and the two agents +cross-verify each other's consolidated skills. + +--- + +## 7. Honest scope & limitations + +- **Where it helps:** recurring tasks with a checkable correctness signal and real + headroom. That is the plugin's actual use case (your repeated daily tasks and + house rules the agent keeps missing). +- **Where it's flat:** saturated tasks on strong models, or noisy tasks with a weak + learning signal — within run-to-run noise. +- **Single seed.** Cells aggregate one seed per config; treat sub-~1.5 pt + differences as noise. Spot seed-robustness check on the one flagged cell + (nano SearchQA gated): seeds 42/43/44 give −1.9 / +3.6 / +4.7 (3-seed mean + **+2.1**), i.e. the tabled −1.9 is a pessimistic draw, not the typical outcome. +- **Keep the gate on.** It is the difference between bounded downside (−2.4) and a + −52.8 collapse. Gate-free mode is for users who cannot hold out a validation set + and is additionally protected by the output-contract guardrail. + +--- + +Back to the module overview: [`docs/sleep/README.md`](README.md) · +full reference: [Documentation & Reproduction Guide](https://microsoft.github.io/SkillOpt/docs/guideline.html#sleep). diff --git a/docs/superpowers/specs/2026-06-07-skillopt-sleep-claude-code-plugin-design.md b/docs/superpowers/specs/2026-06-07-skillopt-sleep-claude-code-plugin-design.md new file mode 100644 index 0000000..e38d529 --- /dev/null +++ b/docs/superpowers/specs/2026-06-07-skillopt-sleep-claude-code-plugin-design.md @@ -0,0 +1,237 @@ +# SkillOpt Sleep — Claude Code self-evolving plugin (design) + +**Status:** approved-for-build (autonomous offline session, 2026-06-07) +**Author:** generated for Yifan Yang, executed autonomously while user is asleep +**Branch:** `feat/claude-code-sleep-plugin` (worktree `my_repo/SkillOpt-sleep`) + +--- + +## 1. One-paragraph summary + +`skillopt-sleep` is a Claude Code plugin that gives a user's local Claude +agent a nightly **sleep cycle**. While the user is offline, it (1) **harvests** +the day's real Claude Code session transcripts from `~/.claude`, (2) **mines** +them into discrete *task records* with checkable outcomes, (3) **replays / +"dreams"** those tasks offline using the user's own API budget, and (4) runs +the **SkillOpt optimizer loop** (reflect → bounded edit → held-out gate) to +consolidate short-term experience into long-term **memory** (`CLAUDE.md`) and +**skills** (`SKILL.md`). Only changes that pass a validation gate are kept, and +every change is written to a **review staging area** the user approves before it +touches live config — mirroring Claude Dream's "input store is never modified" +safety contract. The result: an agent that measurably gets better at *this +user's* recurring work, every night, with zero model-weight training. + +## 2. Why this is the right synthesis of the three ingredients + +| Ingredient | What we take from it | Where it lives in this design | +|---|---|---| +| **SkillOpt** (your paper/code) | Skill = trainable text state; bounded add/delete/replace edits under a textual learning rate; **held-out validation gate**; rejected-edit buffer; epoch-wise slow/meta update. | The `consolidate` stage *is* a single SkillOpt epoch, reusing `skillopt.optimizer.*` and `skillopt.evaluation.gate`. | +| **Claude Dreams** | Async offline job: read a memory store + 1–100 session transcripts → emit a **new, separate** reorganized memory store (dedup / merge / resolve contradictions / surface insights). Input never mutated; output reviewed then adopted or discarded. | The `harvest` + `consolidate-memory` stages and the **staging/adopt** safety model are modeled directly on Dreams. | +| **Agent Sleep paper** (2605.26099) | Agents need periodic offline consolidation: short-term experience buffer → synthetic replay/self-generated data → self-update; "sleep" turns episodes into durable competence. | The whole nightly schedule, the `replay` step, and the short-term→long-term framing. | + +The key novel claim this enables for the project (and a future paper section): +**SkillOpt's validation-gated bounded-edit optimizer is the missing "safe +update rule" for Dream-style memory consolidation.** Dreams reorganize memory +but don't *prove* the reorganization helps; the Sleep paper consolidates but +assumes weight updates. SkillOpt-Sleep consolidates **text** (memory + skills) +and **gates each change on replayed task performance**, so nightly evolution is +both weight-free and regression-protected. + +## 3. Goals / non-goals + +**Goals** +1. A working Claude Code plugin: scheduled (nightly/cron) **and** user-triggered (`/sleep`). +2. Look back over the user's real past prompts & trajectories from local `~/.claude` records. +3. Offline "dream training": re-run mined tasks (mock-env or fresh retry) on the user's budget. +4. Continuous evolution of **memory** (`CLAUDE.md`) and **skills** (`SKILL.md`) via the SkillOpt gate. +5. A reproducible experiment that answers: *does the nightly loop actually improve a held-out score?* +6. Safety: never silently overwrite user config; stage → user approves → adopt. + +**Non-goals (now)** +- Codex version (explicitly deferred by user; architecture keeps it pluggable). +- Anthropic managed Dreams API integration (we *emulate* Dreams locally; managed API is a future backend). +- Model fine-tuning / weight updates (out of scope by design — text-only). +- Fully unattended auto-adopt by default (opt-in; default is review-gated). + +## 4. The local data we read (verified on this machine) + +- **Prompt history:** `~/.claude/history.jsonl` — one JSON/line: `{display, pastedContents, timestamp, project}`. The cross-session list of every prompt the user typed, with project path + epoch-ms timestamp. +- **Full transcripts:** `~/.claude/projects//.jsonl` — one record/line. Record `type` ∈ {`user`,`assistant`,`mode`,`permission-mode`,`attachment`,`file-history-snapshot`,`last-prompt`,…}. User/assistant records carry `message` (role+content blocks), plus `cwd`, `gitBranch`, `timestamp`, `sessionId`, `version`, `userType`. ~215k transcripts present on this box. +- **Deployment targets we may evolve:** + - Project memory: `/CLAUDE.md` (and `~/.claude/CLAUDE.md` global). + - User skills: `~/.claude/skills//SKILL.md` (frontmatter: `name`, `description`, optional `allowed-tools`, `argument-hint`). + - Plugin skills under `~/.claude/plugins/...`. + +Everything stays **on-disk and local**; the only network calls are the LLM +optimizer/replay calls the user already pays for. + +## 5. Architecture + +### 5.1 The nightly Sleep Cycle (stages) + +``` + ┌────────────────────────── SLEEP CYCLE (one "night") ──────────────────────────┐ + │ │ + trigger → │ 1.HARVEST 2.MINE 3.REPLAY 4.CONSOLIDATE 5.STAGE │ → wake report + (cron or │ read ~/.claude scan sessions re-run tasks SkillOpt epoch: write to │ + /sleep) │ transcripts → → task records offline (mock or reflect→edit→ .skillopt-│ + │ + history w/ outcomes & fresh retry) under GATE on held-out sleep/ │ + │ checkable refs current skill/mem replay split staging/ │ + │ ↓ │ + │ 6.ADOPT (opt-in / user-approved) │ + └────────────────────────────────────────────────────────────────────────────────┘ +``` + +**1. Harvest** (`harvest.py`) +Read `history.jsonl` + per-project transcript JSONLs for a time window +(default: since last sleep, fallback last 24–72h). Group by project (`cwd` / +`project`). Emit normalized `SessionDigest` objects: ordered user prompts, +assistant final texts, tool-call summary, files touched (from +`file-history-snapshot`), git branch, errors seen, and **user-feedback signals** +(e.g. "still broken", "that's wrong", "perfect", re-asks of the same thing). + +**2. Mine** (`mine.py`) +Turn digests into `TaskRecord`s — the unit the optimizer trains on. A task is a +self-contained intent (the user's request) plus an *outcome label* and, where +possible, a **checkable reference**: +- *Explicit success/failure* from feedback signals ("works now" after N retries → the early attempts are failures, the fix is the success exemplar). +- *Self-consistency check*: re-derivable answers (math, lookups) get a reference; open-ended ones get an LLM-judge rubric instead. +- Each TaskRecord: `{id, project, intent, context_excerpt, attempted_solution, outcome ∈ {success,fail,mixed}, reference_kind ∈ {exact, rubric, none}, reference, tags}`. +Mining is itself an LLM call (the **miner**), prompt-tunable, with a deterministic regex/heuristic fallback for offline/no-key runs. + +**3. Replay / "Dream"** (`replay.py`) +For mined tasks, re-run the intent **offline** under the *current* skill+memory +to get a fresh trajectory & score. Two modes: +- `mock` (default, safe): reconstruct a sandboxed prompt from the task's captured context (no live repo mutation, no network side effects) and run the target model. Deterministic, cheap, safe to run unattended. +- `fresh` (opt-in): actually re-attempt in a throwaway git worktree of the project. Higher fidelity, heavier, never touches the user's working tree. +Scoring: exact-match / substring for `exact` refs; LLM-judge (0–1) for `rubric` refs; this yields the `hard`/`soft` scores SkillOpt already expects. + +**4. Consolidate** (`consolidate.py`) — *this is one SkillOpt epoch* +Reuse the existing optimizer pieces rather than reinventing: +- `reflect`: partition replayed tasks into failure/success minibatches → propose add/delete/replace edits to **skill** and a parallel proposer for **memory** (`CLAUDE.md`). (Memory consolidation also does Dream-style dedup/merge/contradiction-resolution over existing `CLAUDE.md` lines.) +- `aggregate` + `rank_and_select` under an **edit budget** (textual learning rate). +- `apply_patch_with_report` → candidate skill / candidate memory. +- **GATE** (`skillopt.evaluation.gate.evaluate_gate`): replay a *held-out* slice of tasks with the candidate; accept only if it strictly beats current. Rejected edits go to the rejected-edit buffer (negative feedback) exactly as in the paper. +- A **slow/meta** pass across nights (not just within one night) carries durable, cross-session lessons — the literal "short-term experience → long-term knowledge" of the Sleep paper. Per-night state persists in `~/.skillopt-sleep/state.json`. + +**5. Stage** (`staging/`) +Write `proposed_CLAUDE.md`, `proposed_SKILL.md`, a unified diff, and a +`sleep_report.md` (what changed, why, gate deltas, token cost) into +`/.skillopt-sleep/staging//`. **Nothing live is modified.** + +**6. Adopt** +`/sleep adopt` (or `auto_adopt: true` in config for power users) copies staged +files over the live `CLAUDE.md` / `SKILL.md`, after a `git`-style backup. This +is the only stage that mutates user-facing config, and it is explicit by default +— the Dreams "review the output, then adopt or discard" contract. + +### 5.2 Components & boundaries (each independently testable) + +``` +skillopt/sleep/ + __init__.py + types.py # SessionDigest, TaskRecord, ReplayResult, SleepConfig, SleepReport (dataclasses) + harvest.py # ~/.claude transcripts + history.jsonl -> list[SessionDigest] + mine.py # list[SessionDigest] -> list[TaskRecord] (LLM miner + heuristic fallback) + replay.py # TaskRecord + skill + memory -> ReplayResult (hard/soft) (mock | fresh) + consolidate.py # ReplayResults -> candidate skill+memory -> GATE -> accepted artifacts + memory.py # CLAUDE.md read/merge/dedup/diff (Dream-style) + protected-region markers + state.py # ~/.skillopt-sleep/state.json: last_sleep, night counter, slow/meta memory + staging.py # write/adopt staging dir, backups + cli.py # `python -m skillopt.sleep {run|status|adopt|harvest|dry-run}` + config.py # SleepConfig load/merge (defaults + ~/.skillopt-sleep/config.yaml) + optimizer_backend.py # thin: route reflect/judge to a chosen backend; mock backend for tests + +skillopt-sleep-plugin/ # the Claude Code plugin surface + .claude-plugin/plugin.json + commands/sleep.md # /sleep [run|status|adopt|dry-run] + commands/sleep-status.md + skills/skillopt-sleep/SKILL.md # so Claude knows how to drive the engine + hooks/hooks.json # optional: schedule + on-session-end harvest + scripts/* # shims that call `python -m skillopt.sleep ...` +``` + +**Reuse, don't fork:** `consolidate.py` calls into existing +`skillopt.optimizer.clip.rank_and_select`, `skillopt.gradient.aggregate.merge_patches`, +`skillopt.optimizer.skill.apply_patch_with_report`, and +`skillopt.evaluation.gate.evaluate_gate`. The sleep layer is an **EnvAdapter-shaped +shim** over the user's own life, not a new optimizer. + +### 5.3 Data flow (one task, end to end) + +``` +history.jsonl + .jsonl + └─harvest→ SessionDigest{prompts, finals, tools, feedback} + └─mine→ TaskRecord{intent, attempted, outcome, reference} + └─replay(current skill+mem)→ ReplayResult{hard, soft, trajectory} + └─reflect→ edits(skill), edits(memory) + └─rank/clip(edit_budget)→ candidate + └─GATE(replay held-out)→ accept? → staging/ → (adopt) live CLAUDE.md/SKILL.md +``` + +## 6. Scheduling & triggering + +- **Cron/scheduled:** documented `crontab` line + an optional Claude Code hook; default `0 3 * * *` (3am local; pick an off-:00 minute in practice). The engine is a plain CLI so it works under cron, systemd-timer, or the Claude Code scheduler. +- **User-triggered:** `/sleep run` (full cycle), `/sleep dry-run` (harvest+mine+replay, no edits), `/sleep status`, `/sleep adopt`. +- **On-session-end harvest (optional hook):** cheaply append the just-finished session to the night's buffer so the 3am run has fresh data without a full rescan. + +## 7. Safety model (hard requirements) + +1. **Never mutate live `CLAUDE.md`/`SKILL.md` except via explicit `adopt`** (or opt-in `auto_adopt`). Default = staged + reviewed (Dreams contract). +2. **Backups:** every adopt snapshots the prior file to `staging//backup/`. +3. **Read-only harvest:** transcripts are read, never written. +4. **`fresh` replay runs only in throwaway worktrees**, never the user's checkout; no `rm -rf`, no force-push, network off unless `replay.network: true`. +5. **Budget cap:** `max_tokens_per_night` + `max_tasks_per_night`; stop early when hit, log what was skipped (no silent truncation). +6. **Secret hygiene:** redact obvious secrets from digests before they enter prompts (reuse `_redact_*` ideas from trainer). +7. **PII/scope:** only harvest projects on an allowlist (default: the project the plugin is invoked in) or `projects: all` opt-in. + +## 8. Validation experiment — "does it actually improve?" + +A self-contained, **deterministic-by-default** experiment lives in +`skillopt/sleep/experiments/` and is the acceptance test for the whole idea. + +**Setup:** a synthetic "user persona" (e.g. *researcher who keeps asking for +arXiv-id extraction in a fixed format*, or *programmer who keeps mis-formatting +git commit messages*). We ship 12–20 tiny tasks with **exact checkable +references**, split into `replay` (train) and `holdout` (test). + +**Procedure:** +1. Score the holdout with an **empty** skill+memory → `baseline`. +2. Run `N` sleep nights (each: replay train slice → reflect → gated edit). +3. Score holdout with the evolved skill+memory → `after`. +4. Report `after − baseline`, accept/reject counts, edit count, tokens. + +**Two backends:** +- `mock` (default, **no API key, fully deterministic**): a scripted optimizer that proposes the known-good rule on failure and a scripted judge. Proves the *plumbing* (harvest→mine→replay→gate→adopt) monotonically improves the score and the gate blocks regressions. This is the CI-able acceptance test. +- `anthropic` (opt-in, uses `ANTHROPIC_API_KEY`): the real optimizer/judge, to demonstrate genuine lift on the persona tasks. + +**Success criteria:** +- Mock: `after > baseline`, gate rejects an injected harmful edit, adopt+backup works, re-run is reproducible. (Hard gate in CI.) +- Anthropic (when run): `after ≥ baseline` on holdout with ≥1 accepted, human-readable edit; documented in the wake-up report. + +## 9. Personas (the user's framing) → concrete recurring-task families + +- **Programmer:** commit-message conventions, repo-specific build/test commands, "always run X before Y", framework gotchas → consolidated into project `CLAUDE.md` + a `repo-workflow` skill. +- **Researcher:** citation/format preferences, experiment-logging habits, paper-section style, dataset-path memory → `research-prefs` skill + memory. +- **Finance/analyst:** report formatting, recurring data-pull recipes, terminology → `report-style` skill + memory. +The engine is domain-agnostic; the persona only changes which tasks get mined. + +## 10. Phased delivery + +- **Phase 0 — scaffold + types + harvest** (read-only, no API). Provable on this box's real `~/.claude`. +- **Phase 1 — mine + replay(mock) + consolidate + gate + staging**, with the **mock** optimizer backend and the deterministic experiment green. *(primary deliverable of the offline session)* +- **Phase 2 — plugin surface** (`/sleep`, skill, hooks, plugin.json) wired to the CLI. +- **Phase 3 — real Anthropic backend** for miner/reflect/judge + `fresh` replay in worktrees. +- **Phase 4 — slow/meta cross-night memory**, adopt automation, multi-project, polish + docs. + +This session targets **Phase 0 + Phase 1 fully**, **Phase 2 scaffolded**, and the +**deterministic experiment passing**, all committed (not pushed) for review. + +## 11. Open questions for the user (answer when awake) + +1. **Adopt policy:** keep default *review-gated*, or do you want `auto_adopt` for your own machine? +2. **Scope:** harvest only the invoked project, or all projects in `~/.claude/projects`? +3. **Real-API demo:** want me to spend live `ANTHROPIC_API_KEY` budget on the persona demo, or keep everything mock until you say go? +4. **Skill target:** evolve a *new* dedicated `skillopt-sleep`-managed skill, or also edit your existing hand-written skills in `~/.claude/skills`? +5. **Paper:** should this become a section/figure in the SkillOpt arXiv (Dream+Sleep framing as "deployment-time continual skill optimization")? +``` diff --git a/index.html b/index.html new file mode 100644 index 0000000..2be9a01 --- /dev/null +++ b/index.html @@ -0,0 +1,2736 @@ + + + + + + SkillOpt | Executive Strategy for Self-Evolving Agent Skills + + + + + + + +
+
+
+ Text-space optimization for frozen agents +

SkillOpt

+

+ Executive Strategy for Self-Evolving Agent Skills. SkillOpt treats a compact + natural-language skill document as the trainable state of a frozen language + agent, then learns that document through rollouts, reflection, bounded edits, + and held-out validation gates. +

+
+ Core Idea + Method + View Results + + + Code Repo + + + + Paper + + + + Video + +
+ + + + Related project + SkillLens studies model-generated agent skills. + A companion project page from Microsoft Research. + + + +
+ + +
+
+ +
+
+
+ Project Video +
+

SkillOpt in motion.

+

+ A short visual overview of how SkillOpt treats natural-language skills + as trainable artifacts: roll out, reflect, edit, validate, and export. +

+
+
+
+ +
+

+ Promotional video for the SkillOpt project page. The static paper teaser is shown below for high-resolution inspection. +

+
+ +
+
+ Paper Teaser +
+

The core loop at a glance.

+

+ The teaser summarizes the SkillOpt training loop: rollout evidence, + optimizer-side reflection, bounded skill edits, validation gating, + and the exported reusable skill. +

+
+
+
+ SkillOpt teaser figure showing the target model, optimizer model, bounded edits, validation gate, and exported best skill. +
+

+ Figure from the SkillOpt paper. On small screens, the figure area scrolls horizontally to preserve the original details. +

+
+ +
+
+
01 / Core Idea
+
+

Train the procedure, not the weights.

+

+ SkillOpt makes the skill document itself the optimization target. The + target model, backend, and harness stay fixed; the procedure that guides + evidence gathering, tool use, verification, and output formatting evolves. +

+
+
+ +
+
+

A skill is external state for an agent.

+

+ Instead of fine-tuning a model or hand-maintaining prompts, SkillOpt runs + the frozen agent on scored batches, asks a separate optimizer model to + propose structured edits, and accepts a candidate only when validation + performance improves. +

+
+ Frozen target model + Optimizer model + Add / delete / replace edits + Held-out gate +
+
+ +
+
+ Rollout +

The target model executes tasks with the current skill and records scored trajectories.

+
+
+ Reflect +

The optimizer analyzes success and failure minibatches to find reusable procedures.

+
+
+ Edit +

Candidate add, delete, and replace operations are merged and ranked under a budget.

+
+
+ Gate +

The candidate skill is kept only if it improves held-out selection performance.

+
+
+
+
+ +
+
+
02 / Method
+
+

A training loop for natural-language skills.

+

+ The loop deliberately mirrors a learning algorithm: rollout evidence acts + like a forward pass, reflection acts like a language-level backward pass, + and the textual learning rate bounds how far the skill can move. +

+
+
+ +
+
+

Evidence

+

Rollout batches capture messages, tool calls, verifier feedback, task metadata, and final scores.

+
+
+

Minibatches

+

Failures and successes are reflected separately so edits correct recurring errors while preserving working behavior.

+
+
+

Bounded Edits

+

An edit budget functions as a textual learning rate, preventing useful rules from being overwritten by broad rewrites.

+
+
+

Memory

+

Rejected edits, slow update, and optimizer-side meta skill provide longer-horizon feedback without bloating deployment.

+
+
+ +
+ SkillOpt pipeline showing rollout, reflection, bounded edits, validation gate, slow update, and meta skill. +
+ SkillOpt pipeline from the paper. The frozen target model executes with the current skill; the optimizer model proposes bounded edits; held-out validation decides whether the candidate becomes the new current skill. +
+
+
+ +
+
+
03 / Main Results
+
+

SkillOpt improves GPT and Qwen target models.

+

+ The table reports main-result gains across target models and + execution harnesses, comparing no-skill execution with the final + SkillOpt skill on held-out test splits. +

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Target modelHarnessSearchQASheetOfficeDocVQALiveMathALFWorldAvg gain
OpenAI logoGPT-5.5Direct chat+9.6+38.9+39.0+12.4+29.3+11.9+23.5
OpenAI logoGPT-5.4Direct chat+6.2+21.1+12.8+13.6+7.2+15.6+12.8
OpenAI logoGPT-5.4-miniDirect chat+4.3+11.4+26.7+16.5+4.8+12.7+12.7
OpenAI logoGPT-5.4-nanoDirect chat+19.0+8.2+33.7+49.4+4.0+35.1+24.9
OpenAI logoGPT-5.2Direct chat+11.2+18.9+21.5+16.5+15.2+16.4+16.6
Qwen logoQwen3.5-4BDirect chat+3.1+14.6+15.2+2.1+29.6+50.7+19.2
Qwen logoQwen3.6-35B-A3BDirect chat+7.6+9.3+1.2+3.8+10.4+22.4+9.1
OpenAI logoGPT-5.5Codex+5.5+57.5+12.8+5.0+28.0N/A+21.8
OpenAI logoGPT-5.5Claude Code+4.0+58.3+13.9+3.5+13.3N/A+18.6
+
+ +
+
+
+ Method comparison +

SkillOpt clears the strongest baseline on every benchmark.

+
+
+
+
+
+ +
+ +
+
+
04 / Ablations
+
+

The controls are doing real work.

+

+ The paper isolates the optimizer components that keep skill learning stable: + enough evidence, bounded textual updates, rejected-edit feedback, slow + update, and optimizer-side memory. +

+
+
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ComponentSettingSearchQASpreadsheetLiveMath
Learning ratelr=4 default87.177.561.3
Learning ratewithout lr84.675.757.3
Rejected bufferwith buffer87.177.561.3
Rejected bufferwithout buffer85.572.958.9
Update memorymeta skill + slow update87.177.561.3
Update memorywithout both86.355.059.7
+
+ +
+

What the ablations say

+
+
+ Bounded + Textual learning rates prevent destructive rewrites while keeping enough plasticity to learn new procedures. +
+
+ Gated + Held-out selection turns reflection into propose-and-test optimization rather than unconditional self-editing. +
+
+ Buffered + Rejected edits become negative feedback, helping the optimizer avoid repeating harmful directions. +
+
+
+
+ +
+ Epoch checkpoint trends for SpreadsheetBench, SearchQA, and LiveMath. +
+ Epoch checkpoint trends from the paper. Selection-best checkpoints are compared with train rollout score and unseen test performance. +
+
+
+ +
+
+
05 / Skill Evolution
+
+

A typical run turns failures into concrete operating rules.

+

+ This ALFWorld run uses GPT-5.4-mini as the frozen target model and + GPT-5.5 as the optimizer model. The plot tracks train rollout and + held-out selection scores; hover or focus a point to inspect the + skill edit proposed at that stage. +

+
+
+ +
+
+
+ ALFWorld / train-sel evolution +
+ Train rollout + Selection gate +
+
+
+ + ALFWorld skill evolution scores + Selection score rises from 68.6 percent to 81.4 percent, while rejected edits are visible as downward candidate points. + + + + + + + + 85% + 80% + 75% + 70% + 65% + base + step 1 + step 2 + step 3 + slow + step 4 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ Accepted edits become the current skill only after held-out selection improves. + Step 3 is rescued by a slow update; Step 4 trains higher but fails selection. +
+
+ + +
+ +
+
+ Run setup + Target model: GPT-5.4-mini. Optimizer model: GPT-5.5. The skill starts from a compact ALFWorld instruction file and is edited in text space. +
+
+ Selection rule + Candidate edits are accepted only when held-out selection improves the current best score. +
+
+ Outcome + The selected skill improves final ALFWorld test hard score from 70.9% to 85.8%. +
+
+
+ +
+
+
06 / Transfer
+
+

The exported skill behaves like a reusable artifact.

+

+ SkillOpt exports a compact best_skill.md. The paper tests + whether that artifact transfers across model sizes, execution harnesses, + and nearby benchmarks without further target-side optimization. +

+
+
+ +
+
+ Cross-model + +15.2 +

GPT-5.4 LiveMath skill transferred to GPT-5.4-nano on LiveMathBench.

+
+
+ Cross-harness + +31.8 +

Codex-trained SpreadsheetBench skill transferred into Claude Code.

+
+
+ Self-optimizer + +10.4 +

GPT-5.4-nano used as its own optimizer improved SpreadsheetBench over baseline.

+
+
+ Deployment + 1 file +

The target model consumes only the final skill, not optimizer memory.

+
+
+ +
+ A stronger optimizer model gives the largest gains, but the loop is not merely + distillation from a stronger model. Even matched target-as-optimizer settings + can discover useful edits when the update is constrained, buffered, and + validated. +
+
+ +
+
+
07 / BibTeX
+
+

Citation.

+

+ If you find SkillOpt useful, please cite the arXiv preprint below. +

+
+
+ +
+ + 
@article{yang2026skillopt,
+  title={Skillopt: Executive strategy for self-evolving agent skills},
+  author={Yang, Yifan and Gong, Ziyang and Huang, Weiquan and Yang, Qihao and Zhou, Ziwei and Huang, Zisu and Li, Yan and Gao, Xuemei and Dai, Qi and Liu, Bei and others},
+  journal={arXiv preprint arXiv:2605.23904},
+  year={2026}
+}
+
+
+ +
+ SkillOpt: Executive Strategy for Self-Evolving Agent Skills + Code / Citation +
+
+ + + diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..7fc32db --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,78 @@ +site_name: SkillOpt Documentation +site_url: https://microsoft.github.io/SkillOpt +site_description: "SkillOpt: Agentic Skill Optimization via Reflective Training Loops" +repo_url: https://github.com/microsoft/SkillOpt +repo_name: microsoft/SkillOpt + +theme: + name: material + palette: + - scheme: default + primary: indigo + accent: deep purple + toggle: + icon: material/brightness-7 + name: Switch to dark mode + - scheme: slate + primary: indigo + accent: deep purple + toggle: + icon: material/brightness-4 + name: Switch to light mode + features: + - navigation.instant + - navigation.tracking + - navigation.sections + - navigation.expand + - navigation.top + - content.code.copy + - content.tabs.link + - search.suggest + - search.highlight + icon: + repo: fontawesome/brands/github + font: + text: Inter + code: JetBrains Mono + + + +nav: + - Home: index.md + - Getting Started: + - Installation: guide/installation.md + - First Experiment: guide/first-experiment.md + - Configuration: guide/configuration.md + - Core Concepts: + - Training Loop: guide/training-loop.md + - Skill Document: guide/skill-document.md + - Deep Learning Analogy: guide/dl-analogy.md + - Extension Guides: + - Add a New Benchmark: guide/new-benchmark.md + - Local Environment Smoke Tests: guide/local-env-smoke.md + - Add a New Model Backend: guide/new-backend.md + - Reference: + - Configuration Reference: reference/config.md + - CLI Reference: reference/cli.md + - API Reference: reference/api.md + - Contributing: contributing.md + +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + - attr_list + - md_in_html + - toc: + permalink: true + +plugins: + - search diff --git a/plugins/README.md b/plugins/README.md new file mode 100644 index 0000000..c6fe040 --- /dev/null +++ b/plugins/README.md @@ -0,0 +1,262 @@ +# SkillOpt-Sleep — plugins for Claude Code, Codex, Copilot, and Devin + +**Your coding agent forgets everything between sessions. SkillOpt-Sleep fixes +that.** While you sleep, it reviews what you did today, notices the rules you +keep repeating ("always add a LIMIT", "answers in `\boxed{}`", "cite the +source"), and writes them into your agent's long-term memory and skills — but +only the rules that actually make it score better on *your own* past tasks. You +wake up to an agent that's better at *your* work, and you approve every change +before it sticks. + +One engine, four thin shells. It synthesizes **SkillOpt** (validation-gated +bounded text optimization — the research in this repo), **Claude Dreams** +(offline consolidation; input never mutated; review-then-adopt), and the **agent +sleep** idea (short-term experience → long-term competence). + +> **Open-source tool, decoupled from the research.** The engine lives in the +> top-level [`skillopt_sleep/`](../skillopt_sleep) package with **zero +> dependency** on the paper's `skillopt/` experiment code (the validation gate is +> vendored). Use it without the research stack. + +--- + +| Platform | Folder | Mechanism | Status | +|---|---|---|---| +| **Claude Code** | [`claude-code/`](claude-code) | `.claude-plugin` + `/skillopt-sleep` + `/skillopt-sleep-handoff` commands + skill + hooks | full, installable | +| **Codex** | [`codex/`](codex) | user-level `skillopt-sleep` skill + shared runner | full | +| **Copilot** | [`copilot/`](copilot) | MCP server (`sleep_*` tools) + `copilot-instructions` | full (MCP) | +| **Devin** | [`devin/`](devin) | MCP server (`sleep_*` tools) + Devin ATIF-v1.7 harvest + `.devin/rules` | full (MCP) | + +## Install (pick your agent) + +| Platform | Install | Then | +|---|---|---| +| **Claude Code** | `/plugin marketplace add microsoft/SkillOpt` → `/plugin install skillopt-sleep` | `/skillopt-sleep status` | +| **Codex** | `git clone` → `bash plugins/codex/install.sh` | `/skillopt-sleep status` | +| **Copilot** | `git clone` → register `plugins/copilot/mcp_server.py` as an MCP server | ask "run the sleep cycle" | +| **Devin** | `git clone` → `devin mcp add skillopt-sleep -- python3 plugins/devin/mcp_server.py` | ask "run the sleep cycle" | + +Requirements: Python ≥ 3.10 and the agent's CLI on PATH. All three call the same +[`run-sleep.sh`](run-sleep.sh) → `python -m skillopt_sleep`, so behaviour is +identical everywhere. Default backend is `mock` (no API spend); `--backend +claude|codex|copilot` uses your own budget. + +--- + +## How it works: one "night", in plain terms + +``` +harvest your past sessions → mine the tasks you keep doing → replay them offline + → reflect on failures → propose a few rule edits → KEEP only edits that raise + your held-out score → stage a proposal → (you) review & adopt +``` + +Nothing live changes until you `adopt`; every adopt backs up the prior file. + +### The split that keeps it honest: dream-train / real-val / real-test + +This is the heart of the design, borrowed from the SkillOpt paper's +train/selection/test protocol: + +| Split | Where it comes from | What it's for | +|---|---|---| +| **train** | your real tasks **+ optional "dreamed" variants** | what the optimizer *learns from*. Over-dreaming here is fine — it's imagination. | +| **val** (selection) | **your real tasks only**, held out | the **gate**: an edit is kept only if it raises this score. Stops overfitting. | +| **test** | **your real tasks only**, held out, never seen during optimization | the **final score** we report. Kept as close to your real usage as possible. | + +So you can **dream up extra training examples** to learn a rule robustly, while +the rule is still **judged on real, unseen tasks**. A `dream` task can *never* +land in val or test — that invariant is unit-tested. + +--- + +## What each feature does **for you** (with examples) + +Every control below works on all three platforms (pass it after the action, +e.g. `/skillopt-sleep run --rollouts-k 3`). + +### `--preferences "..."` — tell it your house rules + +The single most useful knob. Free text that steers what the optimizer writes, +as a prior. Use it to encode the conventions you're tired of repeating. + +```bash +# A backend engineer: +/skillopt-sleep run --preferences "Always use async/await, never callbacks. \ + Prefer pytest over unittest. Commit subjects in imperative mood under 50 chars." + +# A data analyst: +/skillopt-sleep run --preferences "Every SQL query must end with LIMIT 1000 unless \ + I say otherwise. Money in USD with 2 decimals. Prefer CTEs over nested subqueries." + +# A researcher: +/skillopt-sleep run --preferences "Cite sources as [Author, Year]. Math answers in \ + \\boxed{}. Keep explanations under 150 words unless I ask for depth." +``` +*What it does for you:* the next morning your agent already follows these +without you re-typing them, and the rules are validated against your real tasks +(if a "preference" actually hurts your held-out score, the gate drops it). + +### `--gate on|off` — strict vs. greedy + +- `on` (default): an edit is kept **only if it raises your held-out score**. + Safe — blocks plausible-but-wrong rules and reward-hacking. +- `off`: greedy — keep edits without the strict check (still reports whether + quality moved). + +*What it does for you:* leave it `on` for trust. Flip it `off` when you're +exploring and want to see everything the optimizer proposes. + +### `--rollouts-k K` — learn from contrast, not just failure + +Re-runs each task `K` times and learns from the difference between the **good** +and **bad** attempts, not just a single failure. + +```bash +/skillopt-sleep run --rollouts-k 3 +``` +*What it does for you:* a much stronger signal. If your agent gets a task right 1 +time in 3, the optimizer figures out *what the winning attempt did* and makes it +reliable. + +### `--optimizer-model` / `--target-model` — optimize cheap, deploy anywhere + +Use a strong model to *write* the rules and a cheap model to *run* your tasks. +The learned skill then helps the cheap model — or any model. + +```bash +/skillopt-sleep run --optimizer-model sonnet --target-model haiku +``` +*What it does for you:* spend a little on a smart optimizer overnight; your +everyday cheap/fast agent inherits the upgrade. (Verified: a skill optimized on +one model lifts a different one — cross-model and even cross-runtime +Codex↔Claude.) + +### `--budget-tokens N` / `--budget-minutes M` — cap the spend + +You decide how much the nightly "dreaming" costs; it auto-plans how many nights +× how many rollouts fit. + +```bash +/skillopt-sleep run --backend claude --budget-tokens 60000 +``` +*What it does for you:* predictable cost. It stops cleanly when the budget is hit +and tells you what it skipped. + +### multi-objective (accuracy ↑, tokens ↓, latency ↓) + +The reward can weight not just correctness but **cost and speed**, so a skill can +learn to be cheaper and faster, not only more accurate. *What it does for you:* +"answer directly instead of opening five files" becomes a learned habit. + +### `--backend handoff` — session-executed calls (no API subprocess) + +For subscription seats and environments where the engine shouldn't spawn +`claude -p` / API calls itself. The engine still runs every deterministic +stage (harvest → mine → replay scoring → gate → stage), but each model call +(attempt / judge / reflect) is written to a prompt file that **your own agent +session answers between engine runs**: + +```bash +python -m skillopt_sleep run --backend handoff --project "$(pwd)" +# exit 3 => .skillopt-sleep-handoff/PROMPTS.md + pending.json were written +# answer each prompt (each in a FRESH context) into answers/.md +# re-run the same command => it resumes from the answers and either +# finishes (exit 0) or stages the next prompt batch (exit 3) +``` + +A typical night converges in 3–6 rounds: baseline attempts → reflect → +candidate re-scoring per accepted edit. Resume is stateless — replay is +deterministic and answers are cached by prompt hash, so re-running skips +everything already answered. Mined tasks are pinned to +`.skillopt-sleep-handoff/tasks.json` on the first round, so the sessions that +answer the prompts can't shift the task set and invalidate earlier answers. +On a completed real run the handoff directory is archived to +`.skillopt-sleep-handoff.night.done`. + +On Claude Code, `/skillopt-sleep-handoff run` drives the whole loop for you, +answering each prompt in an isolated fresh-context subagent. + +**Integrity rule:** answer every prompt in a fresh context (a subagent with no +conversation history). Answering from a session that has already seen the +mined tasks and their references contaminates the held-out gate and fakes the +improvement score. + +*What it does for you:* the sleep cycle runs entirely on your interactive +session's subscription budget — no API key, no headless subprocess — while the +gate, splits, and staging discipline stay in the engine. + +Limitations: `--rollouts-k > 1` gives no contrastive spread (identical prompt +→ identical answer file), and tool-loop tasks fall back to the single-shot +`TOOL_CALL:` marker convention. + +### `schedule` / `unschedule` — set it and forget it + +Built-in nightly scheduling (no manual cron): + +```bash +/skillopt-sleep schedule --hour 3 --minute 17 # runs every night for this project +/skillopt-sleep unschedule # stop it +``` +*What it does for you:* it just gets better while you sleep. The nightly run only +*stages* a proposal — adopting is still your call (or add `--auto-adopt` when you +schedule, if you trust it). + +--- + +## Full action / flag reference + +| Action | Does | +|---|---| +| `status` | nights so far + the latest staged proposal (read-only) | +| `dry-run` | harvest→mine→replay→report; **stages nothing** | +| `run` | full cycle; **stages** a proposal; nothing live changes | +| `adopt` | apply the staged proposal to `CLAUDE.md`/`SKILL.md` (backs up first) | +| `harvest` | debug: print the recurring tasks it mined | +| `schedule` / `unschedule` | install/remove the nightly cron entry | + +| Flag | Default | Meaning | +|---|---|---| +| `--backend mock\|claude\|codex\|copilot\|handoff` | `mock` | who runs/optimizes (mock = free; handoff = your own session answers) | +| `--preferences "..."` | – | your house rules, as a prior | +| `--gate on\|off` | `on` | strict held-out gate vs. greedy | +| `--rollouts-k K` | `1` | multi-rollout contrastive reflection | +| `--optimizer-model` / `--target-model` | – | split the optimizer from the target | +| `--budget-tokens` / `--budget-minutes` | – | cap the nightly spend | +| `--scope invoked\|all` | `invoked` | this project only, or all projects | +| `--auto-adopt` | off | apply without manual review (power users) | + +Deep dive: [the SkillOpt-Sleep guide section](https://microsoft.github.io/SkillOpt/docs/guideline.html#sleep). + +--- + +## Does it actually work? + +Yes — measured with **real models on both Claude and Codex**, scored on held-out +tasks the optimizer never trained on: + +- **gbrain-evals `skillopt-v1`** (the public suite gbrain scores SkillOpt on): + deficient skills go **0.00 → 1.00** on all 4 seeds, including a real tool-use + loop; cross-model transfer is positive; the gate blocks regressions. + → [the SkillOpt-Sleep guide section](https://microsoft.github.io/SkillOpt/docs/guideline.html#sleep) +- **Academic daily-cases** (math / spreadsheet / search-QA, the paper's 4:1:5 + split with dream-augmented train): see + [the SkillOpt-Sleep guide section](https://microsoft.github.io/SkillOpt/docs/guideline.html#sleep). +- **Fresh load-test** (a "SQL must always include LIMIT" analyst, built from + scratch): held-out **0.00 → 1.00** on both backends. + → [the SkillOpt-Sleep guide section](https://microsoft.github.io/SkillOpt/docs/guideline.html#sleep) + +Try the deterministic proof yourself (no API key, no spend): +```bash +python -m skillopt_sleep.experiments.run_experiment --persona researcher --assert-improves +``` +It prints the held-out score rising to 1.0 as the gate accepts the right rules, +and confirms the gate **rejects** an injected harmful edit. + +--- + +## Safety + +- **Read-only** harvest of your sessions. `mock` replay has no side effects. +- Proposals are **staged**, never auto-applied (unless you opt in with `--auto-adopt`). +- Every adopt writes a backup. Per-night token/time budget caps. Secrets redacted. diff --git a/plugins/claude-code/.claude-plugin/marketplace.json b/plugins/claude-code/.claude-plugin/marketplace.json new file mode 100644 index 0000000..2265541 --- /dev/null +++ b/plugins/claude-code/.claude-plugin/marketplace.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://anthropic.com/claude-code/marketplace.schema.json", + "name": "skillopt-sleep", + "description": "SkillOpt-Sleep: give your local Claude agent a nightly sleep cycle that reviews past sessions and consolidates validated memory + skills.", + "owner": { + "name": "Yifan Yang", + "email": "yifanyang@microsoft.com" + }, + "plugins": [ + { + "name": "skillopt-sleep", + "description": "Nightly offline self-evolution: harvest your past Claude Code sessions, replay recurring tasks on your own API budget, and consolidate what the agent learns into validated CLAUDE.md memory and SKILL.md skills, behind a held-out gate, staged for your review. Synthesizes SkillOpt (validation-gated skill optimization), Claude Dreams (offline memory consolidation), and agent sleep/consolidation.", + "author": { + "name": "Yifan Yang" + }, + "category": "productivity", + "source": { + "source": "git-subdir", + "url": "https://github.com/microsoft/SkillOpt.git", + "path": "plugins/claude-code", + "ref": "main" + }, + "homepage": "https://github.com/microsoft/SkillOpt" + } + ] +} diff --git a/plugins/claude-code/.claude-plugin/plugin.json b/plugins/claude-code/.claude-plugin/plugin.json new file mode 100644 index 0000000..d7bee08 --- /dev/null +++ b/plugins/claude-code/.claude-plugin/plugin.json @@ -0,0 +1,22 @@ +{ + "name": "skillopt-sleep", + "description": "Give your local Claude agent a nightly 'sleep cycle': it reviews your past sessions offline, replays recurring tasks on your own API budget, and consolidates what it learns into validated memory (CLAUDE.md) and skills (SKILL.md) so it gets better the more you use it. Synthesizes SkillOpt (validation-gated skill optimization), Claude Dreams (offline memory consolidation), and agent sleep/consolidation.", + "version": "0.1.0", + "author": { + "name": "Yifan Yang", + "email": "yifanyang@microsoft.com" + }, + "homepage": "https://github.com/microsoft/SkillOpt", + "repository": "https://github.com/microsoft/SkillOpt", + "license": "MIT", + "keywords": [ + "skillopt", + "self-improvement", + "memory-consolidation", + "dreams", + "sleep", + "skills", + "continual-learning", + "offline-optimization" + ] +} diff --git a/plugins/claude-code/README.md b/plugins/claude-code/README.md new file mode 100644 index 0000000..8f9f53b --- /dev/null +++ b/plugins/claude-code/README.md @@ -0,0 +1,161 @@ +# SkillOpt-Sleep (Claude Code plugin) + +> Give your local Claude agent a **sleep cycle**. Every night it reviews your +> past sessions offline, replays your recurring tasks on your own API budget, +> and consolidates what it learns into **validated** memory (`CLAUDE.md`) and +> skills (`SKILL.md`). Your agent gets better the more you use it — no +> model-weight training. + +SkillOpt-Sleep is the **deployment-time** companion to +[SkillOpt](https://github.com/microsoft/SkillOpt). SkillOpt trains a skill +offline on a benchmark; SkillOpt-Sleep applies the same discipline to *your own +daily usage*: bounded text edits, accepted only through a held-out validation +gate, with rejected edits kept as negative feedback. + +It synthesizes three ideas: + +| Idea | Contribution | +|---|---| +| **SkillOpt** | skill/memory = trainable text; bounded add/delete/replace edits; **held-out gate** keeps only changes that help. | +| **Claude Dreams** | offline consolidation over past sessions; input never mutated; output **reviewed then adopted**. | +| **Agent sleep** | periodic offline replay turns short-term episodes into long-term skill. | + +## What it does (one "night") + +``` +harvest ~/.claude transcripts → mine recurring tasks → replay offline + → consolidate (reflect → bounded edit → GATE) → stage proposal → (you) adopt +``` + +Nothing live is modified until **you** run `/skillopt-sleep adopt` (the Dreams "review, +then adopt or discard" contract). Every adopt backs up the prior file first. + +## Install + +**Requirements:** Python ≥ 3.10, and the `claude` CLI (and/or `codex` CLI) on PATH. + +```bash +# 1) get the code (the plugin ships inside the SkillOpt repo) +git clone https://github.com/microsoft/SkillOpt.git +cd SkillOpt + +# 2) add the plugin to Claude Code as a local marketplace +/plugin marketplace add ./skillopt-sleep-plugin +/plugin install skillopt-sleep@skillopt-sleep + +# 3) verify +/skillopt-sleep status +``` + +The plugin's bundled runner (`scripts/sleep.sh`) auto-selects a Python ≥ 3.10 +interpreter and calls the `skillopt_sleep` engine in the repo. No `pip install` +is required for the default `mock` backend or for `claude`/`codex` backends — +they shell out to the CLIs you already have. + +## Quick start + +```bash +# from inside any project you use with Claude Code: +/skillopt-sleep dry-run # safe preview: what it would learn, no changes staged +/skillopt-sleep run # full cycle: stages a reviewed proposal (still no live edits) +/skillopt-sleep status # see history + the latest staged proposal +/skillopt-sleep adopt # apply the staged proposal to CLAUDE.md / SKILL.md (with backup) + +/skillopt-sleep-handoff run # same cycle, but THIS session answers the model calls + # (no claude -p subprocess, no API key — subscription-friendly) +``` + +Or call the engine directly (Python ≥ 3.10): + +```bash +python -m skillopt_sleep run --project "$(pwd)" --scope invoked --backend mock +python -m skillopt_sleep run --project "$(pwd)" --backend claude # real lift via Claude +python -m skillopt_sleep run --project "$(pwd)" --backend codex # real lift via Codex +``` + +Default backend is **`mock`** — deterministic, no API spend — so you can try the +plumbing for free. Switch to `--backend claude` or `--backend codex` for genuine +improvement on your own budget. + +### Handoff mode (session answers the model calls) + +`--backend handoff` runs the cycle without any model subprocess: the engine +executes the deterministic stages and writes every model call it needs to +`.skillopt-sleep-handoff/PROMPTS.md` + `pending.json` (exit code 3). You (or +the `/skillopt-sleep-handoff` command, which automates the loop with isolated +fresh-context subagents) write each raw answer to `answers/.md` and re-run +the same command; it resumes from the answers and either finishes or stages +the next batch. Typically 3–6 rounds per night. + +```bash +python -m skillopt_sleep run --backend handoff --project "$(pwd)" +# ... answer .skillopt-sleep-handoff/PROMPTS.md into answers/.md ... +python -m skillopt_sleep run --backend handoff --project "$(pwd)" # resume +``` + +Answer every prompt in a **fresh context** — a session that has already seen +the mined tasks and their references would contaminate the held-out gate. +Details: [the plugins README](../README.md#--backend-handoff--session-executed-calls-no-api-subprocess). + +## Does it actually improve? (real models, public benchmark) + +SkillOpt-Sleep is validated against [gbrain-evals](https://github.com/garrytan/gbrain-evals)' +public `skillopt-v1` suite — the same benchmark gbrain scores its own skill +optimizer against. We take a deliberately **deficient** skill and run one sleep +night; held-out scoring is done by a local rule judge (no judge-API, no way to +grade its own homework). + +| Backend | Seed | Held-out before → after | Nights | +|---|---|---|---| +| **Claude (Haiku 4.5)** | brief-writer | **0.00 → 1.00** | 1 | +| **Codex** | brief-writer | **0.00 → 1.00** | 2 | + +Both took a brief-writer with no risks section / no confidence level and, within +1–2 nights, proposed gated edits that lifted the held-out score to perfect — +into the protected `LEARNED` block, nothing else touched. The Codex 2-night +trace even shows the optimizer **diagnosing its own residual failure** and +adding a meta-rule to fix it. Full writeup + reproduction: +[the SkillOpt-Sleep guide section](https://microsoft.github.io/SkillOpt/docs/guideline.html#sleep). + +Reproduce: + +```bash +git clone https://github.com/garrytan/gbrain-evals /tmp/gbrain-evals +python -m skillopt_sleep.experiments.run_gbrain --backend claude --model haiku \ + --seeds brief-writer --data-root /tmp/gbrain-evals/eval/data/skillopt-v1 \ + --nights 1 --limit-replay 3 --limit-holdout 3 +python -m skillopt_sleep.experiments.run_gbrain --backend codex \ + --seeds brief-writer --data-root /tmp/gbrain-evals/eval/data/skillopt-v1 \ + --nights 1 --limit-replay 3 --limit-holdout 3 +``` + +## Deterministic proof (no API, no keys) + +```bash +python -m skillopt_sleep.experiments.run_experiment --persona researcher --assert-improves +python -m skillopt_sleep.experiments.run_experiment --persona programmer --assert-improves +``` + +Each prints the held-out score rising from baseline toward 1.0 as the gate +accepts the general rules your tasks need, and confirms the gate **rejects** an +injected harmful edit. Recorded output: [the SkillOpt-Sleep guide section](https://microsoft.github.io/SkillOpt/docs/guideline.html#sleep). + +## Schedule it nightly + +```bash +"${CLAUDE_PLUGIN_ROOT}/scripts/install-cron.sh" "$(pwd)" # prints a crontab line; installs nothing +``` + +## Safety + +- **Read-only** harvest of `~/.claude`. `mock` replay has no side effects. +- Proposals are **staged**, never auto-applied (unless you opt in with `--auto-adopt`). +- Every adopt writes a backup under the staging dir's `backup/`. +- Per-night **token/task budget caps**; secrets redacted from prompts. +- `fresh` replay (Phase 3) runs only in throwaway git worktrees. + +## Status + +Phase 1 (engine + deterministic experiment + plugin surface) is complete. +Phase 3 adds the real-API miner/judge and `fresh` worktree replay. See +[`docs/superpowers/specs/2026-06-07-skillopt-sleep-claude-code-plugin-design.md`](../docs/superpowers/specs/2026-06-07-skillopt-sleep-claude-code-plugin-design.md). diff --git a/plugins/claude-code/commands/skillopt-sleep-handoff.md b/plugins/claude-code/commands/skillopt-sleep-handoff.md new file mode 100644 index 0000000..fffad76 --- /dev/null +++ b/plugins/claude-code/commands/skillopt-sleep-handoff.md @@ -0,0 +1,67 @@ +--- +description: Run the SkillOpt-Sleep cycle with the handoff backend — no API subprocess; this session answers the engine's model calls via prompt/answer files, in isolated fresh-context subagents +argument-hint: "[run | dry-run] [--preferences \"...\"] (default: run)" +allowed-tools: Bash, Read, Write, Task +--- + +# /skillopt-sleep-handoff — session-executed sleep cycle + +You are driving **SkillOpt-Sleep in handoff mode**: the Python engine runs +every deterministic stage (harvest → mine → replay scoring → gate → stage) +and outsources each model call (attempt / judge / reflect) to YOU via +prompt files. No `claude -p` subprocess, no API key — the model work runs +on this session's budget, but each prompt MUST be answered in a fresh, +isolated context so the validation gate stays honest. + +## Requested action: $ARGUMENTS + +(If `$ARGUMENTS` is empty, treat it as `run`.) + +## The loop + +Repeat until the engine exits 0 (done) — at most 8 rounds: + +1. **Run the engine** via the bundled runner: + + ```bash + "${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" --backend handoff --project "$(pwd)" --scope invoked + ``` + + - exit 0 → the night is complete; go to "Finish" below. + - exit 3 → pending model calls; continue with step 2. + - anything else → stop and show the user the error output. + +2. **Read the batch**: `Read` `.skillopt-sleep-handoff/pending.json` in the + project. Each entry has `id`, `prompt`, `max_tokens`, `answer_file`. + +3. **Answer each prompt in ISOLATION** — this is the integrity rule: + - For each entry, launch a subagent (Task tool) whose ENTIRE input is + the `prompt` text verbatim. Add nothing: no summary of this session, + no mention of SkillOpt, no other prompts from the batch. + - Take the subagent's reply and `Write` the raw answer text (no + commentary, no code fences) to the entry's `answer_file`. + - NEVER answer from this session's own context — you have seen the + mined tasks and their references, so inline answers would contaminate + the held-out gate and fake the improvement score. + +4. **Re-run the same engine command** — it resumes from the answers + directory and either finishes or stages the next batch. + +## Finish + +- `Read` the `report.md` in the staging dir the engine printed and show + the user: held-out baseline → candidate score, the gate decision, the + proposed edits, and where the proposal is staged. +- Tell the user nothing live changed; offer `/skillopt-sleep adopt`. +- The engine archives `.skillopt-sleep-handoff/` on a completed real run; + do not delete it yourself. + +## Safety reminders + +- **Never** edit `CLAUDE.md` or `SKILL.md` yourself — only `adopt` does + that, with a backup. +- Mined tasks are pinned to `.skillopt-sleep-handoff/tasks.json` on round + one, so sessions created while answering prompts cannot shift the task + set. Do not edit that file. +- If a batch looks like it contains secrets or content the user would not + want re-processed, stop and ask before answering. diff --git a/plugins/claude-code/commands/skillopt-sleep.md b/plugins/claude-code/commands/skillopt-sleep.md new file mode 100644 index 0000000..7fca8ae --- /dev/null +++ b/plugins/claude-code/commands/skillopt-sleep.md @@ -0,0 +1,66 @@ +--- +description: Run or manage the SkillOpt-Sleep self-evolution cycle (review past sessions, replay tasks offline, consolidate validated memory + skills; can also schedule nightly runs) +argument-hint: "[run | dry-run | status | adopt | harvest | schedule | unschedule] (default: status)" +allowed-tools: Bash, Read +--- + +# /skillopt-sleep — SkillOpt-Sleep nightly self-evolution + +You are driving **SkillOpt-Sleep**: a tool that lets this user's Claude agent +improve offline by reviewing past sessions, replaying recurring tasks, and +consolidating what it learns into **validated** memory (`CLAUDE.md`) and skills +(`SKILL.md`). It is gated like SkillOpt: a change is kept only if it improves a +held-out replay score, and nothing live is modified until the user adopts it. + +## Requested action: $ARGUMENTS + +(If `$ARGUMENTS` is empty, treat it as `status`.) + +## How to run it + +The engine is the `skillopt_sleep` Python package in this repo. Use the +**plugin's bundled runner** so the right interpreter and repo are on the path: + +```bash +"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" --project "$(pwd)" --scope invoked +``` + +`` is one of: + +| action | what it does | +|--------------|--------------| +| `status` | show how many nights have run + the latest staged proposal (READ-ONLY) | +| `dry-run` | harvest → mine → replay → report, but **stage nothing** (safe preview) | +| `run` | full cycle: also **stage** a reviewed proposal (still does NOT touch live files) | +| `adopt` | apply the latest staged proposal to live `CLAUDE.md` / `SKILL.md` (backs up first) | +| `harvest` | debug: print the recurring tasks mined from recent sessions | +| `schedule` | install a nightly cron entry for this project (`--hour --minute`, off-:00 by default) | +| `unschedule` | remove the nightly cron entry (`--all` to remove every managed entry) | + +Default backend is `mock` (deterministic, no API spend). To use real budget for +genuine improvement, add `--backend claude` or `--backend codex`. To steer what +the optimizer writes, add `--preferences ""`. + +## Steps to follow + +1. **Run the requested action** via the bundled runner above. Capture stdout. +2. **For `run` / `dry-run`:** after it completes, `Read` the generated + `report.md` in the staging dir it prints, and show the user: + - held-out score: baseline → candidate (the proof it helped) + - the gate decision (accept/reject) and the exact edits it proposes + - where the proposal is staged +3. **For `run` that produced an accepted proposal:** tell the user the diff is + staged and that **nothing live changed yet**. Offer to run `/skillopt-sleep adopt`. +4. **For `adopt`:** confirm which live files were updated and that backups were + written under the staging dir's `backup/`. +5. **Never** edit `CLAUDE.md` or `SKILL.md` yourself — only the `adopt` action + does that, with a backup. Respect the review gate. + +## Safety reminders + +- Harvest is **read-only** over `~/.claude`. Replay in `mock` mode runs no + shell side effects. +- The cycle stages proposals; the user is in control of adoption. +- If the user asks to schedule this nightly, point them at + `${CLAUDE_PLUGIN_ROOT}/scripts/install-cron.sh` (prints a crontab line; does + not install anything without confirmation). diff --git a/plugins/claude-code/hooks/hooks.json b/plugins/claude-code/hooks/hooks.json new file mode 100644 index 0000000..6ea666b --- /dev/null +++ b/plugins/claude-code/hooks/hooks.json @@ -0,0 +1,16 @@ +{ + "hooks": { + "SessionEnd": [ + { + "matcher": "*", + "hooks": [ + { + "type": "command", + "command": "\"${CLAUDE_PLUGIN_ROOT}/hooks/on-session-end.sh\"", + "async": true + } + ] + } + ] + } +} diff --git a/plugins/claude-code/hooks/on-session-end.sh b/plugins/claude-code/hooks/on-session-end.sh new file mode 100755 index 0000000..bd84be2 --- /dev/null +++ b/plugins/claude-code/hooks/on-session-end.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# SkillOpt-Sleep SessionEnd hook (async, best-effort, NON-BLOCKING). +# +# This does NOT run the optimizer. It only appends a tiny marker so the next +# nightly cycle knows there is fresh activity to harvest, and (optionally) +# nudges the user once that a sleep cycle is available. It must never fail the +# session or spend API budget. +set -uo pipefail + +PLUGIN_ROOT="${CLAUDE_PLUGIN_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +STATE_DIR="${HOME}/.skillopt-sleep" +mkdir -p "$STATE_DIR" 2>/dev/null || exit 0 + +# Record that a session just ended (cheap; used for "is there new data?"). +printf '%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${PWD}" \ + >> "$STATE_DIR/session-end.log" 2>/dev/null || true + +exit 0 diff --git a/plugins/claude-code/scripts/install-cron.sh b/plugins/claude-code/scripts/install-cron.sh new file mode 100755 index 0000000..5726acc --- /dev/null +++ b/plugins/claude-code/scripts/install-cron.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Print (does NOT install) a crontab line that runs SkillOpt-Sleep nightly. +# The user copies the line into `crontab -e` if they want it. +set -euo pipefail + +PLUGIN_ROOT="${CLAUDE_PLUGIN_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +RUNNER="$PLUGIN_ROOT/scripts/sleep.sh" +PROJECT="${1:-$(pwd)}" +BACKEND="${2:-mock}" + +# 3:17am local — deliberately off the :00 mark so many users don't all hit the +# API at once (and we leave room for jitter). +MIN=17 +HOUR=3 + +cat <> "${PROJECT}/.skillopt-sleep/cron.log" 2>&1 +# +# For fully-autonomous adoption (power users), append: --auto-adopt +# To spend real API budget for genuine lift, set BACKEND=anthropic above. +# ──────────────────────────────────────────────────────────────────────────── +EOF diff --git a/plugins/claude-code/scripts/run-sleep.sh b/plugins/claude-code/scripts/run-sleep.sh new file mode 100755 index 0000000..310d8de --- /dev/null +++ b/plugins/claude-code/scripts/run-sleep.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# SkillOpt-Sleep shared runner — used by all platform plugins (Claude Code, +# Codex, Copilot). Resolves the repo root (which contains the skillopt_sleep +# package), picks a Python >= 3.10, and execs the engine CLI. +# +# Usage: run-sleep.sh [args...] +set -euo pipefail + +# This script lives at /plugins/run-sleep.sh, so the repo root (which +# holds skillopt_sleep/) is one level up. CLAUDE_PLUGIN_ROOT (if set by Claude +# Code) points at the plugin dir; the engine is then two levels above it. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [ -d "$SCRIPT_DIR/../skillopt_sleep" ]; then + REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +elif [ -n "${CLAUDE_PLUGIN_ROOT:-}" ] && [ -d "$CLAUDE_PLUGIN_ROOT/../../skillopt_sleep" ]; then + REPO_ROOT="$(cd "$CLAUDE_PLUGIN_ROOT/../.." && pwd)" +elif [ -n "${SKILLOPT_SLEEP_REPO:-}" ] && [ -d "$SKILLOPT_SLEEP_REPO/skillopt_sleep" ]; then + REPO_ROOT="$SKILLOPT_SLEEP_REPO" +else + # last resort: search upward from CWD + d="$PWD" + while [ "$d" != "/" ]; do + [ -d "$d/skillopt_sleep" ] && { REPO_ROOT="$d"; break; } + d="$(dirname "$d")" + done +fi +if [ -z "${REPO_ROOT:-}" ]; then + echo "[sleep] ERROR: could not locate the skillopt_sleep package. Set SKILLOPT_SLEEP_REPO to the repo root." >&2 + exit 1 +fi + +PY="" +# Allow explicit Python override (useful on macOS with old system Python). +if [ -n "${SKILLOPT_SLEEP_PYTHON:-}" ]; then + PY="$SKILLOPT_SLEEP_PYTHON" +else + for cand in python3.12 python3.11 python3.10 python3; do + if command -v "$cand" >/dev/null 2>&1; then + ver="$("$cand" -c 'import sys; print("%d%d" % sys.version_info[:2])' 2>/dev/null || echo 0)" + if [ "${ver:-0}" -ge 310 ]; then PY="$cand"; break; fi + fi + done +fi +if [ -z "$PY" ]; then + echo "[sleep] ERROR: need Python >= 3.10 (found none)." >&2 + exit 1 +fi + +if [ "$#" -eq 0 ]; then set -- status; fi +cd "$REPO_ROOT" +exec "$PY" -m skillopt_sleep "$@" diff --git a/plugins/claude-code/scripts/sleep.sh b/plugins/claude-code/scripts/sleep.sh new file mode 100755 index 0000000..20a9f36 --- /dev/null +++ b/plugins/claude-code/scripts/sleep.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Claude Code plugin runner — thin wrapper over the shared runner so all +# platform plugins share one engine launcher. +# +# After marketplace install the plugin is isolated in a cache directory and +# the repo-relative path no longer works. We try four locations: +# 1. Co-located run-sleep.sh (bundled copy — works in marketplace cache) +# 2. Repo-relative ../../run-sleep.sh (dev checkout) +# 3. CLAUDE_PLUGIN_ROOT/../run-sleep.sh (plugin env variable) +# 4. SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh (explicit env) +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +SHARED="" +if [ -f "$HERE/run-sleep.sh" ]; then + SHARED="$HERE/run-sleep.sh" +elif [ -f "$(cd "$HERE/../.." 2>/dev/null && pwd)/run-sleep.sh" ]; then + SHARED="$(cd "$HERE/../.." && pwd)/run-sleep.sh" +elif [ -n "${CLAUDE_PLUGIN_ROOT:-}" ] && [ -f "$(cd "$CLAUDE_PLUGIN_ROOT/.." 2>/dev/null && pwd)/run-sleep.sh" ]; then + SHARED="$(cd "$CLAUDE_PLUGIN_ROOT/.." && pwd)/run-sleep.sh" +elif [ -n "${SKILLOPT_SLEEP_REPO:-}" ] && [ -f "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" ]; then + SHARED="$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" +fi + +if [ -z "$SHARED" ]; then + echo "[sleep] ERROR: cannot locate run-sleep.sh." >&2 + echo "[sleep] Set SKILLOPT_SLEEP_REPO to the SkillOpt repo root, or pip install skillopt." >&2 + exit 1 +fi +exec bash "$SHARED" "$@" diff --git a/plugins/claude-code/skills/skillopt-sleep/SKILL.md b/plugins/claude-code/skills/skillopt-sleep/SKILL.md new file mode 100644 index 0000000..b7f4019 --- /dev/null +++ b/plugins/claude-code/skills/skillopt-sleep/SKILL.md @@ -0,0 +1,126 @@ +--- +name: skillopt-sleep +description: "Use when the user wants their Claude agent to self-improve from past usage, asks about a nightly/offline 'sleep' or 'dream' cycle, memory/skill consolidation, or says things like 'make my agent better the more I use it', 'review my past sessions', 'learn my preferences', 'consolidate what you learned', 'run the sleep cycle', or wants to schedule offline self-optimization. Drives the skillopt_sleep engine: harvest past sessions -> mine recurring tasks -> replay offline -> consolidate validated CLAUDE.md/SKILL.md behind a held-out gate." +--- + +# SkillOpt-Sleep: offline self-evolution for a local Claude agent + +SkillOpt-Sleep gives the user's agent a **sleep cycle**. While the user is +offline (e.g. nightly), it reviews their real past Claude Code sessions, +re-runs recurring tasks on their own API budget, and consolidates what it +learns into **memory** (`CLAUDE.md`) and **skills** (`SKILL.md`) — but only +keeps changes that pass a held-out validation gate, and only after the user +adopts them. The agent gets measurably better at *this* user's recurring work, +with no model-weight training. It is the deployment-time analogue of training: +short-term experience → long-term competence. + +It synthesizes three ideas: +- **SkillOpt** — the skill/memory doc is trainable text; bounded add/delete/replace + edits; accepted only through a held-out gate; rejected edits become negative feedback. +- **Claude Dreams** — offline consolidation that reads past sessions and rebuilds + memory (dedup/merge/resolve); the input is never mutated; output is reviewed then adopted. +- **Agent sleep** — periodic offline replay turns episodes into durable skill. + +## When to use this skill + +Trigger when the user wants any of: +- "make my agent learn from how I use it" / "get better the more I use it" / "remember my preferences across sessions" +- a nightly/scheduled or on-demand **offline self-improvement / dream / sleep** run +- to **review past sessions/trajectories** and distill recurring tasks +- to **consolidate** feedback into `CLAUDE.md` or a managed skill +- to **schedule** the cycle (cron) or **adopt** a staged proposal + +## The cycle (six stages) + +1. **Harvest** — read `~/.claude/projects/*/.jsonl` + `~/.claude/history.jsonl` (READ-ONLY) → session digests. +2. **Mine** — digests → `TaskRecord`s (recurring intents + outcome labels + checkable refs where possible). +3. **Replay** — re-run tasks offline under the *current* skill+memory → (hard, soft) scores. +4. **Consolidate** — reflect on failures → propose bounded edits → **gate** on a held-out slice; accept only if it strictly improves. +5. **Stage** — write `proposed_CLAUDE.md`, `proposed_SKILL.md`, a diff, and `report.md` into `/.skillopt-sleep/staging//`. **Nothing live changes.** +6. **Adopt** — explicit (or opt-in auto): copy staged files over live ones, backing up first. + +## How to drive it + +Prefer the `/skillopt-sleep` command. Under the hood it calls the bundled runner: + +```bash +"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" status # what's happened +"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" dry-run --project "$(pwd)" # safe preview +"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" run --project "$(pwd)" # full cycle, stages a proposal +"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" adopt --project "$(pwd)" # apply staged proposal (with backup) +``` + +- Default backend is `mock` (deterministic, **no API spend**) — good for trying the plumbing. +- Add `--backend claude` or `--backend codex` to spend the user's real budget for genuine improvement. +- Scope defaults to the invoked project; `--scope all` harvests every project. + +### Scheduling + +```bash +"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" schedule --project "$(pwd)" --hour 3 --minute 17 +"${CLAUDE_PLUGIN_ROOT}/scripts/sleep.sh" unschedule --project "$(pwd)" +``` + +Installs a nightly cron entry. `unschedule --all` removes every managed entry. + +## All CLI flags + +| Flag | Default | Description | +|------|---------|-------------| +| `--project PATH` | cwd | Project directory to evolve | +| `--scope all\|invoked` | invoked | Harvest scope | +| `--backend mock\|claude\|codex\|copilot` | mock | Replay backend (mock = no API spend) | +| `--model NAME` | backend default | Override the model used for replay | +| `--source claude\|codex\|auto` | claude | Transcript source | +| `--lookback-hours N` | 72 | Harvest window | +| `--max-sessions N` | unlimited | Cap harvested sessions | +| `--max-tasks N` | 40 | Cap mined tasks | +| `--target-skill-path PATH` | auto | Explicit SKILL.md to evolve | +| `--tasks-file PATH` | — | Reviewed TaskRecord JSON (skip harvest) | +| `--progress` | off | Print phase progress to stderr | +| `--auto-adopt` | off | Auto-adopt if gate passes | +| `--edit-budget N` | 4 | Max bounded edits per night | +| `--json` | off | Machine-readable JSON output | + +## Config keys (`~/.skillopt-sleep/config.json`) + +Beyond the CLI flags, advanced behavior is controlled via config: + +- **`preferences`** — free-text house rules injected into the optimizer's reflect step (e.g. "Always use async/await", "Answers in `\boxed{}`"). +- **`gate_mode`** — `on` (default, validation-gated) or `off` (greedy, accept all edits). +- **`gate_metric`** — `hard`, `soft`, or `mixed` (default). Controls how the held-out gate scores. +- **`dream_rollouts`** — >1 enables multi-rollout contrastive reflection per task. +- **`recall_k`** — >0 recalls K similar past tasks into the dream (long-term memory). +- **`evolve_memory`** / **`evolve_skill`** — independently toggle CLAUDE.md vs SKILL.md consolidation. + +## Memory consolidation + +The sleep cycle can consolidate both: +- **SKILL.md** — the managed skill file (bounded edits: add/delete/replace) +- **CLAUDE.md** — the project memory (same bounded edits) + +Both are gated by the same held-out validation score. Set `evolve_memory: false` to consolidate only skills, or `evolve_skill: false` for only memory. + +## Hard rules + +- **Never** hand-edit the user's `CLAUDE.md` / `SKILL.md` as part of this skill. + Only the `adopt` action changes live files, and it backs them up first. +- Harvest is read-only. `mock` replay has no side effects. +- Always show the user the **held-out baseline → candidate** score and the + exact proposed edits before suggesting adoption. Evidence before adoption. +- If asked whether it really helps, run + `python -m skillopt_sleep.experiments.run_experiment --persona researcher --json` + — a deterministic demo that proves held-out lift and that the gate blocks + harmful edits. + +## Validate / demo + +```bash +# deterministic proof (no API): held-out score rises, gate blocks regressions +python -m skillopt_sleep.experiments.run_experiment --persona researcher --assert-improves +python -m skillopt_sleep.experiments.run_experiment --persona programmer --assert-improves +``` + +See the SkillOpt-Sleep guide section for recorded output and +`docs/superpowers/specs/2026-06-07-skillopt-sleep-claude-code-plugin-design.md` +for the full design. diff --git a/plugins/codex/README.md b/plugins/codex/README.md new file mode 100644 index 0000000..c545514 --- /dev/null +++ b/plugins/codex/README.md @@ -0,0 +1,96 @@ +# SkillOpt-Sleep — Codex integration + +Give your **Codex** agent a nightly **sleep cycle**: it reviews past sessions +offline, replays your recurring tasks on your own Codex budget, and consolidates +what it learns into validated memory + skills behind a held-out gate. Same engine +as the Claude Code plugin (`skillopt_sleep`), wrapped for Codex. + +> **Verified on Codex:** on the public +> [gbrain-evals](https://github.com/garrytan/gbrain-evals) `skillopt-v1` +> benchmark, a deliberately deficient skill goes **0.00 → 1.00** on a held-out +> set with the Codex backend (incl. the tool-use seed via a real tool loop). +> See [the SkillOpt-Sleep guide section](https://microsoft.github.io/SkillOpt/docs/guideline.html#sleep). + +## What Codex supports (and what we use) + +Codex (`@openai/codex`) extends via **`AGENTS.md`** instructions, **skills** at +`~/.agents/skills//SKILL.md`, and plugins that can distribute skills. +Custom prompts are deprecated in Codex, so this integration is skill-first: the +installed `skillopt-sleep` skill contains the launch commands and operating +rules. The shared runner remains a plain shell entrypoint that the skill calls. + +## Install + +```bash +git clone SkillOpt-Sleep +cd SkillOpt-Sleep +bash plugins/codex/install.sh # installs the skill +export SKILLOPT_SLEEP_REPO="$(pwd)" # so the runner is found from anywhere +``` + +If a previous install created `~/.codex/prompts/sleep.md`, the installer moves +that deprecated prompt aside with a `.skillopt-legacy*.bak` suffix. + +Requires Python ≥ 3.10 and the `codex` CLI on PATH. + +## Use + +Mention `$skillopt-sleep` where Codex supports explicit skill mentions, or ask +Codex in natural language: + +```text +Use the skillopt-sleep skill to run status for this project. +Use the skillopt-sleep skill to run a dry-run for this project. +Use the skillopt-sleep skill to run the full cycle for this project with the Codex backend. +Use the skillopt-sleep skill to adopt the latest staged proposal. +``` + +Or call the engine directly: + +```bash +python -m skillopt_sleep dry-run --project "$(pwd)" --source codex --backend mock +python -m skillopt_sleep run --project "$(pwd)" --source codex --backend codex \ + --max-sessions 5 --max-tasks 3 --progress +python -m skillopt_sleep run --project "$(pwd)" --source codex --backend codex \ + --target-skill-path .agents/skills/example/SKILL.md \ + --max-sessions 5 --max-tasks 3 --progress +``` + +`--source codex` reads Codex Desktop archived sessions from +`~/.codex/archived_sessions`. Use `--codex-home /path/to/.codex` to point at a +different Codex home, or `--source auto` to try Codex archives first and fall +back to Claude Code transcripts. Default backend is `mock` (no API spend). +`--backend codex` uses your Codex budget for real improvement. Bound live runs +with `--max-sessions` and `--max-tasks`; add `--progress` because Codex-backed +mining, replay, and reflection can be slow and otherwise quiet. Use +`--target-skill-path` to stage/adopt into a repo-scoped Codex skill such as +`.agents/skills//SKILL.md`; target runs over-sample mined tasks and +prefer tasks that match the target skill's path, headings, and content. All the +controllable knobs (`--gate on|off`, `--rollouts-k`, `--budget-tokens`, +`--preferences`, optimizer/target split) work identically — see +[the SkillOpt-Sleep guide section](https://microsoft.github.io/SkillOpt/docs/guideline.html#sleep). + +For privacy-sensitive projects, split the run into reviewable steps: + +```bash +python -m skillopt_sleep harvest --project "$(pwd)" --source codex \ + --target-skill-path .agents/skills/example/SKILL.md \ + --max-sessions 5 --max-tasks 3 \ + --output reviewed-tasks.json + +python -m skillopt_sleep dry-run --project "$(pwd)" --backend codex \ + --tasks-file reviewed-tasks.json --progress --json +``` + +Inspect/redact the JSON and set `"reviewed": true` before using a real backend. +`--tasks-file` skips archive harvest/mining and replays only the reviewed JSON +tasks; real backends refuse task files still marked `"reviewed": false`. + +## Notes / status + +- Codex's `exec` runs shell, so the real-tool-loop replay (e.g. the + `tool_called: search` benchmark seed) works natively. +- This integration no longer installs a `.codex/prompts` slash command. Skills + are the reusable Codex workflow surface; mention `skillopt-sleep` explicitly + or ask for a sleep/dream/offline self-improvement run and Codex can load the + skill. diff --git a/plugins/codex/install.sh b/plugins/codex/install.sh new file mode 100755 index 0000000..11b0735 --- /dev/null +++ b/plugins/codex/install.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Install the SkillOpt-Sleep Codex integration as a user-level Codex skill. +# Idempotent; prints what it does. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +CODEX_HOME="${CODEX_HOME:-$HOME/.codex}" +AGENTS_SKILLS="${HOME}/.agents/skills" +LEGACY_PROMPT="$CODEX_HOME/prompts/sleep.md" + +echo "[install] repo: $REPO_ROOT" + +# 1) user-level skill +mkdir -p "$AGENTS_SKILLS/skillopt-sleep" +cp "$REPO_ROOT/plugins/codex/skills/skillopt-sleep/SKILL.md" "$AGENTS_SKILLS/skillopt-sleep/SKILL.md" +echo "[install] skill -> $AGENTS_SKILLS/skillopt-sleep/SKILL.md" + +# 2) retire the old custom prompt entrypoint from previous installs +if [ -f "$LEGACY_PROMPT" ]; then + backup="${LEGACY_PROMPT}.skillopt-legacy.bak" + if [ -e "$backup" ]; then + backup="${LEGACY_PROMPT}.skillopt-legacy.$(date +%Y%m%d%H%M%S).bak" + fi + mv "$LEGACY_PROMPT" "$backup" + echo "[install] legacy prompt -> $backup" +fi + +# 3) record the repo location so the runner is found from anywhere +echo "[install] add to your shell profile:" +echo " export SKILLOPT_SLEEP_REPO=\"$REPO_ROOT\"" + +# 4) optional: append an AGENTS.md hint (only if the user opts in) +cat < mine recurring tasks -> replay offline -> consolidate validated memory + skills behind a held-out gate." +--- + +# SkillOpt-Sleep: offline self-evolution for a local Codex agent + +SkillOpt-Sleep gives the user's Codex agent a sleep cycle. While the user is +offline or on demand, it reviews past local sessions, re-runs recurring tasks +on the user's own budget, and consolidates what it learns into memory and +skills. It keeps only changes that pass a held-out validation gate, and live +files change only after the user explicitly adopts a staged proposal. There is +no model-weight training. + +## When to use + +Trigger when the user wants any of: + +- Codex to learn from past sessions or get better the more they use it; +- a nightly/scheduled or on-demand sleep/dream/offline self-improvement run; +- to review past sessions and distill recurring tasks; +- to consolidate feedback into memory or managed skills; +- to run `status`, `harvest`, `dry-run`, `run`, or `adopt` for SkillOpt-Sleep. + +## The cycle + +1. **Harvest** - read local session transcripts according to the engine + configuration and normalize them into session digests. +2. **Mine** - turn digests into recurring `TaskRecord`s with outcomes and + checkable references where possible. +3. **Replay** - re-run mined tasks offline under the current skill and memory. +4. **Consolidate** - reflect on failures and propose bounded edits. +5. **Gate** - accept edits only when the held-out validation score improves. +6. **Stage** - write the proposal under + `/.skillopt-sleep/staging//`; nothing live changes. +7. **Adopt** - only after explicit user approval, copy staged files over live + files with backups. + +## How to drive it + +Invoke the bundled runner via shell (Codex `exec` has shell access). The runner +finds the engine and a Python >= 3.10 automatically. + +```bash +# point at the repo if it isn't auto-detected from CWD: +export SKILLOPT_SLEEP_REPO=/path/to/SkillOpt-Sleep +bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" status --project "$(pwd)" +bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" harvest --project "$(pwd)" +bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" dry-run --project "$(pwd)" --backend mock +bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" run --project "$(pwd)" --backend codex +bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" run --project "$(pwd)" --source codex # harvest from Codex Desktop +bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" adopt --project "$(pwd)" +``` + +Actions are `status`, `harvest`, `dry-run`, `run`, `adopt`, `schedule`, and `unschedule`. + +- Default backend is `mock`, which is deterministic and spends no API budget. +- `--backend codex` uses the user's Codex budget for real improvement. +- `--source codex` reads Codex Desktop archived sessions from `~/.codex/archived_sessions`; + use `--codex-home /path/to/.codex` if the archive lives elsewhere. +- Keep `dry-run --backend mock` as the first smoke check unless the user + explicitly asked for a real optimization run. + +### Scheduling + +```bash +bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" schedule --project "$(pwd)" --hour 3 --minute 17 +bash "$SKILLOPT_SLEEP_REPO/plugins/run-sleep.sh" unschedule --project "$(pwd)" +``` + +Installs a nightly cron entry. `unschedule --all` removes every managed entry. + +### All backends + +- `--backend mock` — deterministic, no API spend (default) +- `--backend claude` — uses the Claude CLI +- `--backend codex` — uses the Codex CLI +- `--backend copilot` — uses the GitHub Copilot CLI + +### Additional flags + +| Flag | Description | +|------|-------------| +| `--auto-adopt` | Auto-adopt if the gate passes (default: stage only) | +| `--edit-budget N` | Max bounded edits per night (default: 4) | +| `--lookback-hours N` | Harvest window in hours (default: 72) | +| `--json` | Machine-readable JSON output | + +### Config keys (`~/.skillopt-sleep/config.json`) + +- **`preferences`** — free-text house rules for the optimizer +- **`gate_mode`** — `on` (validation-gated, default) or `off` (greedy) +- **`gate_metric`** — `hard` | `soft` | `mixed` (default) +- **`dream_rollouts`** — >1 for multi-rollout contrastive reflection +- **`recall_k`** — >0 recalls similar past tasks from the archive + +### Memory consolidation + +The sleep cycle consolidates both **memory** (AGENTS.md / CLAUDE.md) and **skills** (SKILL.md) by default. Each is independently toggleable via `evolve_memory` / `evolve_skill` config keys. Both are gated by the same held-out validation score. + +## Steps + +1. Run the requested action; capture stdout. +2. For `dry-run` and `run`, report the held-out baseline -> candidate score, + gate action, task count, session count, and exact proposed edits. +3. If a staging directory is printed, read `report.md` before summarizing. +4. `run` only stages a proposal; nothing live changes until `adopt`. +5. Offer adoption only after the user has reviewed the staged proposal. +6. Never hand-edit the user's `AGENTS.md`, memory, or skills as a substitute + for `adopt`; adoption is the safety boundary and writes backups first. + +## Hard rules + +- Harvest is read-only. Do not edit archived sessions or raw transcripts. +- Keep raw secrets, credentials, private user data, and unsanitized transcript + contents out of messages, logs, generated artifacts, and commits. +- Show validation evidence before recommending adoption. +- Treat generated edits as proposals, not as source of truth. +- Do not rely on deprecated custom prompts or `/sleep` slash commands for this + Codex integration. This skill is the entrypoint. + +## Validate + +```bash +python -m skillopt_sleep dry-run --project "$(pwd)" --backend mock --json +python -m skillopt_sleep.experiments.run_gbrain --backend codex \ + --seeds brief-writer --data-root /path/to/gbrain-evals/eval/data/skillopt-v1 \ + --nights 2 --limit-replay 3 --limit-holdout 3 +``` + +A deficient skill goes 0.00 -> 1.00 on a held-out set; the optimizer's edits +are gated on real-task performance. diff --git a/plugins/copilot/README.md b/plugins/copilot/README.md new file mode 100644 index 0000000..6171381 --- /dev/null +++ b/plugins/copilot/README.md @@ -0,0 +1,76 @@ +# SkillOpt-Sleep — GitHub Copilot integration + +Give **Copilot** (CLI or VS Code) a nightly **sleep cycle** via a tiny **MCP +server** that exposes the `skillopt_sleep` engine as tools. MCP is GitHub's +supported way to extend Copilot, so this works across Copilot CLI, VS Code, and +other MCP clients with the same server. + +## What's here + +| File | Purpose | +|---|---| +| `mcp_server.py` | stdlib-only MCP (stdio) server exposing `sleep_*` tools | +| `mcp-config.example.json` | drop-in MCP server config | +| `copilot-instructions.snippet.md` | paste into `.github/copilot-instructions.md` | + +## Install + +Requires Python ≥ 3.10. No third-party packages — the server is pure stdlib. + +1. **Register the MCP server.** Add the server to your Copilot MCP config + (Copilot CLI: `~/.copilot/mcp-config.json`; VS Code: your MCP settings). + Use `mcp-config.example.json` as a template — set `SKILLOPT_SLEEP_REPO` to + this repo's path: + + ```json + { + "mcpServers": { + "skillopt-sleep": { + "command": "python3", + "args": ["/abs/path/SkillOpt-Sleep/plugins/copilot/mcp_server.py"], + "env": { "SKILLOPT_SLEEP_REPO": "/abs/path/SkillOpt-Sleep" } + } + } + } + ``` + +2. **(Optional) Tell Copilot about it.** Append + `copilot-instructions.snippet.md` to your repo's + `.github/copilot-instructions.md` so Copilot reaches for the tools when the + user asks to "run the sleep cycle". + +## Use + +Ask Copilot things like *"run the sleep cycle"*, *"what did the last sleep +propose?"*, *"adopt the staged sleep proposal"*. Copilot calls the MCP tools: +`sleep_status`, `sleep_dry_run`, `sleep_run`, `sleep_adopt`, `sleep_harvest`. + +Each tool takes optional `project`, `backend` (`mock`/`claude`/`codex`/`copilot`), and +`scope` arguments. Default backend is `mock` (no API spend). The `copilot` +backend drives the GitHub Copilot CLI (`copilot -p ... --output-format json`) +and requires the `copilot` CLI to be installed and authenticated. + +For speed, the `copilot` backend runs each call against an isolated +`COPILOT_HOME` with built-in MCP servers and custom instructions disabled, so +your user MCP servers (including this project's own) are not spawned per call +(~5x faster). Override with `SKILLOPT_SLEEP_COPILOT_HOME=`, pick a model +with `SKILLOPT_SLEEP_COPILOT_MODEL`, or set `SKILLOPT_SLEEP_COPILOT_FULL_ENV=1` +to use your real Copilot environment instead. + +## Verify the server directly (no Copilot needed) + +```bash +printf '%s\n' \ + '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}' \ + '{"jsonrpc":"2.0","id":2,"method":"tools/list"}' \ + | SKILLOPT_SLEEP_REPO="$(pwd)" python3 plugins/copilot/mcp_server.py +``` +You should see the server info and the five `sleep_*` tools. + +## Notes / status + +- MCP is the stable, official Copilot extension surface, so this is the most + portable of the three integrations (one server → CLI + IDE). +- The engine and all its controls (gate on/off, multi-rollout, budget, + preferences, optimizer/target split) are identical across platforms — see + [the SkillOpt-Sleep guide section](https://microsoft.github.io/SkillOpt/docs/guideline.html#sleep). diff --git a/plugins/copilot/copilot-instructions.snippet.md b/plugins/copilot/copilot-instructions.snippet.md new file mode 100644 index 0000000..298ead9 --- /dev/null +++ b/plugins/copilot/copilot-instructions.snippet.md @@ -0,0 +1,43 @@ + + +## SkillOpt-Sleep (offline self-evolution) + +This project has SkillOpt-Sleep available via an MCP server (`skillopt-sleep`). +It gives the agent a nightly "sleep cycle": it reviews past sessions, replays +recurring tasks offline, and consolidates validated memory + skills behind a +held-out gate. + +When the user asks to "run the sleep cycle", "review my past sessions", "learn +my preferences", or "make the agent improve from past usage", use the MCP tools: + +- `sleep_status` — what's happened + the latest staged proposal +- `sleep_dry_run` — safe preview, stages nothing +- `sleep_run` — full cycle, stages a reviewed proposal (nothing live changes) +- `sleep_adopt` — apply the staged proposal (backs up first) +- `sleep_harvest` — list mined recurring tasks +- `sleep_schedule` — install a nightly cron entry (set `hour`/`minute`) +- `sleep_unschedule` — remove the nightly cron entry + +### Key parameters (pass as MCP tool arguments) + +- `backend` — `mock` (default, free), `claude`, `codex`, or `copilot` +- `source` — `claude`, `codex`, or `auto` (where to read transcripts) +- `target_skill_path` — explicit SKILL.md to evolve +- `tasks_file` — pre-built TaskRecord JSON (skip harvest) +- `max_tasks` / `max_sessions` — cap workload +- `auto_adopt` — auto-adopt if the gate passes +- `json` — machine-readable output for programmatic use + +### Advanced config (`~/.skillopt-sleep/config.json`) + +- `preferences` — free-text house rules for the optimizer +- `gate_mode` — `on` (default) or `off`; `dream_rollouts` — >1 for more signal +- `evolve_memory` / `evolve_skill` — toggle which docs consolidate + +Always show the user the held-out baseline → candidate score and the proposed +edits before suggesting `sleep_adopt`. Never hand-edit the user's memory/skill +files; only `sleep_adopt` does that, with a backup. diff --git a/plugins/copilot/mcp-config.example.json b/plugins/copilot/mcp-config.example.json new file mode 100644 index 0000000..80b31fa --- /dev/null +++ b/plugins/copilot/mcp-config.example.json @@ -0,0 +1,11 @@ +{ + "mcpServers": { + "skillopt-sleep": { + "command": "python3", + "args": ["plugins/copilot/mcp_server.py"], + "env": { + "SKILLOPT_SLEEP_REPO": "${workspaceFolder}" + } + } + } +} diff --git a/plugins/copilot/mcp_server.py b/plugins/copilot/mcp_server.py new file mode 100755 index 0000000..fe50542 --- /dev/null +++ b/plugins/copilot/mcp_server.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +"""SkillOpt-Sleep — minimal MCP server (stdio, stdlib-only). + +Exposes the sleep engine as MCP tools so any MCP-capable client (GitHub Copilot +CLI / VS Code, Claude Desktop, etc.) can drive it. No third-party deps: speaks +JSON-RPC 2.0 over stdio with just the handful of MCP methods clients need. + +Tools exposed: + - sleep_status : how many nights have run + the latest staged proposal + - sleep_dry_run : harvest+mine+replay, report only (no staging) + - sleep_run : full cycle, stages a proposal (nothing live changes) + - sleep_adopt : apply the latest staged proposal (with backup) + - sleep_harvest : debug — list mined recurring tasks + +Each tool shells out to `python -m skillopt_sleep ...` and returns its +stdout. Configure your client to launch: python plugins/copilot/mcp_server.py +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys + +REPO_ROOT = os.environ.get("SKILLOPT_SLEEP_REPO") or os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "..") +) +PROTOCOL_VERSION = "2024-11-05" + +TOOLS = [ + {"name": "sleep_status", "action": "status", + "description": "Show how many SkillOpt-Sleep nights have run and the latest staged proposal."}, + {"name": "sleep_dry_run", "action": "dry-run", + "description": "Preview a sleep cycle (harvest+mine+replay) without staging anything."}, + {"name": "sleep_run", "action": "run", + "description": "Run a full sleep cycle; stages a reviewed proposal. Nothing live changes until adopt."}, + {"name": "sleep_adopt", "action": "adopt", + "description": "Apply the latest staged proposal to CLAUDE.md/SKILL.md (backs up first)."}, + {"name": "sleep_harvest", "action": "harvest", + "description": "Debug: list the recurring tasks mined from recent sessions."}, + {"name": "sleep_schedule", "action": "schedule", + "description": "Install a nightly cron entry to run the sleep cycle automatically."}, + {"name": "sleep_unschedule", "action": "unschedule", + "description": "Remove the nightly cron entry for a project."}, +] +_BY_NAME = {t["name"]: t for t in TOOLS} + +_TOOL_SCHEMA = { + "type": "object", + "properties": { + "project": {"type": "string", + "description": "Project dir to evolve (default: cwd)."}, + "backend": {"type": "string", "enum": ["mock", "claude", "codex", "copilot"], + "description": "mock = no API spend (default); claude/codex/copilot = real."}, + "scope": {"type": "string", "enum": ["invoked", "all"], + "description": "Harvest scope (default: invoked project only)."}, + "source": {"type": "string", "enum": ["claude", "codex", "auto"], + "description": "Transcript source (default: claude)."}, + "model": {"type": "string", + "description": "Backend-specific model override."}, + "tasks_file": {"type": "string", + "description": "Path to reviewed TaskRecord JSON (skips harvest)."}, + "target_skill_path": {"type": "string", + "description": "Explicit SKILL.md path to evolve/stage/adopt."}, + "progress": {"type": "boolean", + "description": "Print phase progress to stderr."}, + "max_sessions": {"type": "integer", + "description": "Cap harvested sessions per run."}, + "max_tasks": {"type": "integer", + "description": "Cap mined tasks per run."}, + "lookback_hours": {"type": "integer", + "description": "Harvest window in hours (default: 72)."}, + "auto_adopt": {"type": "boolean", + "description": "Auto-adopt if gate passes (default: false)."}, + "json": {"type": "boolean", + "description": "Return machine-readable JSON output."}, + "edit_budget": {"type": "integer", + "description": "Max bounded edits per night (default: 4)."}, + "hour": {"type": "integer", + "description": "Hour for schedule (0-23, default: 3)."}, + "minute": {"type": "integer", + "description": "Minute for schedule (0-59, default: 17)."}, + }, + "additionalProperties": False, +} + + +def _run_engine(action: str, args: dict) -> str: + py = sys.executable or "python3" + cmd = [py, "-m", "skillopt_sleep", action] + # String-valued flags + for flag, key in [ + ("--project", "project"), ("--backend", "backend"), + ("--scope", "scope"), ("--source", "source"), + ("--model", "model"), ("--tasks-file", "tasks_file"), + ("--target-skill-path", "target_skill_path"), + ]: + val = args.get(key) + if val: + cmd += [flag, str(val)] + # Integer-valued flags + for flag, key in [ + ("--max-sessions", "max_sessions"), ("--max-tasks", "max_tasks"), + ("--lookback-hours", "lookback_hours"), ("--edit-budget", "edit_budget"), + ("--hour", "hour"), ("--minute", "minute"), + ]: + val = args.get(key) + if val is not None: + cmd += [flag, str(int(val))] + # Boolean flags + for flag, key in [ + ("--progress", "progress"), ("--auto-adopt", "auto_adopt"), + ("--json", "json"), + ]: + if args.get(key): + cmd.append(flag) + try: + proc = subprocess.run(cmd, cwd=REPO_ROOT, capture_output=True, text=True, timeout=3600) + except Exception as e: + return f"[error] failed to run engine: {e}" + out = (proc.stdout or "").strip() + err = (proc.stderr or "").strip() + return out + (("\n[stderr]\n" + err) if err else "") + + +def _result(id_, result): + return {"jsonrpc": "2.0", "id": id_, "result": result} + + +def _error(id_, code, message): + return {"jsonrpc": "2.0", "id": id_, "error": {"code": code, "message": message}} + + +def handle(req: dict): + method = req.get("method") + id_ = req.get("id") + if method == "initialize": + return _result(id_, { + "protocolVersion": PROTOCOL_VERSION, + "capabilities": {"tools": {}}, + "serverInfo": {"name": "skillopt-sleep", "version": "0.1.0"}, + }) + if method in ("notifications/initialized", "initialized"): + return None # notification, no response + if method == "tools/list": + return _result(id_, {"tools": [ + {"name": t["name"], "description": t["description"], "inputSchema": _TOOL_SCHEMA} + for t in TOOLS + ]}) + if method == "tools/call": + params = req.get("params") or {} + name = params.get("name") + tool = _BY_NAME.get(name) + if not tool: + return _error(id_, -32602, f"unknown tool: {name}") + text = _run_engine(tool["action"], params.get("arguments") or {}) + return _result(id_, {"content": [{"type": "text", "text": text}]}) + if method == "ping": + return _result(id_, {}) + return _error(id_, -32601, f"method not found: {method}") + + +def main() -> int: + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + req = json.loads(line) + except Exception: + continue + resp = handle(req) + if resp is not None: + sys.stdout.write(json.dumps(resp) + "\n") + sys.stdout.flush() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/plugins/copilot/skillopt/README.md b/plugins/copilot/skillopt/README.md new file mode 100644 index 0000000..c4910a2 --- /dev/null +++ b/plugins/copilot/skillopt/README.md @@ -0,0 +1,98 @@ +# SkillOpt — GitHub Copilot integration + +Give **Copilot** (CLI or VS Code) direct access to the **SkillOpt** research +engine via a tiny **MCP server**. MCP is GitHub's supported way to extend +Copilot, so this works across Copilot CLI, VS Code, and other MCP clients with +the same server. + +SkillOpt is **validation-gated, text-space skill optimization**: it reflects on +rollouts, makes bounded edits to a skill, and keeps a change only if it improves +a held-out validation set. This plugin exposes the repo's training and eval +entry points (`scripts/train.py`, `scripts/eval_only.py`) as Copilot tools. + +> This is the companion to the **SkillOpt-Sleep** plugin (`../mcp_server.py`, +> `sleep_*` tools). Sleep evolves a *local coding agent* from your past +> sessions; this server drives the *research* training/eval loops on the +> benchmark configs in [`../../../configs`](../../../configs). + +## What's here + +| File | Purpose | +|---|---| +| `mcp_server.py` | stdlib-only MCP (stdio) server exposing `skillopt_*` tools | +| `mcp-config.example.json` | drop-in MCP server config | +| `copilot-instructions.snippet.md` | paste into `.github/copilot-instructions.md` | + +## Install + +Requires Python ≥ 3.10. The MCP server itself is pure stdlib, but the tools it +launches need SkillOpt's runtime deps — install the package first: + +```bash +pip install -e . # or: pip install -r requirements.txt +``` + +1. **Register the MCP server.** Add the server to your Copilot MCP config + (Copilot CLI: `~/.copilot/mcp-config.json`; VS Code: your MCP settings). + Use `mcp-config.example.json` as a template — set `SKILLOPT_REPO` to this + repo's path: + + ```json + { + "mcpServers": { + "skillopt": { + "command": "python3", + "args": ["/abs/path/SkillOpt/plugins/copilot/skillopt/mcp_server.py"], + "env": { "SKILLOPT_REPO": "/abs/path/SkillOpt" } + } + } + } + ``` + +2. **(Optional) Tell Copilot about it.** Append + `copilot-instructions.snippet.md` to your repo's + `.github/copilot-instructions.md` so Copilot reaches for the tools when the + user asks to "optimize a skill" or "train on a benchmark". + +## Use + +Ask Copilot things like *"what configs can I run?"*, *"optimize the searchqa +skill"*, or *"evaluate this skill on the dataset"*. Copilot calls the MCP tools: +`skillopt_list_configs`, `skillopt_train`, `skillopt_eval`. + +| Tool | Required args | Notes | +|---|---|---| +| `skillopt_list_configs` | — | Lists `configs/**/*.yaml` you can pass as `config`. | +| `skillopt_train` | `config` | Runs a reflective optimization loop. Long-running; spends budget. | +| `skillopt_eval` | `config`, `skill` | Evaluates one skill markdown file; no training. | + +Common optional args (both train and eval): `env`, `backend`, +`optimizer_model`, `target_model`, `out_root`, `cfg_options` (space-separated +`KEY=VALUE` YAML overrides), and `extra_args` (raw passthrough flags for the +underlying script). `skillopt_train` also accepts `num_epochs`, `batch_size`, +`seed`, and `use_gate`. + +Runs can be very long. The server's subprocess timeout defaults to 6 hours; +override it with the `SKILLOPT_RUN_TIMEOUT` environment variable (seconds). + +## Verify the server directly (no Copilot needed) + +```bash +printf '%s\n' \ + '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}' \ + '{"jsonrpc":"2.0","id":2,"method":"tools/list"}' \ + '{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"skillopt_list_configs","arguments":{}}}' \ + | SKILLOPT_REPO="$(pwd)" python3 plugins/copilot/skillopt/mcp_server.py +``` + +You should see the server info, the three `skillopt_*` tools, and the list of +benchmark configs. + +## Notes / status + +- MCP is the stable, official Copilot extension surface, so this is portable + across Copilot CLI and IDE from one server. +- `skillopt_list_configs` is filesystem-only and safe to call anytime; + `skillopt_train` / `skillopt_eval` shell out to the repo scripts and require + the SkillOpt runtime deps (and, for real backends, model credentials — see + [`../../../.env.example`](../../../.env.example)). diff --git a/plugins/copilot/skillopt/copilot-instructions.snippet.md b/plugins/copilot/skillopt/copilot-instructions.snippet.md new file mode 100644 index 0000000..b53c4a5 --- /dev/null +++ b/plugins/copilot/skillopt/copilot-instructions.snippet.md @@ -0,0 +1,33 @@ + + +## SkillOpt (research skill-optimization engine) + +This repo exposes the core **SkillOpt** training/eval engine via an MCP server +(`skillopt`). SkillOpt is validation-gated, text-space skill optimization: it +reflects on rollouts, makes bounded edits to a skill, and keeps a change only +if it improves a held-out validation set. + +When the user asks to "optimize a skill", "train on ", "run +SkillOpt", "evaluate this skill", or "what configs can I run", use the MCP +tools: + +- `skillopt_list_configs` — list the benchmark YAML configs you can pass as `config` +- `skillopt_train` — run a reflective skill-optimization loop on a config (long-running; spends API/compute budget) +- `skillopt_eval` — evaluate a single skill markdown file on a dataset (no training) + +Guidance: +- Always run `skillopt_list_configs` first if you don't already know a valid `config` path. +- `skillopt_train` and `skillopt_eval` are long-running and consume the user's + model backend/budget — confirm the `config`, `backend`, and model choices + with the user before launching, and surface the held-out gate result when the + run finishes. +- For one-off YAML overrides use `cfg_options` (e.g. `seed=123 batch_size=40`); + for any other underlying flag use `extra_args`. + +This is distinct from the **SkillOpt-Sleep** MCP server (`skillopt-sleep`, +`sleep_*` tools), which evolves a local coding agent from past sessions rather +than running the research benchmarks. diff --git a/plugins/copilot/skillopt/mcp-config.example.json b/plugins/copilot/skillopt/mcp-config.example.json new file mode 100644 index 0000000..eb2aba5 --- /dev/null +++ b/plugins/copilot/skillopt/mcp-config.example.json @@ -0,0 +1,11 @@ +{ + "mcpServers": { + "skillopt": { + "command": "python3", + "args": ["plugins/copilot/skillopt/mcp_server.py"], + "env": { + "SKILLOPT_REPO": "${workspaceFolder}" + } + } + } +} diff --git a/plugins/copilot/skillopt/mcp_server.py b/plugins/copilot/skillopt/mcp_server.py new file mode 100644 index 0000000..853877f --- /dev/null +++ b/plugins/copilot/skillopt/mcp_server.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +"""SkillOpt (research engine) — minimal MCP server (stdio, stdlib-only). + +Exposes the core SkillOpt skill-optimization engine as MCP tools so any +MCP-capable client (GitHub Copilot CLI / VS Code, Claude Desktop, etc.) can +drive it. No third-party deps: speaks JSON-RPC 2.0 over stdio with just the +handful of MCP methods clients need. + +This is the companion to the SkillOpt-Sleep MCP server (``../mcp_server.py``). +Where Sleep evolves a *local agent* from past sessions, this server drives the +*research* training/eval loops from this repo (``scripts/train.py`` / +``scripts/eval_only.py``) against the benchmark configs in ``configs/``. + +Tools exposed: + - skillopt_list_configs : discover the benchmark YAML configs you can use + - skillopt_train : run a reflective skill-optimization (training) loop + - skillopt_eval : evaluate a single skill on a dataset (no training) + +``skillopt_train`` and ``skillopt_eval`` shell out to the repo's entry-point +scripts and stream back their stdout/stderr. Configure your client to launch: + python plugins/copilot/skillopt/mcp_server.py +""" +from __future__ import annotations + +import glob +import json +import os +import subprocess +import sys + +# Repo root: three levels up from plugins/copilot/skillopt/mcp_server.py +REPO_ROOT = os.environ.get("SKILLOPT_REPO") or os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "..", "..") +) +PROTOCOL_VERSION = "2024-11-05" + +# Training/eval runs are long; give the engine plenty of headroom. +RUN_TIMEOUT_SECONDS = int(os.environ.get("SKILLOPT_RUN_TIMEOUT", "21600")) # 6h + + +def _list_configs() -> str: + """List the benchmark configs available under configs/ (filesystem only).""" + pattern = os.path.join(REPO_ROOT, "configs", "**", "*.yaml") + paths = sorted(glob.glob(pattern, recursive=True)) + if not paths: + return f"[no configs found under {os.path.join(REPO_ROOT, 'configs')}]" + rels = [os.path.relpath(p, REPO_ROOT).replace(os.sep, "/") for p in paths] + lines = ["Available SkillOpt configs (pass as `config`):", ""] + lines += [f" - {r}" for r in rels] + return "\n".join(lines) + + +def _run_script(script_rel: str, args: dict, *, required: tuple[str, ...] = ()) -> str: + """Shell out to a repo entry-point script, mapping args -> --flags.""" + for key in required: + if not args.get(key): + return f"[error] missing required argument: {key}" + + py = sys.executable or "python3" + cmd = [py, os.path.join("scripts", script_rel)] + + # Ordered flags that the train/eval scripts accept directly. + flag_args = ( + "config", "skill", "split", "env", "backend", + "optimizer_model", "target_model", "out_root", + "num_epochs", "batch_size", "seed", "use_gate", + ) + for key in flag_args: + val = args.get(key) + if val is None or val == "": + continue + cmd += [f"--{key}", str(val)] + + # cfg-options: arbitrary KEY=VALUE YAML overrides (nargs="+"). + cfg_options = args.get("cfg_options") + if cfg_options: + if isinstance(cfg_options, str): + cfg_options = cfg_options.split() + cmd += ["--cfg-options", *[str(x) for x in cfg_options]] + + # extra_args: raw passthrough for any other train/eval flag. + extra = args.get("extra_args") + if extra: + if isinstance(extra, str): + extra = extra.split() + cmd += [str(x) for x in extra] + + try: + proc = subprocess.run( + cmd, cwd=REPO_ROOT, capture_output=True, text=True, + timeout=RUN_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired: + return f"[error] run exceeded {RUN_TIMEOUT_SECONDS}s timeout: {' '.join(cmd)}" + except Exception as e: # noqa: BLE001 + return f"[error] failed to run script: {e}" + out = (proc.stdout or "").strip() + err = (proc.stderr or "").strip() + body = out + (("\n[stderr]\n" + err) if err else "") + return body or f"[done] exit code {proc.returncode}, no output" + + +TOOLS = [ + { + "name": "skillopt_list_configs", + "description": "List the benchmark YAML configs under configs/ that can be passed as `config` to train/eval.", + }, + { + "name": "skillopt_train", + "description": "Run a SkillOpt reflective skill-optimization (training) loop on a benchmark config. Long-running; uses your model backend/budget.", + }, + { + "name": "skillopt_eval", + "description": "Evaluate a single skill markdown file on a dataset without training (scripts/eval_only.py).", + }, +] +_BY_NAME = {t["name"]: t for t in TOOLS} + +_NO_ARGS_SCHEMA = {"type": "object", "properties": {}, "additionalProperties": False} + +_COMMON_PROPS = { + "config": {"type": "string", + "description": "Path to a benchmark YAML config (e.g. configs/searchqa/default.yaml). See skillopt_list_configs."}, + "env": {"type": "string", "description": "Override the environment/adapter name (e.g. searchqa, alfworld)."}, + "backend": {"type": "string", "description": "Model backend (e.g. azure_openai, claude, codex, qwen, minimax)."}, + "optimizer_model": {"type": "string", "description": "Model used for reflection/skill rewriting (the optimizer)."}, + "target_model": {"type": "string", "description": "Model used to execute tasks (the target)."}, + "out_root": {"type": "string", "description": "Output directory root for run artifacts."}, + "cfg_options": {"type": "string", "description": "Space-separated YAML overrides, e.g. 'seed=123 batch_size=40'."}, + "extra_args": {"type": "string", "description": "Raw passthrough flags for the underlying script, e.g. '--workers 8 --max_turns 30'."}, +} + +_TRAIN_SCHEMA = { + "type": "object", + "properties": { + **_COMMON_PROPS, + "num_epochs": {"type": "integer", "description": "Number of optimization epochs."}, + "batch_size": {"type": "integer", "description": "Tasks per optimization step."}, + "seed": {"type": "integer", "description": "Random seed."}, + "use_gate": {"type": "string", "enum": ["true", "false"], + "description": "Whether to keep the held-out validation gate on (default on)."}, + }, + "required": ["config"], + "additionalProperties": False, +} + +_EVAL_SCHEMA = { + "type": "object", + "properties": { + **_COMMON_PROPS, + "skill": {"type": "string", "description": "Path to the skill markdown file to evaluate."}, + "split": {"type": "string", "description": "Dataset split to evaluate (default: all)."}, + }, + "required": ["config", "skill"], + "additionalProperties": False, +} + +_SCHEMA_BY_NAME = { + "skillopt_list_configs": _NO_ARGS_SCHEMA, + "skillopt_train": _TRAIN_SCHEMA, + "skillopt_eval": _EVAL_SCHEMA, +} + + +def _result(id_, result): + return {"jsonrpc": "2.0", "id": id_, "result": result} + + +def _error(id_, code, message): + return {"jsonrpc": "2.0", "id": id_, "error": {"code": code, "message": message}} + + +def _dispatch(name: str, args: dict) -> str: + if name == "skillopt_list_configs": + return _list_configs() + if name == "skillopt_train": + return _run_script("train.py", args, required=("config",)) + if name == "skillopt_eval": + return _run_script("eval_only.py", args, required=("config", "skill")) + return f"[error] unknown tool: {name}" + + +def handle(req: dict): + method = req.get("method") + id_ = req.get("id") + if method == "initialize": + return _result(id_, { + "protocolVersion": PROTOCOL_VERSION, + "capabilities": {"tools": {}}, + "serverInfo": {"name": "skillopt", "version": "0.1.0"}, + }) + if method in ("notifications/initialized", "initialized"): + return None # notification, no response + if method == "tools/list": + return _result(id_, {"tools": [ + {"name": t["name"], "description": t["description"], + "inputSchema": _SCHEMA_BY_NAME[t["name"]]} + for t in TOOLS + ]}) + if method == "tools/call": + params = req.get("params") or {} + name = params.get("name") + if name not in _BY_NAME: + return _error(id_, -32602, f"unknown tool: {name}") + text = _dispatch(name, params.get("arguments") or {}) + return _result(id_, {"content": [{"type": "text", "text": text}]}) + if method == "ping": + return _result(id_, {}) + return _error(id_, -32601, f"method not found: {method}") + + +def main() -> int: + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + req = json.loads(line) + except Exception: + continue + resp = handle(req) + if resp is not None: + sys.stdout.write(json.dumps(resp) + "\n") + sys.stdout.flush() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/plugins/devin/README.md b/plugins/devin/README.md new file mode 100644 index 0000000..3a6bbd7 --- /dev/null +++ b/plugins/devin/README.md @@ -0,0 +1,66 @@ +# SkillOpt-Sleep — Devin integration + +Give **Devin** (Cognition) a nightly **sleep cycle** via a tiny **MCP server** +that exposes the `skillopt_sleep` engine as tools. MCP is Devin's supported way +to add custom tooling, so this works in Devin's CLI and IDE. + +Devin doesn't write transcripts in the format the engine consumes, so this +plugin adds a **Devin-specific harvester** that converts every locally available +source into the Claude Code-compatible JSONL the engine reads. + +## What's here + +| File | Purpose | +|---|---| +| `mcp_server.py` | stdlib-only MCP (stdio) server exposing `sleep_*` tools | +| `harvest_devin.py` | converts Devin ATIF-v1.7 transcripts + agentmemory + `.devin/skills` into JSONL, with `taskKey` + outcome envelopes | +| `judge.py` | reference judge for the deferred/judge branch of the validation gate | +| `mcp-config.example.json` | drop-in MCP server config | +| `devin-rules.snippet.md` | paste into `.devin/rules/skillopt-sleep.md` | + +## What it harvests + +| Source | Where | +|---|---| +| Devin transcripts (ATIF-v1.7) | `~/.local/share/devin/cli/transcripts/*.json` | +| agentmemory | `~/.agentmemory/standalone.json` | +| Skill files | `.devin/skills/*/SKILL.md` | + +Workspaces are auto-detected from `~/.config/Devin/User/workspaceStorage/*/workspace.json`. +After `sleep_adopt`, the evolved skill is synced to `.devin/skills/skillopt-sleep-learned/SKILL.md`. + +## Install + +Requires Python ≥ 3.10. No third-party packages — the server is pure stdlib. + +1. **Register the MCP server.** Use `mcp-config.example.json` as a template; set + `args` to the absolute path of this `mcp_server.py`. The engine is found + automatically (this plugin lives inside the SkillOpt repo). Or via the Devin + CLI: + + ```bash + devin mcp add skillopt-sleep \ + --env "SKILLOPT_DEVIN_CLAUDE_HOME=$HOME/.skillopt-sleep-devin" \ + -- python3 /abs/path/to/SkillOpt/plugins/devin/mcp_server.py + ``` + +2. **(Optional)** copy `devin-rules.snippet.md` to `.devin/rules/skillopt-sleep.md` + so Devin proactively offers the tools. + +3. Ask Devin: *"run the sleep cycle"*, *"what did the last sleep propose?"*, *"adopt it"*. + +## Tools + +| Tool | What it does | +|---|---| +| `sleep_status` | nights run so far + latest staged proposal | +| `sleep_dry_run` | preview cycle — no staging, no changes | +| `sleep_run` | full cycle; stages a proposal for review | +| `sleep_adopt` | apply the staged proposal; syncs skill to the workspace | +| `sleep_harvest` | debug: list the recurring tasks mined | +| `sleep_schedule` | install a nightly cron entry (`--hour` / `--minute`) | +| `sleep_unschedule` | remove the nightly cron entry | + +Default backend is `mock` (no API spend); `--backend claude|codex` uses your own +budget. Same engine and `sleep_*` interface as the other plugins — all call +`python -m skillopt_sleep`. diff --git a/plugins/devin/devin-rules.snippet.md b/plugins/devin/devin-rules.snippet.md new file mode 100644 index 0000000..7ca59a8 --- /dev/null +++ b/plugins/devin/devin-rules.snippet.md @@ -0,0 +1,19 @@ +# SkillOpt-Sleep (Devin) + +You have access to a nightly self-evolution cycle via the `skillopt-sleep` MCP +server. Use these tools to improve your long-term skills over time: + +- **`sleep_status`** — how many nights have run + the latest staged proposal +- **`sleep_dry_run`** — preview a cycle without changing anything +- **`sleep_run`** — run a full cycle; stages a proposal for review +- **`sleep_adopt`** — apply the staged proposal to `.devin/skills/skillopt-sleep-learned/SKILL.md` +- **`sleep_harvest`** — debug: list the recurring tasks mined from recent sessions +- **`sleep_schedule`** / **`sleep_unschedule`** — install/remove a nightly cron run + +When a user asks about the sleep cycle, skill evolution, or improving your +long-term memory, prefer calling these tools over explaining the concept. + +Default backend is `mock` (no API spend). Pass `backend: "claude"` or +`backend: "codex"` with your own API key for real LLM-driven optimization. + +Place this file at `.devin/rules/skillopt-sleep.md` in your workspace. diff --git a/plugins/devin/fixtures/devin_sample.json b/plugins/devin/fixtures/devin_sample.json new file mode 100644 index 0000000..0f522ef --- /dev/null +++ b/plugins/devin/fixtures/devin_sample.json @@ -0,0 +1,21 @@ +{ + "schema_version": "ATIF-v1.7", + "session_id": "demo-001", + "steps": [ + { + "source": "user", + "message": "Fix the failing NullPointerException in OrderService.persist() in the dutch-kis project", + "timestamp": "2026-06-20T10:00:00Z" + }, + { + "source": "agent", + "message": "The repository call returns an Optional that is being unwrapped with .get(). I'll switch to orElseThrow(NotFoundException::new) so the missing-row case is handled.", + "timestamp": "2026-06-20T10:00:05Z" + }, + { + "source": "agent", + "message": "Applied the fix and ran the suite: rtk mvn test -Dtest=OrderServiceTest -> BUILD SUCCESS, 142 passed, 0 failed.", + "timestamp": "2026-06-20T10:01:00Z" + } + ] +} diff --git a/plugins/devin/harvest_devin.py b/plugins/devin/harvest_devin.py new file mode 100644 index 0000000..723dc69 --- /dev/null +++ b/plugins/devin/harvest_devin.py @@ -0,0 +1,533 @@ +#!/usr/bin/env python3 +"""Convert Devin IDE local data into Claude Code-format JSONL transcripts. + +Devin (Cognition) does not persist agent conversation transcripts to disk in a +format the sleep engine understands. This script bridges that gap by synthesising +JSONL files from every locally available source: + + 1. **Devin transcripts** (~/.local/share/devin/cli/transcripts/*.json) + Native ATIF-v1.7 format — source:"user" / source:"agent" messages + converted directly to user/assistant JSONL turns. + + 2. **agentmemory** (~/.agentmemory/standalone.json) + Memories saved by the `agentmemory` MCP server — each memory's title + becomes a synthetic user prompt; its content becomes the assistant reply. + + 3. **Skill files** (.devin/skills/*/SKILL.md) + Each skill description is converted to a session where the user asked + "use the skill" and the assistant described how to apply it. + +Output layout (mirrors ~/.claude/projects//.jsonl): + /projects//.jsonl + +Workspace auto-detection order: + 1. ``SKILLOPT_DEVIN_WORKSPACES`` env var — colon-separated abs paths + 2. Devin registry: ``~/.config/Devin/User/workspaceStorage/*/workspace.json`` + 4. Working directory fallback + +Usage (standalone): + python harvest_devin.py [--out-dir PATH] [--workspaces PATH ...] +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional +from urllib.parse import unquote, urlparse + +# ── cross-platform path resolution (Linux + Windows + macOS) ────────────────── +# +# Devin is a VS Code-family app, so its user-data dir moves with the OS: +# Linux ~/.config/, Windows %APPDATA%\, macOS +# ~/Library/Application Support/. Resolve all candidates and let callers +# keep whichever actually exists. + +def _app_data_roots(app: str) -> List[str]: + """User-data dir candidates for a VS Code-family app, current OS first.""" + home = os.path.expanduser("~") + roots: List[str] = [] + if os.name == "nt": + appdata = os.environ.get("APPDATA") or os.path.join(home, "AppData", "Roaming") + roots.append(os.path.join(appdata, app)) + elif sys.platform == "darwin": + roots.append(os.path.join(home, "Library", "Application Support", app)) + # XDG / Linux (also a sensible fallback everywhere) + xdg = os.environ.get("XDG_CONFIG_HOME") or os.path.join(home, ".config") + roots.append(os.path.join(xdg, app)) + # de-dupe, preserve order + return list(dict.fromkeys(roots)) + + +def _devin_transcript_candidates() -> List[str]: + """Where the Devin CLI may store ATIF transcripts, per OS.""" + home = os.path.expanduser("~") + cands: List[str] = [] + if os.name == "nt": + for base in (os.environ.get("LOCALAPPDATA"), os.environ.get("APPDATA")): + if base: + cands.append(os.path.join(base, "devin", "cli", "transcripts")) + elif sys.platform == "darwin": + cands.append(os.path.join(home, "Library", "Application Support", + "devin", "cli", "transcripts")) + cands.append(os.path.join(home, ".local", "share", "devin", "cli", "transcripts")) + return list(dict.fromkeys(cands)) + + +def _first_existing(paths: List[str]) -> str: + """First path that exists, else the first candidate (for nice messaging).""" + for p in paths: + if os.path.exists(p): + return p + return paths[0] if paths else "" + + +def _uri_to_path(folder: str) -> str: + """Convert a VS Code ``file://`` workspace URI to a local path, cross-platform. + + Linux: file:///home/u/proj -> /home/u/proj + Windows: file:///c%3A/Users/u/p -> c:/Users/u/p + """ + if not folder.startswith("file://"): + return folder + path = unquote(urlparse(folder).path) + # Windows drive paths come through as '/C:/...' — strip the leading slash. + if os.name == "nt" and re.match(r"^/[A-Za-z]:", path): + path = path[1:] + return path + +# ── workspace auto-detection ───────────────────────────────────────────────── + +def _workspaces_from_registry(storage_root: str) -> List[tuple]: + """Read VS Code-style workspaceStorage to get (mtime, path) pairs.""" + results: List[tuple] = [] + if not os.path.isdir(storage_root): + return results + for entry in os.scandir(storage_root): + ws_json = os.path.join(entry.path, "workspace.json") + if not os.path.isfile(ws_json): + continue + try: + with open(ws_json, encoding="utf-8") as f: + data = json.load(f) + folder = _uri_to_path(data.get("folder", "")) + if folder and os.path.isdir(folder): + results.append((os.path.getmtime(ws_json), folder)) + except Exception: + continue + return results + + +def _detect_workspaces() -> List[str]: + """Return known workspace paths (Devin registry), newest first.""" + env_val = os.environ.get("SKILLOPT_DEVIN_WORKSPACES", "") + if env_val: + # os.pathsep so Windows 'C:\a;C:\b' splits correctly (not on the drive colon) + return [p for p in env_val.split(os.pathsep) if p and os.path.isdir(p)] + + registries: List[str] = [ + os.path.join(r, "User", "workspaceStorage") + for r in _app_data_roots("Devin") + ] + + seen: set = set() + results: List[tuple] = [] + for registry in registries: + for mtime, folder in _workspaces_from_registry(registry): + if folder not in seen: + seen.add(folder) + results.append((mtime, folder)) + results.sort(reverse=True) + paths = [p for _, p in results] + return paths if paths else [os.getcwd()] + +# ── helpers ─────────────────────────────────────────────────────────────────── + +def _slug(path: str) -> str: + """SHA-256 of abs-path, first 16 hex chars — matches Claude Code's scheme.""" + return hashlib.sha256(os.path.abspath(path).encode()).hexdigest()[:16] + + +def _iso(epoch_ms: Optional[float] = None) -> str: + dt = (datetime.fromtimestamp(epoch_ms / 1000.0, tz=timezone.utc) + if epoch_ms is not None else datetime.now(tz=timezone.utc)) + return dt.strftime("%Y-%m-%dT%H:%M:%S.000Z") + + +def _write_session( + out_dir: str, project: str, session_id: str, + user_prompts: List[str], assistant_replies: List[str], + timestamp_base_ms: float, + task_key: Optional[str] = None, +) -> None: + slug = _slug(project) + session_dir = os.path.join(out_dir, "projects", slug) + os.makedirs(session_dir, exist_ok=True) + out_path = os.path.join(session_dir, f"{session_id}.jsonl") + ts = timestamp_base_ms + with open(out_path, "w", encoding="utf-8") as f: + for user_text, asst_text in zip(user_prompts, assistant_replies): + user_rec = { + "type": "user", + "message": {"role": "user", "content": user_text}, + "cwd": project, + "timestamp": _iso(ts), + "sessionId": session_id, + "version": "1.0", + } + if task_key: + # grouping key so the miner can collapse repeats into one recurring task + user_rec["taskKey"] = task_key + f.write(json.dumps(user_rec, ensure_ascii=False) + "\n") + # space the reply >=5s after the prompt so a single-turn session + # isn't misclassified as a <3s headless replay and dropped by the + # engine's harvest filter (skillopt_sleep Issue #62). + ts += 5000 + f.write(json.dumps({ + "type": "assistant", + "message": {"role": "assistant", "content": asst_text}, + "timestamp": _iso(ts), + "sessionId": session_id, + "version": "1.0", + }, ensure_ascii=False) + "\n") + ts += 2000 + + +def _append_history(out_dir: str, display: str, project: str, timestamp_ms: float) -> None: + record = {"display": display, "timestamp": timestamp_ms, "project": project} + with open(os.path.join(out_dir, "history.jsonl"), "a", encoding="utf-8") as f: + f.write(json.dumps(record, ensure_ascii=False) + "\n") + + +def _infer_project(text: str, workspaces: List[str]) -> str: + for ws in workspaces: + if os.path.basename(ws.rstrip("/")).lower() in text.lower(): + return ws + return workspaces[0] if workspaces else os.getcwd() + +# ── task identity + outcome extraction (fuel for the validation gate) ───────── +# +# SkillOpt's gate only works "where tasks recur and have a checkable correctness +# signal." These helpers add the two things a raw transcript lacks: +# * a stable taskKey so repeats collapse into one recurring task, and +# * an outcome envelope (success + verifier + re-runnable reference) so the +# held-out replay has something to score against. + +_LANG_HINTS = [ + ("java", r"(java|spring|maven|\bmvn\b|gradle|\.java\b|lombok)"), + ("python", r"(python|pytest|\bpip\b|\.py\b|django|flask)"), + ("ts", r"(typescript|\.tsx?\b|\bnpm\b|jest|node)"), + ("js", r"(javascript|\.jsx?\b)"), + ("sql", r"(\bsql\b|select\s|mariadb|mysql|postgres|\.sql\b)"), + ("go", r"(golang|\bgo test\b|\.go\b)"), + ("rust", r"(rust|cargo|\.rs\b)"), +] +_INTENT_HINTS = [ + ("fix", r"(fix|bug|error|fail|npe|exception|broken|crash)"), + ("implement", r"(implement|add|create|build|introduce|support)"), + ("refactor", r"(refactor|clean ?up|rename|extract|simplify)"), + ("test", r"(test|coverage|assert)"), + ("review", r"(review|audit|inspect)"), + ("optimize", r"(optimi[sz]e|perf|speed up|slow)"), + ("explain", r"(explain|understand|what does|how does)"), +] +_STOPWORDS = {"please", "this", "that", "with", "from", "into", "should", + "would", "code", "using", "the", "have"} + + +def _normalize_task_key(text: str, project: str) -> str: + """Stable '::' grouping key for a task.""" + low = text.lower() + lang = next((n for n, pat in _LANG_HINTS if re.search(pat, low)), "general") + intent = next((n for n, pat in _INTENT_HINTS if re.search(pat, low)), "task") + # target: prefer a CamelCase identifier, then a filename, then first real word + m = re.search(r"\b([A-Z][a-z0-9]+(?:[A-Z][a-z0-9]+)+)\b", text) # CamelCase + if not m: + m = re.search(r"\b([\w-]+\.\w+)\b", text) # filename.ext + if m: + target = m.group(1) + else: + # first content word that isn't a stopword or an intent verb (e.g. "implement") + target = next((w for w in re.findall(r"[a-zA-Z]{4,}", low) + if w not in _STOPWORDS + and not any(re.search(pat, w) for _, pat in _INTENT_HINTS)), + "general") + target = re.sub(r"[^a-zA-Z0-9]+", "-", target).strip("-").lower()[:40] or "general" + return f"{lang}:{intent}:{target}" + + +_PASS_PAT = re.compile( + r"(build success|all tests? pass(?:ed)?|\b\d+ passed\b|\b0 failed\b|" + r"tests? pass(?:ed)?|✓|no errors)", re.IGNORECASE) +_FAIL_PAT = re.compile( + r"(build failure|tests? failed|\b[1-9]\d* failed\b|error:|traceback|" + r"assertion ?error)", re.IGNORECASE) # note: "0 failed" must NOT match +_CMD_PAT = re.compile( + r"((?:rtk\s+)?(?:mvn|gradle|pytest|npm(?:\s+run)?\s+test|yarn\s+test|" + r"go\s+test|cargo\s+test)[^\n`]*)", re.IGNORECASE) + + +def _detect_outcome(messages: List[str]) -> Optional[Dict[str, Any]]: + """Best-effort checkable signal from agent messages. None ⇒ no hard signal.""" + blob = "\n".join(m for m in messages if m) + pass_hit, fail_hit = _PASS_PAT.search(blob), _FAIL_PAT.search(blob) + if not pass_hit and not fail_hit: + return None + verifier = "tests" if re.search(r"test|pytest", blob, re.IGNORECASE) else "build" + out: Dict[str, Any] = { + "success": bool(pass_hit) and not fail_hit, + "verifier": verifier, + "evidence": (pass_hit or fail_hit).group(0).strip(), + } + cmd = _CMD_PAT.search(blob) + if cmd: + # keep only the command itself, dropping any "-> result" / ": output" tail + repro = re.split(r"\s*(?:->|→|:|,)\s*", cmd.group(1))[0].strip() + out["reference"] = {"repro": repro} + return out + + +def _build_rubric(user_prompt: str) -> List[str]: + """Derive checkable criteria from the task so a judge has something to score.""" + crit: List[str] = [] + ids = re.findall(r"\b([A-Z][a-z0-9]+(?:[A-Z][a-z0-9]+)+|[\w-]+\.\w+)\b", user_prompt) + for i in dict.fromkeys(ids): # dedupe, preserve order + crit.append(f"Addresses {i}") + intent = _normalize_task_key(user_prompt, "").split(":")[1] + crit.append({ + "fix": "Resolves the reported defect without introducing new errors", + "implement": "Implements the requested behavior end to end", + "refactor": "Preserves behavior while improving structure", + "test": "Adds or fixes tests that actually exercise the change", + "optimize": "Improves performance without changing results", + }.get(intent, "Satisfies the user's stated request")) + crit.append("Response is concrete and actionable, not a restatement of the task") + return crit[:5] + + +def _judge_rubric_fallback(user_prompt: str) -> Dict[str, Any]: + """When no hard signal exists, attach a rubric and mark the task for judge + scoring. success=None tells the gate to defer/judge rather than trust it. + The actual scoring is done by judge.py (or the engine) at replay time.""" + return { + "success": None, + "verifier": "judge", + "rubric": _build_rubric(user_prompt or ""), + } + + +def _write_outcome(out_dir: str, session_id: str, task_key: str, project: str, + ts_ms: float, outcome: Dict[str, Any]) -> None: + rec = {"type": "outcome", "sessionId": session_id, "taskKey": task_key, + "project": project, "timestamp": _iso(ts_ms), **outcome} + with open(os.path.join(out_dir, "outcomes.jsonl"), "a", encoding="utf-8") as f: + f.write(json.dumps(rec, ensure_ascii=False) + "\n") + +# ── source 1: Devin ATIF-v1.7 transcripts ──────────────────────────────────── + +def harvest_devin_transcripts( + transcripts_dir: str, out_dir: str, workspaces: List[str] +) -> int: + """Convert Devin CLI ATIF-v1.7 transcripts to Claude Code JSONL.""" + if not os.path.isdir(transcripts_dir): + return 0 + written = 0 + for entry in os.scandir(transcripts_dir): + if not entry.name.endswith(".json"): + continue + try: + with open(entry.path, encoding="utf-8") as f: + data = json.load(f) + except Exception: + continue + if data.get("schema_version", "").startswith("ATIF"): + pass # Devin native format + else: + continue + session_id = data.get("session_id") or entry.name[:-5] + steps = data.get("steps") or [] + user_prompts: List[str] = [] + agent_replies: List[str] = [] + project = "" + ts_base: Optional[float] = None + for step in steps: + src = step.get("source", "") + msg = str(step.get("message") or "").strip() + if not msg or src == "system": + continue + if src == "user": + user_prompts.append(msg) + if not project: + project = _infer_project(msg, workspaces) + elif src == "agent": + agent_replies.append(msg) + if ts_base is None: + raw_ts = step.get("timestamp", "") + if raw_ts: + try: + from datetime import datetime as _dt + ts_base = _dt.fromisoformat( + raw_ts.replace("Z", "+00:00") + ).timestamp() * 1000 + except Exception: + pass + if not user_prompts: + continue + if not project: + project = workspaces[0] if workspaces else os.getcwd() + if ts_base is None: + ts_base = datetime.now(tz=timezone.utc).timestamp() * 1000 + # Identity + outcome: what makes this trajectory replayable & gradeable. + task_key = _normalize_task_key(user_prompts[0], project) + outcome = _detect_outcome(agent_replies) or _judge_rubric_fallback(user_prompts[0]) + # Pair turns; pad shorter list + n = max(len(user_prompts), len(agent_replies)) + user_prompts += [""] * (n - len(user_prompts)) + agent_replies += [""] * (n - len(agent_replies)) + sid = f"devin_{session_id}" + _write_session( + out_dir, project, sid, + user_prompts=[p for p in user_prompts if p], + assistant_replies=[r if r else "[no reply recorded]" for r, p in + zip(agent_replies, user_prompts) if p], + timestamp_base_ms=ts_base, + task_key=task_key, + ) + _write_outcome(out_dir, sid, task_key, project, ts_base, outcome) + _append_history( + out_dir, + display=(user_prompts[0] or session_id)[:120], + project=project, + timestamp_ms=ts_base, + ) + written += 1 + return written + + +# ── source 2: agentmemory ───────────────────────────────────────────────────── + +def harvest_agentmemory(agentmemory_path: str, out_dir: str, + workspaces: List[str]) -> int: + if not os.path.isfile(agentmemory_path): + return 0 + with open(agentmemory_path, encoding="utf-8") as f: + data = json.load(f) + memories: Dict[str, Any] = data.get("mem:memories", {}) + written = 0 + base_ts = datetime.now(tz=timezone.utc).timestamp() * 1000 - len(memories) * 60_000 + for i, (mem_id, mem) in enumerate(memories.items()): + title = str(mem.get("title", "")).strip() + content = str(mem.get("content", "")).strip() + if not title or not content: + continue + project = _infer_project(title + " " + content, workspaces) + ts = base_ts + i * 60_000 + _write_session(out_dir, project, mem_id, + user_prompts=[title], + assistant_replies=[content], + timestamp_base_ms=ts) + _append_history(out_dir, display=title[:120], project=project, timestamp_ms=ts) + written += 1 + return written + +# ── source 3: skill files (.devin/skills) ───────────────────────────────────── + +def harvest_skills(workspaces: List[str], out_dir: str) -> int: + written = 0 + seen_ids: set = set() + for ws in workspaces: + skills_root = os.path.join(ws, ".devin", "skills") + if not os.path.isdir(skills_root): + continue + for skill_dir in os.scandir(skills_root): + if not skill_dir.is_dir(): + continue + skill_md = os.path.join(skill_dir.path, "SKILL.md") + if not os.path.isfile(skill_md): + continue + sid = f"skill_{skill_dir.name}" + if sid in seen_ids: + continue + seen_ids.add(sid) + with open(skill_md, encoding="utf-8") as f: + raw = f.read() + body = re.sub(r"^---.*?---\s*", "", raw, flags=re.DOTALL).strip() + if not body: + continue + first_line = body.split("\n")[0].lstrip("# ").strip() + user_ask = f"Please use the {skill_dir.name} skill: {first_line}" + ts = datetime.now(tz=timezone.utc).timestamp() * 1000 - 3_600_000 + _write_session(out_dir, ws, sid, + user_prompts=[user_ask], + assistant_replies=[body[:1200]], + timestamp_base_ms=ts) + _append_history(out_dir, display=user_ask[:120], project=ws, timestamp_ms=ts) + written += 1 + return written + +# ── main ───────────────────────────────────────────────────────────────────── + +def main(argv=None) -> int: + parser = argparse.ArgumentParser( + description="Generate SkillOpt-Sleep transcripts from Devin local data" + ) + parser.add_argument( + "--out-dir", + default=os.path.expanduser("~/.skillopt-sleep-devin"), + help="Output claude_home dir (default: ~/.skillopt-sleep-devin)", + ) + parser.add_argument( + "--agentmemory", + default=os.path.expanduser("~/.agentmemory/standalone.json"), + help="Path to agentmemory standalone.json", + ) + parser.add_argument( + "--devin-transcripts", + default=_first_existing(_devin_transcript_candidates()), + help="Devin CLI ATIF transcripts directory (default: per-OS auto-detect)", + ) + parser.add_argument( + "--workspaces", nargs="*", + help="Workspace paths (default: auto-detect from Devin registry)", + ) + parser.add_argument("--quiet", action="store_true") + args = parser.parse_args(argv) + + out_dir = os.path.expanduser(args.out_dir) + os.makedirs(out_dir, exist_ok=True) + os.makedirs(os.path.join(out_dir, "projects"), exist_ok=True) + + workspaces = args.workspaces or _detect_workspaces() + workspaces = [ws for ws in workspaces if os.path.isdir(ws)] + if not workspaces: + workspaces = [os.getcwd()] + + total = 0 + devin_transcripts = os.path.expanduser(args.devin_transcripts) + n = harvest_devin_transcripts(devin_transcripts, out_dir, workspaces) + if not args.quiet: + print(f"[harvest_devin] devin : {n} sessions") + total += n + + n = harvest_agentmemory(args.agentmemory, out_dir, workspaces) + if not args.quiet: + print(f"[harvest_devin] agentmemory : {n} sessions") + total += n + + n = harvest_skills(workspaces, out_dir) + if not args.quiet: + print(f"[harvest_devin] skill files : {n} sessions") + total += n + + if not args.quiet: + print(f"[harvest_devin] total : {total} synthetic sessions → {out_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/plugins/devin/judge.py b/plugins/devin/judge.py new file mode 100644 index 0000000..cb92495 --- /dev/null +++ b/plugins/devin/judge.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Reference judge for SkillOpt-Sleep — score a candidate reply against a rubric. + +Tasks harvested without a hard test/build signal get ``verifier: "judge"`` and a +``rubric`` (see ``_build_rubric`` in harvest_devin.py). This module is the +scorer the validation gate calls for those tasks: given the rubric and a +candidate reply produced during replay, it returns a score in ``[0, 1]``. The +gate accepts a skill edit only if the *new* skill scores strictly higher on the +held-out tasks. + +It is self-contained on purpose — in a full deployment the SkillOpt engine owns +replay+scoring, but having a runnable reference here lets you sanity-check the +judge path without the engine. + +Backends (select via ``SKILLOPT_JUDGE``): + * ``heuristic`` (default) — keyword-coverage, offline, no API key, deterministic. + * ``claude`` — LLM judge via the Anthropic API (needs ANTHROPIC_API_KEY). + +Usage: + python judge.py --rubric rubric.json --reply reply.txt + echo "" | python judge.py --rubric-inline '["Addresses OrderService", ...]' +""" +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from typing import List + +_STOPWORDS = {"addresses", "resolves", "implements", "without", "introducing", + "behavior", "request", "response", "concrete", "actionable", "not", + "the", "and", "that", "with", "stated", "reported", "actually", + "preserves", "improving", "structure", "requested", "satisfies"} + +# Cheap, fast model is the right default for a judge. +_JUDGE_MODEL = os.environ.get("SKILLOPT_JUDGE_MODEL", "claude-haiku-4-5-20251001") + + +def _content_words(text: str) -> List[str]: + return [w for w in re.findall(r"[A-Za-z][A-Za-z0-9_.\-]{3,}", text.lower()) + if w not in _STOPWORDS] + + +def heuristic_score(reply: str, rubric: List[str]) -> float: + """Fraction of rubric criteria whose key content words appear in the reply. + + Crude but deterministic: each criterion is 'met' if at least one of its + content words shows up in the candidate reply. Good enough to smoke-test the + gate wiring; swap in the claude backend for real judging. + """ + if not rubric: + return 0.0 + low = reply.lower() + met = 0 + for criterion in rubric: + words = _content_words(criterion) + if not words: # nothing to check → treat as met + met += 1 + continue + if any(w in low for w in words): + met += 1 + return round(met / len(rubric), 3) + + +def claude_score(reply: str, rubric: List[str]) -> float: + """LLM judge via the Anthropic API. Returns a 0..1 score. + + Stdlib-only (urllib) so this file stays dependency-free. Falls back to the + heuristic if the key is missing or the call fails, so the gate never hard-errors. + """ + api_key = os.environ.get("ANTHROPIC_API_KEY") + if not api_key: + print("[judge] ANTHROPIC_API_KEY unset — using heuristic", file=sys.stderr) + return heuristic_score(reply, rubric) + import urllib.request + + rubric_block = "\n".join(f"- {c}" for c in rubric) + prompt = ( + "You are scoring an AI agent's reply against a rubric. For each criterion, " + "decide if the reply satisfies it. Respond with ONLY a number between 0 and " + "1 — the fraction of criteria satisfied.\n\n" + f"Rubric:\n{rubric_block}\n\nReply:\n{reply}\n\nScore:" + ) + body = json.dumps({ + "model": _JUDGE_MODEL, + "max_tokens": 8, + "messages": [{"role": "user", "content": prompt}], + }).encode() + req = urllib.request.Request( + "https://api.anthropic.com/v1/messages", data=body, + headers={"content-type": "application/json", "x-api-key": api_key, + "anthropic-version": "2023-06-01"}, + ) + try: + with urllib.request.urlopen(req, timeout=30) as resp: + data = json.load(resp) + text = "".join(b.get("text", "") for b in data.get("content", [])) + m = re.search(r"[01](?:\.\d+)?", text) + return max(0.0, min(1.0, float(m.group(0)))) if m else heuristic_score(reply, rubric) + except Exception as exc: # network/auth/parse — degrade gracefully + print(f"[judge] claude backend failed ({exc}) — using heuristic", file=sys.stderr) + return heuristic_score(reply, rubric) + + +def score(reply: str, rubric: List[str]) -> float: + backend = os.environ.get("SKILLOPT_JUDGE", "heuristic") + return claude_score(reply, rubric) if backend == "claude" else heuristic_score(reply, rubric) + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(description="Score a reply against a rubric (0..1)") + g = p.add_mutually_exclusive_group(required=True) + g.add_argument("--rubric", help="Path to a JSON file containing a list of criteria") + g.add_argument("--rubric-inline", help="Inline JSON list of criteria") + p.add_argument("--reply", help="Path to the reply text (default: stdin)") + args = p.parse_args(argv) + + rubric = (json.load(open(args.rubric, encoding="utf-8")) if args.rubric + else json.loads(args.rubric_inline)) + reply = (open(args.reply, encoding="utf-8").read() if args.reply + else sys.stdin.read()) + print(score(reply, rubric)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/plugins/devin/mcp-config.example.json b/plugins/devin/mcp-config.example.json new file mode 100644 index 0000000..2a6e426 --- /dev/null +++ b/plugins/devin/mcp-config.example.json @@ -0,0 +1,11 @@ +{ + "mcpServers": { + "skillopt-sleep": { + "command": "python3", + "args": ["/abs/path/to/SkillOpt/plugins/devin/mcp_server.py"], + "env": { + "SKILLOPT_DEVIN_CLAUDE_HOME": "~/.skillopt-sleep-devin" + } + } + } +} diff --git a/plugins/devin/mcp_server.py b/plugins/devin/mcp_server.py new file mode 100644 index 0000000..fe57168 --- /dev/null +++ b/plugins/devin/mcp_server.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +"""SkillOpt-Sleep — Devin MCP server (stdio, stdlib-only). + +Exposes the sleep engine as MCP tools so Devin (Cognition) can drive it. No +third-party deps: speaks JSON-RPC 2.0 over stdio with just the handful of MCP +methods clients need. Same `sleep_*` interface and engine flags as +`plugins/copilot`, plus a Devin-specific harvest step. + +Before each data-reading action this server runs `harvest_devin.py` to convert +locally available Devin data (ATIF-v1.7 transcripts, agentmemory memories, and +.devin skill files) into the Claude Code-compatible JSONL the engine consumes, +writing it under SKILLOPT_DEVIN_CLAUDE_HOME and pointing the engine there with +`--claude-home`. After `sleep_adopt` the evolved skill is synced back into the +workspace's `.devin/skills/`. + +Tools: sleep_status, sleep_dry_run, sleep_run, sleep_adopt, sleep_harvest, +sleep_schedule, sleep_unschedule. Each shells out to +`python -m skillopt_sleep ...`. Configure Devin to launch: + python plugins/devin/mcp_server.py +""" +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys + +# expanduser wraps the whole value so a "~/..." env var is expanded too (not +# just a default) — otherwise a literal ~ dir gets created. +REPO_ROOT = os.path.expanduser( + os.environ.get("SKILLOPT_SLEEP_REPO") + or os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +) +PLUGIN_DIR = os.path.dirname(os.path.abspath(__file__)) +CLAUDE_HOME = os.path.expanduser( + os.environ.get("SKILLOPT_DEVIN_CLAUDE_HOME", "~/.skillopt-sleep-devin") +) +MANAGED_SKILL_NAME = os.environ.get("SKILLOPT_MANAGED_SKILL", "skillopt-sleep-learned") +PROTOCOL_VERSION = "2024-11-05" + +TOOLS = [ + {"name": "sleep_status", "action": "status", + "description": "Show how many SkillOpt-Sleep nights have run and the latest staged proposal."}, + {"name": "sleep_dry_run", "action": "dry-run", + "description": "Preview a sleep cycle (harvest+mine+replay) without staging anything."}, + {"name": "sleep_run", "action": "run", + "description": "Run a full sleep cycle; stages a reviewed proposal. Nothing live changes until adopt."}, + {"name": "sleep_adopt", "action": "adopt", + "description": "Apply the latest staged proposal to the managed SKILL.md and sync it into .devin/skills/."}, + {"name": "sleep_harvest", "action": "harvest", + "description": "Debug: list the recurring tasks mined from recent Devin sessions."}, + {"name": "sleep_schedule", "action": "schedule", + "description": "Install a nightly cron entry to run the sleep cycle automatically."}, + {"name": "sleep_unschedule", "action": "unschedule", + "description": "Remove the nightly cron entry for a project."}, +] +_BY_NAME = {t["name"]: t for t in TOOLS} + +_TOOL_SCHEMA = { + "type": "object", + "properties": { + "project": {"type": "string", + "description": "Project dir to evolve (default: cwd)."}, + "backend": {"type": "string", "enum": ["mock", "claude", "codex", "copilot"], + "description": "mock = no API spend (default); claude/codex/copilot = real."}, + "scope": {"type": "string", "enum": ["invoked", "all"], + "description": "Harvest scope (default: invoked project only)."}, + "source": {"type": "string", "enum": ["claude", "codex", "auto"], + "description": "Transcript source (default: claude)."}, + "model": {"type": "string", + "description": "Backend-specific model override."}, + "tasks_file": {"type": "string", + "description": "Path to reviewed TaskRecord JSON (skips harvest)."}, + "target_skill_path": {"type": "string", + "description": "Explicit SKILL.md path to evolve/stage/adopt."}, + "progress": {"type": "boolean", + "description": "Print phase progress to stderr."}, + "max_sessions": {"type": "integer", + "description": "Cap harvested sessions per run."}, + "max_tasks": {"type": "integer", + "description": "Cap mined tasks per run."}, + "lookback_hours": {"type": "integer", + "description": "Harvest window in hours (default: 72)."}, + "auto_adopt": {"type": "boolean", + "description": "Auto-adopt if gate passes (default: false)."}, + "json": {"type": "boolean", + "description": "Return machine-readable JSON output."}, + "edit_budget": {"type": "integer", + "description": "Max bounded edits per night (default: 4)."}, + "hour": {"type": "integer", + "description": "Hour for schedule (0-23, default: 3)."}, + "minute": {"type": "integer", + "description": "Minute for schedule (0-59, default: 17)."}, + }, + "additionalProperties": False, +} + +# actions that read harvested Devin data (schedule/unschedule/adopt don't) +_HARVEST_ACTIONS = {"status", "dry-run", "run", "harvest"} + + +def _run_harvest() -> str: + """Convert local Devin data into the JSONL the engine reads, under CLAUDE_HOME.""" + harvester = os.path.join(PLUGIN_DIR, "harvest_devin.py") + env = dict(os.environ) + env["PYTHONPATH"] = REPO_ROOT + os.pathsep + env.get("PYTHONPATH", "") + try: + proc = subprocess.run( + [sys.executable, harvester, "--out-dir", CLAUDE_HOME], + capture_output=True, text=True, timeout=60, env=env, + ) + out = (proc.stdout or "").strip() + err = (proc.stderr or "").strip() + return out + (("\n[harvest stderr]\n" + err) if err else "") + except Exception as exc: + return f"[harvest_devin] warning: {exc}" + + +def _sync_skill(project: str) -> str: + """After adopt, copy the evolved skill into the workspace's .devin/skills/.""" + src = os.path.join(CLAUDE_HOME, "skills", MANAGED_SKILL_NAME, "SKILL.md") + if not (os.path.isfile(src) and project and os.path.isdir(project)): + return "" + dot_root = os.path.join(project, ".devin") + if not os.path.isdir(dot_root): + return "" + dst_dir = os.path.join(dot_root, "skills", MANAGED_SKILL_NAME) + os.makedirs(dst_dir, exist_ok=True) + dst = os.path.join(dst_dir, "SKILL.md") + shutil.copy2(src, dst) + return f"\n[sleep] synced evolved skill → {dst}" + + +def _run_engine(action: str, args: dict) -> str: + harvest_out = _run_harvest() if action in _HARVEST_ACTIONS else "" + + py = sys.executable or "python3" + cmd = [py, "-m", "skillopt_sleep", action, "--claude-home", CLAUDE_HOME] + # Devin transcripts are converted to the Claude format, so default source=claude + if not args.get("source"): + cmd += ["--source", "claude"] + # String-valued flags + for flag, key in [ + ("--project", "project"), ("--backend", "backend"), + ("--scope", "scope"), ("--source", "source"), + ("--model", "model"), ("--tasks-file", "tasks_file"), + ("--target-skill-path", "target_skill_path"), + ]: + val = args.get(key) + if val: + cmd += [flag, str(val)] + # Integer-valued flags + for flag, key in [ + ("--max-sessions", "max_sessions"), ("--max-tasks", "max_tasks"), + ("--lookback-hours", "lookback_hours"), ("--edit-budget", "edit_budget"), + ("--hour", "hour"), ("--minute", "minute"), + ]: + val = args.get(key) + if val is not None: + cmd += [flag, str(int(val))] + # Boolean flags + for flag, key in [ + ("--progress", "progress"), ("--auto-adopt", "auto_adopt"), ("--json", "json"), + ]: + if args.get(key): + cmd.append(flag) + + env = dict(os.environ) + env["PYTHONPATH"] = REPO_ROOT + os.pathsep + env.get("PYTHONPATH", "") + try: + proc = subprocess.run(cmd, cwd=REPO_ROOT, capture_output=True, + text=True, timeout=3600, env=env) + except Exception as e: + return f"[harvest]\n{harvest_out}\n[error] failed to run engine: {e}" + out = (proc.stdout or "").strip() + err = (proc.stderr or "").strip() + result = (f"[harvest]\n{harvest_out}\n\n" if harvest_out else "") + f"[engine]\n{out}" + if err: + result += f"\n[stderr]\n{err}" + if action == "adopt": + result += _sync_skill(args.get("project") or os.getcwd()) + return result + + +def _result(id_, result): + return {"jsonrpc": "2.0", "id": id_, "result": result} + + +def _error(id_, code, message): + return {"jsonrpc": "2.0", "id": id_, "error": {"code": code, "message": message}} + + +def handle(req: dict): + method = req.get("method") + id_ = req.get("id") + if method == "initialize": + return _result(id_, { + "protocolVersion": PROTOCOL_VERSION, + "capabilities": {"tools": {}}, + "serverInfo": {"name": "skillopt-sleep", "version": "0.1.0"}, + }) + if method in ("notifications/initialized", "initialized"): + return None + if method == "tools/list": + return _result(id_, {"tools": [ + {"name": t["name"], "description": t["description"], "inputSchema": _TOOL_SCHEMA} + for t in TOOLS + ]}) + if method == "tools/call": + params = req.get("params") or {} + name = params.get("name") + tool = _BY_NAME.get(name) + if not tool: + return _error(id_, -32602, f"unknown tool: {name}") + text = _run_engine(tool["action"], params.get("arguments") or {}) + return _result(id_, {"content": [{"type": "text", "text": text}]}) + if method == "ping": + return _result(id_, {}) + return _error(id_, -32601, f"method not found: {method}") + + +def main() -> int: + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + req = json.loads(line) + except Exception: + continue + resp = handle(req) + if resp is not None: + sys.stdout.write(json.dumps(resp) + "\n") + sys.stdout.flush() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/plugins/openclaw/README.md b/plugins/openclaw/README.md new file mode 100644 index 0000000..b443456 --- /dev/null +++ b/plugins/openclaw/README.md @@ -0,0 +1,112 @@ +# OpenClaw Plugin for SkillOpt-Sleep + +Thin shell for running [SkillOpt-Sleep](https://github.com/microsoft/SkillOpt) on [OpenClaw](https://github.com/openclaw/openclaw). + +## What it does + +Adds a nightly "sleep cycle" to any OpenClaw agent. The cycle: + +1. **Harvests** recent session transcripts from `~/.openclaw/agents//sessions/*.jsonl` +2. **Mines** recurring task patterns using the optimizer LLM +3. **Replays** each pattern with the current `SKILL.md` (baseline) and a candidate `SKILL.md` (with proposed edits) +4. **Gates** the candidate against the held-out score (rejects regressions) +5. **Stages** the accepted proposal in `~/.skillopt-sleep/staging//` +6. Leaves adoption to the operator (Ethan) + +Nothing live changes until you adopt. Every adopt backs up first. + +## Install + +The plugin is a thin wrapper around the engine at `~/.openclaw/workspace/SkillOpt/skillopt_sleep/`: + +```bash +# 1. Clone the engine (one-time) +cd ~/.openclaw/workspace +git clone https://github.com/microsoft/SkillOpt.git + +# 2. Install the OpenClaw skill (this folder) +ln -s /path/to/openclaw ~/.openclaw/workspace/skills/skillopt-sleep + +# 3. Configure +cp ~/.openclaw/workspace/skills/skillopt-sleep/config.json ~/.skillopt-sleep/config.json +$EDITOR ~/.skillopt-sleep/config.json +# Set backend = "openclaw-deepseek" +# Set model = "deepseek-v4-pro" (or "deepseek-v4-flash" for budget) + +# 4. Set API key +echo 'export DEEPSEEK_API_KEY="sk-..."' >> ~/.openclaw/.env + +# 5. Add the nightly cron +(crontab -l 2>/dev/null; echo "0 3 * * * cd ~/.openclaw/workspace/skills/skillopt-sleep && bash run_sleep_cron.sh >> ~/.skillopt-sleep/nightly.log 2>&1") | crontab - +``` + +## Use + +### Manual trigger + +```bash +# Run one cycle now +python3 ~/.openclaw/workspace/skills/skillopt-sleep/run_sleep.py + +# Dry run (report only) +python3 ~/.openclaw/workspace/skills/skillopt-sleep/run_sleep.py --dry-run + +# One category only +python3 ~/.openclaw/workspace/skills/skillopt-sleep/run_sleep.py --tasks tests/research-cron-tasks.json +``` + +### Slash command + +```bash +# In any OpenClaw session +/sleep status +/sleep run +/sleep run research-cron +/sleep dry-run +/sleep adopt # adopt most recent accepted proposal +/sleep reject # discard most recent +/sleep cost +``` + +## Architecture + +``` +plugins/openclaw/ +├── README.md # this file +├── run_sleep_cron.sh # wrapper for cron invocation +├── run_sleep.py # main entry point +├── slash_sleep.py # /sleep command implementation +├── skillopt_sleep_openclaw.py # DeepSeek + Ollama backend +├── config.json # engine config +├── SKILL.md # OpenClaw skill manifest +└── tests/ # held-out test sets + ├── research-cron-tasks.json + ├── devops-tasks.json + └── wiki-tasks.json +``` + +The OpenClaw shell is one engine (skillopt_sleep/) + one backend (DeepSeek/Ollama) + four thin wrappers (cron, slash, skill, tests). + +## Why this matters for OpenClaw + +OpenClaw currently has no built-in "self-evolving skills" mechanism. The community has: + +- **Manual skills** — Ethan writes them +- **LLM-generated skills** — one-shot, no validation +- **Self-revision** — unbounded, no quality bar + +SkillOpt-Sleep adds a 4th option: **validated self-evolution**. The skill is the training target, the engine is the optimizer, the gate is the quality bar, the operator is the human-in-the-loop. + +## Validation + +Validated on the public [gbrain-evals](https://github.com/garrytan/gbrain-evals) `skillopt-v1` benchmark with real Claude and Codex (deficient skills 0.00 → 1.00 on held-out, all 4 seeds). + +End-to-end test on our own 14-task held-out set: pipeline runs, gate correctly rejects non-improvements, staging artifacts land in `~/.skillopt-sleep/staging//`. + +## Cost + +Measured: ~$0.02/night with `deepseek-v4-pro` at 12 tasks/night. ~$0.59/month, $7.18/year. + +## License + +MIT (same as SkillOpt core). diff --git a/plugins/openclaw/SKILL.md b/plugins/openclaw/SKILL.md new file mode 100644 index 0000000..66b24ac --- /dev/null +++ b/plugins/openclaw/SKILL.md @@ -0,0 +1,129 @@ +--- +name: skillopt-sleep +description: Validate and refine agent skills through nightly sleep cycles with held-out gates. Wraps Microsoft's SkillOpt-Sleep engine for the OpenClaw/DeepSeek stack. +--- + +# skillopt-sleep — OpenClaw Adaptation of Microsoft SkillOpt-Sleep + +A nightly self-improvement loop that reads our session transcripts, mines recurring workflow patterns, replays them with proposed skill edits, and gates the proposals against a held-out test set. Only improvements that beat baseline are staged for human adoption. + +## When To Use + +- After Hermes's Weekly Skill Review (or as its replacement) +- When a skill is being used 10+ times/week and could be tighter +- Before promoting a new skill from `skill-proposals/` to `skills/` +- When a skill regresses in observed quality + +## What It Does (One Cycle) + +``` +harvest session transcripts -> mine recurring task patterns + -> replay each pattern (current skill vs proposed) + -> GATE: must improve held-out score + -> stage proposal + -> Ethan adopts (manual) +``` + +Nothing live changes until Ethan adopts. Every adopt backs up first. + +## Architecture + +``` +skills/skillopt-sleep/ +├── SKILL.md # this file +├── config.json # engine config (backend, budgets, etc.) +├── run_sleep.py # entry point +└── skillopt_sleep_openclaw.py # DeepSeek/Ollama backend +``` + +The engine itself is at `~/.openclaw/workspace/SkillOpt/skillopt_sleep/` (cloned from microsoft/SkillOpt). + +## Usage + +```bash +# Run one cycle with current config +cd ~/.openclaw/workspace/skills/skillopt-sleep +python3 run_sleep.py + +# Dry run (report only, no staging) +python3 run_sleep.py --dry-run + +# Use a pre-built task set (recommended for testing) +python3 run_sleep.py --tasks tests/research-cron-tasks.json +``` + +## Scheduling + +```bash +python3 slash_sleep.py schedule --hour 3 --minute 17 +python3 slash_sleep.py unschedule +python3 slash_sleep.py unschedule --all +``` + +Installs a nightly cron entry using the shared SkillOpt-Sleep scheduler. This is an alternative to the external `run_sleep_cron.sh` script. + +## Alternative backends + +While OpenClaw defaults to `openclaw-deepseek` (DeepSeek V4 Pro + Ollama), the shared engine also supports: +- `--backend mock` — deterministic, no API spend (for testing) +- `--backend claude` — uses the Claude CLI +- `--backend codex` — uses the Codex CLI +- `--backend copilot` — uses the GitHub Copilot CLI + +These can be used via the engine directly (`python -m skillopt_sleep`). + +## Shared-engine flags + +When invoking the engine directly, all standard flags are available: +- `--source codex` / `--source auto` — harvest from Codex Desktop sessions +- `--tasks-file PATH` — use a pre-built task set +- `--target-skill-path PATH` — explicit SKILL.md target +- `--max-tasks N` / `--max-sessions N` — cap workload +- `--progress` — print phase progress +- `--json` — machine-readable output +- `--auto-adopt` — auto-adopt if gate passes + +Config keys: `preferences`, `gate_mode`, `gate_metric`, `dream_rollouts`, `recall_k`, `evolve_memory`, `evolve_skill`. + +## Config (config.json) + +Key knobs: +- `backend: "openclaw-deepseek"` — our custom backend +- `model: "deepseek-v4-pro"` — optimizer model +- `edit_budget: 3` — max bounded edits per night +- `gate_mode: "on"` — validation-gated (rejects regressions) +- `auto_adopt: false` — require Ethan to adopt manually +- `max_tasks_per_night: 12` — cap to control cost + +## Cost Estimate + +Per night: 12 tasks × (1 attempt + 1 judge + 1 reflect) × ~$0.005/1K tokens × ~3K tokens/call ≈ **$0.50-2.00/night**. + +## Outputs + +- Report: `~/.skillopt-sleep/state.json` (running totals) +- Staging: `~/.skillopt-sleep/staging//` + - `report.md` — readable summary + - `best_skill.md` — proposed skill + - `edits.json` — bounded edit list + - `before.md` / `after.md` — diffs + +## Held-Out Test Sets (Phase 2) + +Located at `tests/-tasks.json`. Each task has: +- `prompt` — the recurring task +- `reference` — exact-match gold answer +- `rubric` — soft score rubric (0-1) +- `domain` — research/devops/wiki/etc. + +Currently building for 3 categories: +- research-cron-output +- devops-infrastructure-check +- wiki-canonical-guide + +## When NOT To Use + +- For a one-off workflow (not a recurring pattern) +- During a crisis/incident (humans must lead) +- When session transcripts are < 24h old (not enough signal) +- For skills < 300 tokens (over-optimization risk) diff --git a/plugins/openclaw/config.json b/plugins/openclaw/config.json new file mode 100644 index 0000000..60bc40e --- /dev/null +++ b/plugins/openclaw/config.json @@ -0,0 +1,30 @@ +{ + "_comment": "OpenClaw adaptation of skillopt-sleep. Edit and run via run_sleep.py", + + "claude_home": "/home/ethanclaw/.openclaw/agents", + "invoked_project": "/home/ethanclaw/.openclaw/workspace", + "projects": "invoked", + "lookback_hours": 168, + + "max_tasks_per_night": 12, + "max_tokens_per_night": 800000, + "holdout_fraction": 0.34, + "val_fraction": 0.34, + "test_fraction": 0.0, + + "backend": "openclaw-deepseek", + "model": "deepseek-v4-pro", + "gate_mode": "on", + "edit_budget": 3, + "gate_metric": "mixed", + "gate_mixed_weight": 0.5, + "replay_mode": "fresh", + "evolve_memory": true, + "evolve_skill": true, + "llm_mine": false, + + "auto_adopt": false, + "managed_skill_name": "skillopt-sleep-learned", + "redact_secrets": true, + "seed": 42 +} diff --git a/plugins/openclaw/run_sleep.py b/plugins/openclaw/run_sleep.py new file mode 100755 index 0000000..516d758 --- /dev/null +++ b/plugins/openclaw/run_sleep.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""run_sleep.py — OpenClaw entry point for SkillOpt-Sleep. + +Runs one nightly sleep cycle: + 1. harvest recent session transcripts + 2. mine recurring task patterns + 3. replay tasks with current skill (baseline) + candidate skill (with proposed edit) + 4. gate candidate vs baseline on held-out accuracy + 5. stage the proposal in ~/.skillopt-sleep/staging// + 6. leave adoption to Ethan (auto_adopt=false) + +Usage: + python3 run_sleep.py # one cycle, default config + python3 run_sleep.py --dry-run # compute report only, no staging + python3 run_sleep.py --tasks path.json # use a pre-built task file +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +# Ensure the skillopt_sleep package is importable (it lives in the cloned repo) +REPO = Path("/home/ethanclaw/.openclaw/workspace/SkillOpt") +sys.path.insert(0, str(REPO)) + +# Register our backend before importing cycle +from skillopt_sleep_openclaw import OpenClawDeepSeekBackend +import skillopt_sleep.backend as _b +_b._BACKENDS = getattr(_b, "_BACKENDS", {}) +_b._BACKENDS["openclaw-deepseek"] = OpenClawDeepSeekBackend + +# Patch get_backend to know about our backend +_orig_get_backend = _b.get_backend + +def get_backend(name, model="", codex_path=""): + if name == "openclaw-deepseek": + return OpenClawDeepSeekBackend(model=model or "deepseek-v4-pro") + return _orig_get_backend(name, model=model, codex_path=codex_path) + +_b.get_backend = get_backend + +from skillopt_sleep.cycle import run_sleep_cycle +from skillopt_sleep.config import load_config + + +def main() -> int: + ap = argparse.ArgumentParser(description="OpenClaw SkillOpt-Sleep nightly cycle") + ap.add_argument("--dry-run", action="store_true", help="Compute but don't stage") + ap.add_argument("--config", default="/home/ethanclaw/.openclaw/workspace/skills/skillopt-sleep/config.json") + ap.add_argument("--tasks", default=None, help="Path to pre-built tasks JSON") + ap.add_argument("--verbose", action="store_true") + args = ap.parse_args() + + # Load config from file then override with our defaults + overrides = {} + if os.path.exists(args.config): + with open(args.config) as f: + overrides.update(json.load(f)) + overrides.pop("_comment", None) + + cfg = load_config(**overrides) + + seed_tasks = None + if args.tasks: + from skillopt_sleep.types import TaskRecord + with open(args.tasks) as f: + raw = json.load(f) + # Translate our test-set fields → TaskRecord fields + seed_tasks = [] + for t in raw: + seed_tasks.append(TaskRecord( + id=t['id'], + project=t.get('project', 'openclaw'), + intent=t.get('intent') or t.get('prompt', ''), + context_excerpt=t.get('context_excerpt', ''), + attempted_solution=t.get('attempted_solution', ''), + outcome=t.get('outcome', 'unknown'), + reference_kind=t.get('reference_kind', 'rubric'), + reference=t.get('reference', ''), + judge=t.get('judge', {}), + tags=t.get('tags', []), + source_sessions=t.get('source_sessions', []), + split=t.get('split', 'train'), + )) + + print(f"[skillopt-sleep] starting cycle...") + print(f" backend: {cfg.get('backend')}") + print(f" project: {cfg.get('invoked_project')}") + print(f" max tasks: {cfg.get('max_tasks_per_night')}") + print(f" edit budget: {cfg.get('edit_budget')}") + print(f" dry_run: {args.dry_run}") + + outcome = run_sleep_cycle(cfg, seed_tasks=seed_tasks, dry_run=args.dry_run) + + r = outcome.report + print(f"\n=== Report — night {r.night} ===") + print(f" sessions harvested: {r.n_sessions}") + print(f" tasks mined: {r.n_tasks} (replayed: {r.n_replayed})") + print(f" baseline: {r.baseline_score:.3f} -> candidate: {r.candidate_score:.3f}") + print(f" gate: {r.gate_action} accepted={r.accepted}") + print(f" tokens: {r.tokens_used}") + if r.edits: + print(f" applied edits ({len(r.edits)}):") + for e in r.edits: + print(f" [{e.target}/{e.op}] {e.content[:80]}...") + if r.rejected_edits: + print(f" rejected edits ({len(r.rejected_edits)}) — kept as negative feedback") + if r.notes: + for n in r.notes: + print(f" note: {n}") + if outcome.staging_dir: + print(f"\n STAGED at: {outcome.staging_dir}") + print(f" Review with: ls {outcome.staging_dir}") + + return 0 if r.accepted or r.candidate_score >= r.baseline_score else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/plugins/openclaw/run_sleep_cron.sh b/plugins/openclaw/run_sleep_cron.sh new file mode 100755 index 0000000..3053593 --- /dev/null +++ b/plugins/openclaw/run_sleep_cron.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# run_sleep_cron.sh — wrapper for cron-driven nightly sleep cycle +# +# Usage: bash run_sleep_cron.sh [category1 category2 ...] +# No args: run on all categories in tests/ +# With args: run only on listed categories (research-cron, devops, wiki) +# +# Cron (3am MYT daily): +# 0 3 * * * cd /home/ethanclaw/.openclaw/workspace/skills/skillopt-sleep && bash run_sleep_cron.sh >> ~/.skillopt-sleep/nightly.log 2>&1 + +set -euo pipefail + +SKILL_DIR="/home/ethanclaw/.openclaw/workspace/skills/skillopt-sleep" +TESTS_DIR="$SKILL_DIR/tests" +LOG_DIR="$HOME/.skillopt-sleep/logs" +mkdir -p "$LOG_DIR" + +TIMESTAMP=$(date +%Y%m%d-%H%M%S) +LOG_FILE="$LOG_DIR/night-$TIMESTAMP.log" + +# category → test file map +declare -A CATEGORIES=( + ["research-cron"]="research-cron-tasks.json" + ["devops"]="devops-tasks.json" + ["wiki"]="wiki-tasks.json" +) + +# Determine which categories to run +if [ $# -eq 0 ]; then + CATS=("research-cron" "devops" "wiki") +else + CATS=("$@") +fi + +{ + echo "==========================================" + echo "SkillOpt-Sleep nightly — $TIMESTAMP" + echo "Categories: ${CATS[*]}" + echo "==========================================" +} | tee -a "$LOG_FILE" + +# Pre-flight: check DeepSeek API key +if ! grep -q "DEEPSEEK_API_KEY=" "$HOME/.openclaw/.env" 2>/dev/null; then + echo "ERROR: DEEPSEEK_API_KEY not found in ~/.openclaw/.env" | tee -a "$LOG_FILE" + exit 1 +fi + +EXIT_CODE=0 +for cat in "${CATS[@]}"; do + tasks_file="$TESTS_DIR/${CATEGORIES[$cat]:-}" + if [ ! -f "$tasks_file" ]; then + echo "SKIP: $cat (no tasks file: $tasks_file)" | tee -a "$LOG_FILE" + continue + fi + + echo "" | tee -a "$LOG_FILE" + echo "--- [$cat] starting cycle ---" | tee -a "$LOG_FILE" + + cd "$SKILL_DIR" + if python3 run_sleep.py --tasks "$tasks_file" 2>&1 | tee -a "$LOG_FILE"; then + echo "--- [$cat] OK ---" | tee -a "$LOG_FILE" + else + EC=$? + echo "--- [$cat] FAILED (exit $EC) ---" | tee -a "$LOG_FILE" + EXIT_CODE=$EC + fi +done + +{ + echo "" + echo "==========================================" + echo "Done. Exit: $EXIT_CODE" + echo "==========================================" +} | tee -a "$LOG_FILE" + +exit $EXIT_CODE diff --git a/plugins/openclaw/skillopt_sleep_openclaw.py b/plugins/openclaw/skillopt_sleep_openclaw.py new file mode 100644 index 0000000..119030a --- /dev/null +++ b/plugins/openclaw/skillopt_sleep_openclaw.py @@ -0,0 +1,275 @@ +"""OpenClaw backend for SkillOpt-Sleep. + +Adapts the skillopt_sleep Backend protocol to our DeepSeek + Ollama stack: + - attempt/judge/reflect -> DeepSeek V4 Pro (or Flash for cost) + - embeddings -> Ollama nomic-embed-text (already configured) + +This backend NEVER mutates live state. It only returns text + EditRecord +proposals that the gate stages for human review. +""" +from __future__ import annotations + +import json +import os +import re +import subprocess +from typing import Any, Dict, List, Optional, Tuple + +from skillopt_sleep.backend import Backend, _normalize, exact_score +from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord + + +# ── DeepSeek + Ollama OpenAI-compatible API client (curl-based, no extra deps) ── + + +def _chat(messages: List[Dict[str, str]], *, model: str, temperature: float = 0.2, max_tokens: int = 1500) -> str: + """Call DeepSeek V4 Pro via curl + jq. No extra Python deps needed.""" + import json as _json + import urllib.request + + api_key = os.environ.get("DEEPSEEK_API_KEY", "") + if not api_key: + # try loading from .env + env_path = os.path.expanduser("~/.openclaw/.env") + if os.path.exists(env_path): + with open(env_path) as f: + for line in f: + if line.startswith("DEEPSEEK_API_KEY="): + api_key = line.split("=", 1)[1].strip() + break + + base = os.environ.get("DEEPSEEK_BASE_URL", "https://api.deepseek.com/v1") + + payload = { + "model": model, + "messages": messages, + "temperature": temperature, + "max_tokens": max_tokens, + "stream": False, + } + req = urllib.request.Request( + f"{base}/chat/completions", + data=_json.dumps(payload).encode("utf-8"), + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {api_key}", + }, + ) + try: + with urllib.request.urlopen(req, timeout=180) as resp: + data = _json.loads(resp.read().decode("utf-8")) + return data["choices"][0]["message"]["content"] + except Exception as e: + return f"[BACKEND_ERROR] {type(e).__name__}: {str(e)[:200]}" + + +def _embed(text: str) -> List[float]: + """Call Ollama for embeddings. Uses the configured nomic-embed-text model.""" + import json as _json + import urllib.request + + try: + req = urllib.request.Request( + "http://127.0.0.1:11434/api/embeddings", + data=_json.dumps({"model": "nomic-embed-text:latest", "prompt": text[:2000]}).encode("utf-8"), + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=30) as resp: + data = _json.loads(resp.read().decode("utf-8")) + return data.get("embedding", []) + except Exception: + return [] + + +# ── Backend implementation ──────────────────────────────────────────────────── + + +class OpenClawDeepSeekBackend(Backend): + """Use DeepSeek V4 Pro for attempt/judge/reflect, Ollama for embeddings. + + - "model" passed to constructor = optimizer model (default: deepseek-v4-pro) + - "judge_model" = judge model (default: deepseek-v4-pro for quality) + - "cheap_model" = budget-fallback (deepseek-v4-flash) + """ + + name = "openclaw-deepseek" + + def __init__( + self, + model: str = "deepseek-v4-pro", + judge_model: str = "deepseek-v4-pro", + cheap_model: str = "deepseek-v4-flash", + ): + self._model = model + self._judge_model = judge_model + self._cheap_model = cheap_model + self._tokens = 0 # rough estimate + + def tokens_used(self) -> int: + return self._tokens + + # ── 1. attempt: produce a response given the task + skill + memory ── + def attempt(self, task: TaskRecord, skill: str, memory: str) -> str: + sys = ( + "You are an OpenClaw agent (Kobe ecosystem). Use the skill and memory below to complete the task. " + "If the task asks for a structured output, follow the rubric exactly. " + "Be concise. No preamble, no explanation unless the task asks for it." + ) + usr = f"""## SKILL +{skill or '(no skill yet)'} + +## MEMORY +{memory or '(no memory yet)'} + +## TASK +{task.intent} + +## CONTEXT (if any) +{task.context_excerpt or '(none)'} + +## RESPONSE +""" + out = _chat( + [{"role": "system", "content": sys}, {"role": "user", "content": usr}], + model=self._model, + temperature=0.2, + ) + self._tokens += len(usr) // 4 + 200 + return out + + # ── 2. judge: score the response ── + def judge(self, task: TaskRecord, response: str) -> Tuple[float, float, str]: + # Hard score: exact-match against task.reference (if available) + hard = exact_score(task.reference or "", response) + + # Soft score: LLM judge against rubric (reference if reference_kind=='rubric') + rubric_text = task.reference if task.reference_kind == "rubric" else "" + if rubric_text: + judge_prompt = f"""You are a strict grader. Score the response 0.0-1.0 against the rubric. + +## TASK +{task.intent} + +## REFERENCE +{task.reference or '(none)'} + +## RUBRIC +{rubric_text} + +## RESPONSE +{response[:3000]} + +## INSTRUCTIONS +Return ONLY a single float 0.0-1.0 on one line. No explanation. No markdown. +""" + try: + j_out = _chat( + [{"role": "user", "content": judge_prompt}], + model=self._judge_model, + temperature=0.0, + max_tokens=20, + ).strip() + soft = float(re.search(r"[\d.]+", j_out.splitlines()[0]).group()) + soft = max(0.0, min(1.0, soft)) + except Exception: + soft = hard + self._tokens += 600 + else: + soft = hard + + rationale = f"hard={hard:.2f} soft={soft:.2f}" + return hard, soft, rationale + + # ── 3. reflect: produce bounded EditRecord proposals ── + def reflect( + self, + failures: List[Tuple[TaskRecord, ReplayResult]], + successes: List[Tuple[TaskRecord, ReplayResult]], + skill: str, + memory: str, + *, + edit_budget: int, + evolve_skill: bool, + evolve_memory: bool, + ) -> List[EditRecord]: + # Compact digest of failures + successes + fail_digest = "\n".join( + f"- TASK: {t.intent[:200]}\n RESPONSE: {r.response[:300]}\n WHY FAIL: {r.judge_rationale or r.fail_reason or 'unknown'}\n REFERENCE: {t.reference[:200]}" + for t, r in failures[:5] + ) or "(none)" + succ_digest = "\n".join( + f"- TASK: {t.intent[:150]} -> OK ({r.judge_rationale or 'high score'})" + for t, r in successes[:3] + ) or "(none)" + + rubric_text = "" + if failures: + rubric_text = f"\n\n## REFERENCE ANSWERS\n{chr(10).join(f'Q: {t.intent[:120]}\\nA: {t.reference}' for t, _ in failures[:3] if t.reference)}" + + sys = ( + "You are SkillOpt-Sleep's bounded-edit optimizer. Your job is to propose 1-4 MINIMAL text edits to a skill or memory document " + "that, if applied, would help future agents do better on the failed tasks. " + "NEVER propose adding new sections wholesale. NEVER delete entire sections. " + "Edit primitives: ADD (append a step/rule at end), DELETE (remove a specific line by exact match), REPLACE (swap a specific line for another by exact match). " + "If you cannot identify a clear, minimal improvement, return an empty list." + ) + usr = f"""## CURRENT SKILL +{skill or '(empty)'} + +## CURRENT MEMORY +{memory or '(empty)'} + +## FAILED TASKS +{fail_digest} + +## SUCCESSFUL TASKS +{succ_digest} +{rubric_text} + +## CONSTRAINTS +- max {edit_budget} edits total +- edits go to {"skill + memory" if (evolve_skill and evolve_memory) else ("skill" if evolve_skill else "memory")} +- if evolve_skill=False, target="memory" only; if evolve_memory=False, target="skill" only +- target must be "skill" or "memory" + +## OUTPUT FORMAT (JSON, no markdown) +{{"edits": [{{"op": "ADD"|"DELETE"|"REPLACE", "target": "skill"|"memory", "content": "the text to add or replace with", "old_text": "for REPLACE/DELETE, the exact line to find", "rationale": "one short sentence why"}}]}} +""" + out = _chat( + [{"role": "system", "content": sys}, {"role": "user", "content": usr}], + model=self._model, + temperature=0.4, + max_tokens=2000, + ) + self._tokens += len(usr) // 3 + 1500 + + # parse + try: + # strip markdown fences if any + cleaned = out.strip() + if cleaned.startswith("```"): + cleaned = re.sub(r"^```[a-z]*\n?", "", cleaned) + cleaned = re.sub(r"\n?```$", "", cleaned) + data = json.loads(cleaned) + edits: List[EditRecord] = [] + for e in data.get("edits", [])[:edit_budget]: + if e.get("op") not in ("ADD", "DELETE", "REPLACE"): + continue + target = e.get("target", "skill") + if target not in ("skill", "memory"): + continue + if not evolve_skill and target == "skill": + continue + if not evolve_memory and target == "memory": + continue + edits.append(EditRecord( + op=e["op"], + target=target, + content=e.get("content", ""), + old_text=e.get("old_text", ""), + rationale=e.get("rationale", ""), + )) + return edits + except Exception as e: + # log + return empty list (no edit is better than a bad edit) + return [] diff --git a/plugins/openclaw/slash_sleep.py b/plugins/openclaw/slash_sleep.py new file mode 100755 index 0000000..c857666 --- /dev/null +++ b/plugins/openclaw/slash_sleep.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +"""slash_sleep.py — OpenClaw slash command equivalent of SkillOpt's /sleep. + +Use from the main session as a /sleep command: + /sleep status — show current state + last 5 nights + /sleep run — trigger one cycle (all categories) right now + /sleep run research-cron — one cycle, single category + /sleep adopt [night] — adopt the most recent (or specified) staged proposal + /sleep reject [night] — discard the most recent (or specified) staging dir + /sleep dry-run — report-only cycle + /sleep cost — estimate per-night cost for current config + +This script is a thin shell over run_sleep.py. It can be invoked either +manually from the main session or by an OpenClaw command handler. +""" +from __future__ import annotations + +import argparse +import json +import os +import shutil +import sys +from pathlib import Path +from datetime import datetime + +SKILL_DIR = Path("/home/ethanclaw/.openclaw/workspace/skills/skillopt-sleep") +STATE_DIR = Path(os.path.expanduser("~/.skillopt-sleep")) # default +STAGING_ROOT = STATE_DIR + +def _resolve_state_dir(): + """Find the actual state dir. + + Priority: scan in order: + 1. ~/.skillopt-sleep/ (default) + 2. /home/ethanclaw/.openclaw/workspace/.skillopt-sleep/ (when staging is there) + 3. /home/ethanclaw/.openclaw/.skillopt-sleep/ (parent of overridden claude_home) + Pick the first one that has a state.json OR staging dir. + """ + candidates = [ + Path(os.path.expanduser("~/.skillopt-sleep")), + Path("/home/ethanclaw/.openclaw/workspace/.skillopt-sleep"), + Path("/home/ethanclaw/.openclaw/.skillopt-sleep"), + ] + # Prefer the one with state.json + for c in candidates: + if (c / "state.json").exists(): + return c + # Then the one with staging + for c in candidates: + if (c / "staging").exists(): + return c + return candidates[0] + +TESTS_DIR = SKILL_DIR / "tests" + + +def status() -> int: + state_dir = _resolve_state_dir() + state_file = state_dir / "state.json" + staging_dir = state_dir / "staging" + print(f"=== SkillOpt-Sleep status ===") + print(f" state dir: {state_dir}") + print(f" staging dir: {staging_dir}") + if staging_dir.exists(): + stages = sorted(staging_dir.iterdir(), key=lambda p: p.stat().st_mtime, reverse=True) + print(f" staging entries: {len(stages)}") + for s in stages[:3]: + print(f" {s.name}") + if not state_file.exists(): + print(" no state.json — run a cycle first (state is written at end of each non-dry-run)") + return 0 + + with open(state_file) as f: + state = json.load(f) + + nights = state.get("history") or state.get("nights", []) + print(f" total nights: {len(nights)}") + print(f" accepted: {sum(1 for n in nights if n.get('accepted'))}") + print(f" rejected: {sum(1 for n in nights if not n.get('accepted'))}") + if nights: + last = nights[-1] + print(f" last night: {last.get('night')}") + print(f" accepted: {last.get('accepted')}") + print(f" baseline: {last.get('baseline'):.3f} -> candidate: {last.get('candidate'):.3f}") + print(f" staging: {last.get('staging') or '(none)'}") + return 0 + + +def run_category(category: str, *, dry_run: bool = False) -> int: + cat_to_file = { + "research-cron": "research-cron-tasks.json", + "devops": "devops-tasks.json", + "wiki": "wiki-tasks.json", + } + tasks_file = TESTS_DIR / cat_to_file.get(category, f"{category}-tasks.json") + if not tasks_file.exists(): + print(f"ERROR: no tasks file for category '{category}': {tasks_file}") + return 1 + + cmd = [sys.executable, str(SKILL_DIR / "run_sleep.py")] + if dry_run: + cmd.append("--dry-run") + cmd.extend(["--tasks", str(tasks_file)]) + + print(f"=== /sleep run {category}{' (dry-run)' if dry_run else ''} ===") + print(f" cmd: {' '.join(cmd)}") + rc = os.system(" ".join(f'"{c}"' for c in cmd)) + return rc + + +def run_all(*, dry_run: bool = False) -> int: + rc = 0 + for cat in ("research-cron", "devops", "wiki"): + r = run_category(cat, dry_run=dry_run) + if r != 0: + rc = r + return rc + + +def adopt(night: str = None) -> int: + state_dir = _resolve_state_dir() + state_file = state_dir / "state.json" + if not state_file.exists(): + print("ERROR: no state to adopt from") + return 1 + with open(state_file) as f: + state = json.load(f) + nights = state.get("history") or state.get("nights", []) + if not nights: + print("ERROR: no nights recorded") + return 1 + + target = None + if night: + target = next((n for n in nights if str(n.get("night")) == night), None) + if not target: + print(f"ERROR: night '{night}' not found") + return 1 + else: + # most recent accepted + candidates = [n for n in nights if n.get("accepted") and n.get("staging")] + if not candidates: + print("ERROR: no accepted nights with staging to adopt") + return 1 + target = candidates[-1] + + staging = target["staging"] + if not os.path.isdir(staging): + print(f"ERROR: staging dir missing: {staging}") + return 1 + + print(f"=== /sleep adopt night {target['night']} ===") + print(f" staging: {staging}") + print(f" baseline: {target.get('baseline'):.3f} candidate: {target.get('candidate'):.3f}") + + # Read proposed skill from staging + manifest = Path(staging) / "manifest.json" + if manifest.exists(): + with open(manifest) as f: + m = json.load(f) + proposed = m.get("proposed_skill") + if proposed and Path(proposed).exists(): + live = STATE_DIR / "live_skill.md" + backup = STATE_DIR / f"live_skill.md.bak-{target['night']}" + if live.exists(): + shutil.copy2(live, backup) + print(f" backed up current live skill → {backup}") + shutil.copy2(proposed, live) + print(f" adopted proposed skill → {live}") + print() + print("✅ Adoption complete. Next cycle will use the new skill.") + return 0 + + print("ERROR: no proposed_skill in manifest") + return 1 + + +def reject(night: str = None) -> int: + state_dir = _resolve_state_dir() + state_file = state_dir / "state.json" + if not state_file.exists(): + print("ERROR: no state") + return 1 + with open(state_file) as f: + state = json.load(f) + nights = state.get("history") or state.get("nights", []) + target = None + if night: + target = next((n for n in nights if str(n.get("night")) == night), None) + else: + candidates = [n for n in reversed(nights) if n.get("staging")] + target = candidates[0] if candidates else None + + if not target or not target.get("staging"): + print("ERROR: nothing to reject") + return 1 + + staging = target["staging"] + if os.path.isdir(staging): + shutil.rmtree(staging) + print(f"🗑️ Removed staging: {staging}") + # remove from state + state["history"] = [n for n in nights if n.get("night") != target["night"]] + with open(state_file, "w") as f: + json.dump(state, f, indent=2) + print("✅ Rejected. State updated.") + return 0 + + +def schedule_cmd(hour: int, minute: int) -> int: + """Install a nightly cron entry via the shared SkillOpt-Sleep scheduler. + + Note: this schedules the shared engine (``python -m skillopt_sleep run``), + not the OpenClaw-specific ``run_sleep.py``. Use ``run_sleep_cron.sh`` if + you need the OpenClaw-native backend and category task files instead. + """ + try: + from skillopt_sleep.scheduler import schedule + except ImportError: + print("ERROR: skillopt_sleep.scheduler not available — is SkillOpt-Sleep installed?") + return 1 + project = str(SKILL_DIR) + ok, msg = schedule(project, hour=hour, minute=minute) + print(msg) + return 0 if ok else 1 + + +def unschedule_cmd(all_projects: bool) -> int: + """Remove cron entry via the shared SkillOpt-Sleep scheduler.""" + try: + from skillopt_sleep.scheduler import unschedule + except ImportError: + print("ERROR: skillopt_sleep.scheduler not available — is SkillOpt-Sleep installed?") + return 1 + project = str(SKILL_DIR) + ok, msg = unschedule(project, all_projects=all_projects) + print(msg) + return 0 if ok else 1 + + +def cost() -> int: + """Estimate per-night cost based on the actual measurement from Phase 2. + + From the real dry-run: 5 devops tasks used 14,427 tokens total. + That is ~2,885 tokens per task (all 3 phases combined). + """ + cfg_path = SKILL_DIR / "config.json" + cfg = {} + if cfg_path.exists(): + cfg = json.loads(cfg_path.read_text()) + cfg.pop("_comment", None) + + max_tasks = cfg.get("max_tasks_per_night", 12) + model = cfg.get("model", "deepseek-v4-pro") + # DeepSeek V4 pricing + if "pro" in model: + cost_in = 0.435 # per 1M + cost_out = 0.87 + elif "flash" in model: + cost_in = 0.14 + cost_out = 0.28 + else: + cost_in, cost_out = 0.5, 1.0 + + # Measured: ~2,900 tokens per task, 30% output / 70% input + toks_per_task = 2900 + input_toks = int(toks_per_task * 0.7) + output_toks = int(toks_per_task * 0.3) + + cost_in_total = (input_toks * max_tasks / 1_000_000) * cost_in + cost_out_total = (output_toks * max_tasks / 1_000_000) * cost_out + cost = cost_in_total + cost_out_total + + print(f"=== Cost estimate (per actual measurement) ===") + print(f" model: {model}") + print(f" max tasks/night: {max_tasks}") + print(f" ~tokens/night: {toks_per_task * max_tasks:,}") + print(f" cost/night: ${cost:.3f}") + print(f" cost/month (30 nights): ${cost*30:.2f}") + print(f" cost/year (365 nights): ${cost*365:.2f}") + return 0 + + +def main(): + ap = argparse.ArgumentParser(description="OpenClaw /sleep command") + sub = ap.add_subparsers(dest="cmd", required=True) + + sub.add_parser("status", help="show state + last 5 nights") + p_run = sub.add_parser("run", help="trigger one cycle") + p_run.add_argument("category", nargs="?", default=None, + choices=["research-cron", "devops", "wiki", None]) + p_run.add_argument("--dry-run", action="store_true") + sub.add_parser("dry-run", help="report-only cycle (all categories)") + p_adopt = sub.add_parser("adopt", help="adopt most recent accepted staging") + p_adopt.add_argument("night", nargs="?", default=None) + p_reject = sub.add_parser("reject", help="discard most recent staging") + p_reject.add_argument("night", nargs="?", default=None) + sub.add_parser("cost", help="estimate cost") + p_schedule = sub.add_parser("schedule", help="install nightly cron entry") + p_schedule.add_argument("--hour", type=int, default=3, help="hour (0-23)") + p_schedule.add_argument("--minute", type=int, default=17, help="minute (0-59)") + p_unschedule = sub.add_parser("unschedule", help="remove cron entry") + p_unschedule.add_argument("--all", dest="all_projects", action="store_true", + help="remove entries for all projects") + + args = ap.parse_args() + + if args.cmd == "status": + return status() + if args.cmd == "run": + if args.category: + return run_category(args.category, dry_run=args.dry_run) + return run_all(dry_run=args.dry_run) + if args.cmd == "dry-run": + return run_all(dry_run=True) + if args.cmd == "adopt": + return adopt(args.night) + if args.cmd == "reject": + return reject(args.night) + if args.cmd == "cost": + return cost() + if args.cmd == "schedule": + return schedule_cmd(args.hour, args.minute) + if args.cmd == "unschedule": + return unschedule_cmd(args.all_projects) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/plugins/openclaw/tests/devops-tasks.json b/plugins/openclaw/tests/devops-tasks.json new file mode 100644 index 0000000..678c57f --- /dev/null +++ b/plugins/openclaw/tests/devops-tasks.json @@ -0,0 +1,87 @@ +[ + { + "id": "do-01", + "reference": "[STATUS] devops-agent | Site Uptime \u2192 geoxylia.com OK (200) | 14/06 22:30 MYT", + "rubric": "Score 1.0 if output matches the exact format [STATUS] devops-agent | Site Uptime \u2192 geoxylia.com OK (200) | DD/MM HH:MM MYT, with a real current time. Score 0.5 if format is close but missing one field. Score 0.0 if wrong format or hallucinated values.", + "project": "devops-infrastructure-check", + "intent": "Site Uptime check. Run: `curl -o /dev/null -s -w '%{http_code}' https://geoxylia.com`. Interpret the result 200, and report in our standard format: 'STATUS | TASK \u2192 RESULT | TIME'. If not 200, escalate.", + "context_excerpt": "", + "attempted_solution": "", + "outcome": "unknown", + "reference_kind": "rubric", + "judge": {}, + "tags": [ + "devops-infrastructure-check" + ], + "source_sessions": [], + "split": "val" + }, + { + "id": "do-02", + "reference": "Backup complete. Files: 87, Size: 1.2G, Last: 2026-06-14 22:00:00 MYT", + "rubric": "Score 1.0 if output includes the exact 'Backup complete. Files: N, Size: X, Last: timestamp' structure with plausible values. Score 0.5 if structure is close but one field missing. Score 0.0 if hallucinated or wrong structure.", + "project": "devops-infrastructure-check", + "intent": "Daily Memory Backup. Confirm this ran successfully by checking: `ls -t ~/backups/memory/memory-backup-*.tar.gz | head -3`. Report the file count, total size, and most recent backup time. Use format: 'Backup complete. Files: [N], Size: [X], Last: [timestamp]'.", + "context_excerpt": "", + "attempted_solution": "", + "outcome": "unknown", + "reference_kind": "rubric", + "judge": {}, + "tags": [ + "devops-infrastructure-check" + ], + "source_sessions": [], + "split": "val" + }, + { + "id": "do-03", + "reference": "1) Vercel CSP missing frame-ancestors: MEDIUM. Allows clickjacking if anyone embeds our pages; not exploitable for our content, but best-practice gap.\n2) OpenClaw plaintext API keys: LOW. The config is chmod 600, loopback-only, not in git. Standard OpenClaw behavior. Rotating would add zero real security given current exposure.", + "rubric": "Score 1.0 if both are classified correctly (MEDIUM and LOW respectively) and justifications are accurate (not panicky, not dismissive). Score 0.5 if classifications are wrong by one tier or justifications are weak. Score 0.0 if both over-classified as CRITICAL or both wrong.", + "project": "devops-infrastructure-check", + "intent": "Security Check daily run. Two findings: 1) Vercel CSP header missing 'frame-ancestors' directive, 2) OpenClaw config has 3 plaintext API keys. Classify each as: CRITICAL / HIGH / MEDIUM / LOW / INFO. Justify each in 1 sentence.", + "context_excerpt": "", + "attempted_solution": "", + "outcome": "unknown", + "reference_kind": "rubric", + "judge": {}, + "tags": [ + "devops-infrastructure-check" + ], + "source_sessions": [], + "split": "train" + }, + { + "id": "do-04", + "reference": "[INCIDENT] supabase.audit_results: anon role has no RLS policy \u2014 anyone with the URL can read all audit results. Fix: add policy 'audit_results_select_own' granting SELECT WHERE user_id = auth.uid(). Severity: HIGH (data exposure). Estimated 2-min fix.", + "rubric": "Score 1.0 if: (a) severity correctly identified as HIGH, (b) fix is a real RLS policy (not just 'enable RLS' since it's already enabled), (c) under 50 words, (d) Telegram-friendly format. Score 0.5 if severity right but fix is generic. Score 0.0 if missing severity or wrong fix.", + "project": "devops-infrastructure-check", + "intent": "Incident Check. The Supabase RLS check returned: 'table public.audit_results: rls enabled but policy missing for anon role'. Interpret severity, propose fix, and format as a Telegram alert (max 50 words).", + "context_excerpt": "", + "attempted_solution": "", + "outcome": "unknown", + "reference_kind": "rubric", + "judge": {}, + "tags": [ + "devops-infrastructure-check" + ], + "source_sessions": [], + "split": "val" + }, + { + "id": "do-05", + "reference": "\ud83d\udee1\ufe0f Week security digest:\n\n\u2022 0 critical incidents, 1 high resolved (Supabase RLS policy added)\n\u2022 22 plaintext secrets: expected OpenClaw behavior, no action\n\u2022 1 medium open: Vercel CSP frame-ancestors, schedule for next sprint\n\nTrend: stable. No regressions vs last week.", + "rubric": "Score 1.0 if all 3 priority tiers mentioned with correct counts, ends with a trend statement, Telegram-friendly. Score 0.5 if structure is right but one tier wrong. Score 0.0 if missing a tier or wrong format.", + "project": "devops-infrastructure-check", + "intent": "Weekly security digest. Synthesize this week's findings: 22 plaintext secrets in openclaw.json (expected), 0 critical incidents, 1 high (Supabase RLS), 1 medium (CSP frame-ancestors), 0 low. Output a 3-bullet Telegram status.", + "context_excerpt": "", + "attempted_solution": "", + "outcome": "unknown", + "reference_kind": "rubric", + "judge": {}, + "tags": [ + "devops-infrastructure-check" + ], + "source_sessions": [], + "split": "train" + } +] \ No newline at end of file diff --git a/plugins/openclaw/tests/research-cron-tasks.json b/plugins/openclaw/tests/research-cron-tasks.json new file mode 100644 index 0000000..503bd50 --- /dev/null +++ b/plugins/openclaw/tests/research-cron-tasks.json @@ -0,0 +1,87 @@ +[ + { + "id": "rc-01", + "reference": "COMPETITOR MOVES: Otterly adds Perplexity tracker, joining Profound and LLMRefs in multi-platform citations.\nBACKLINK OPPORTUNITIES: 3 SEO directories (G2, Capterra, GetApp) have not been claimed.\nAGENCY BLUEPRINT: Top 2 agency sites bundle GEO audit + content refresh as $3K/mo tier.\nACTION ITEMS: Build Perplexity citation test into GeoXylia audit; claim G2 listing by Friday.", + "rubric": "Score 1.0 if all 4 section headings present in correct order, each with a substantive (not generic) 1-sentence content. Score 0.5 if headings present but content is generic. Score 0.0 if any heading missing or order wrong.", + "project": "research-cron-output", + "intent": "Weekly Competitive Deep Dive for GeoXylia. The competitor otterly.ai just added a Perplexity citation tracker. Produce the report header (top section) in our standard format: COMPETITOR MOVES, BACKLINK OPPORTUNITIES, AGENCY BLUEPRINT, ACTION ITEMS. Keep it to 4 lines, one per section heading with a 1-sentence placeholder.", + "context_excerpt": "", + "attempted_solution": "", + "outcome": "unknown", + "reference_kind": "rubric", + "judge": {}, + "tags": [ + "research-cron-output" + ], + "source_sessions": [], + "split": "train" + }, + { + "id": "rc-02", + "reference": "1. 'ai seo audit tool': 420 imp, pos 8.2, on page 1 \u2014 needs CTR lift (snippet/schema).\n2. 'geo audit tool': 230 imp, pos 12.5, page 2 \u2014 target blog post could push to page 1.\n3. 'llm optimization': 85 imp, pos 18.3, deep page-2 \u2014 fresh content with answer capsule could compete.", + "rubric": "Score 1.0 if the response correctly identifies 'ai seo audit tool', 'geo audit tool', and 'llm optimization' as the top 3 (NOT 'best free seo audit' which is already converting well, NOT 'free audit tool' which has too few impressions). Each must have correct impression count, position, and a substantive rationale. Score 0.5 if correct 3 keywords but rationale is weak. Score 0.0 if wrong keywords selected.", + "project": "research-cron-output", + "intent": "GSC keyword opportunity scan. From this snippet of GSC data, identify the top 3 keyword opportunities (high impressions, low CTR, position 5-15):\n\n1. 'ai seo audit tool' \u2014 420 imp, 12 clicks, pos 8.2\n2. 'best free seo audit' \u2014 1100 imp, 95 clicks, pos 4.1\n3. 'geo audit tool' \u2014 230 imp, 4 clicks, pos 12.5\n4. 'llm optimization' \u2014 85 imp, 1 click, pos 18.3\n5. 'free audit tool' \u2014 50 imp, 0 clicks, pos 22.0\n\nOutput: one line per opportunity, format 'KEYWORD: impressions, position, why-it-matters (1 short clause)'.", + "context_excerpt": "", + "attempted_solution": "", + "outcome": "unknown", + "reference_kind": "rubric", + "judge": {}, + "tags": [ + "research-cron-output" + ], + "source_sessions": [], + "split": "train" + }, + { + "id": "rc-03", + "reference": "Google AI Overviews now show source links more prominently + author bylines. For GeoXylia: this favors pages with clear authorship (add author schema to blog posts). Action: this week, add author + E-E-A-T schema markup to top 10 blog posts. Source: Google Search Central blog.", + "rubric": "Score 1.0 if: (a) under 60 words, (b) names the change, (c) gives GeoXylia-specific implication, (d) gives a concrete action item, (e) cites the source. Score 0.5 if missing 1-2 of these. Score 0.0 if over 60 words or missing 3+.", + "project": "research-cron-output", + "intent": "Daily Industry News scan. The Google Search Central blog just announced: 'AI Overviews now showing source links more prominently, with author bylines for E-E-A-T-heavy content.' Write a 1-paragraph Telegram alert (max 60 words) for Ethan. Include: 1) what changed, 2) what it means for GeoXylia, 3) any action item.", + "context_excerpt": "", + "attempted_solution": "", + "outcome": "unknown", + "reference_kind": "rubric", + "judge": {}, + "tags": [ + "research-cron-output" + ], + "source_sessions": [], + "split": "val" + }, + { + "id": "rc-04", + "reference": "Hi [Name], I saw seo-skill.com's resources page is one of the most-respected SEO learning hubs in the industry \u2014 your 2026 algorithm breakdown was spot-on. We just published a free 2026 AI SEO Audit comparison that your readers would find genuinely useful (no paywall, no signup). It covers the 8 leading AI-audit tools with hands-on screenshots and a clear feature matrix. GeoXylia is the only fully-free option in the comparison, so it's a natural fit for a 'tools to know' section. Mind if I share the link for inclusion?", + "rubric": "Score 1.0 if exactly 4 sentences, all four functional pieces present (compliment / mention resource / audience benefit / GeoXylia one-liner), conversational tone, no aggressive sales language. Score 0.5 if 3 of 4 pieces present or tone is too salesy. Score 0.0 if more than 5 sentences or missing 2+ pieces.", + "project": "research-cron-output", + "intent": "Backlink Outreach draft for the blog post 'Free AI SEO Audit Tool: 2026 Comparison'. The prospect is seo-skill.com (a popular SEO training site with a 'resources' page). Write a 4-sentence outreach email: 1) compliment, 2) mention our resource, 3) explain audience benefit, 4) one-line about GeoXylia.", + "context_excerpt": "", + "attempted_solution": "", + "outcome": "unknown", + "reference_kind": "rubric", + "judge": {}, + "tags": [ + "research-cron-output" + ], + "source_sessions": [], + "split": "train" + }, + { + "id": "rc-05", + "reference": "1) DO MORE: AI citation / LLM-mention topics \u2014 the 0.9% CTR at position 9.4 means we're visible but need richer answer capsules to lift CTR. Target 2x posts/week on this cluster.\n2) PAUSE: Pure schema-markup how-tos \u2014 'Schema Markup for SEO' has 0 clicks at position 41, the audience isn't searching this way. Rework as 'How to appear in AI answers' framing.\n3) TEST: 'Perplexity vs ChatGPT citation rates for [niche]' \u2014 unexplored angle, could capture comparison-intent traffic.", + "rubric": "Score 1.0 if all 3 are specific (not generic), cite actual data from the prompt, and contain a clear actionable change. Score 0.5 if 2 of 3 are specific. Score 0.0 if generic advice or no data citations.", + "project": "research-cron-output", + "intent": "Performance \u2192 Strategy feedback loop. Last week's top blog post was 'AI Citation Audit: Does Your Site Appear in ChatGPT?' with 4,200 impressions and 38 clicks (CTR 0.9%, position 9.4). The bottom post was 'Schema Markup for SEO: A 2026 Guide' with 110 impressions and 0 clicks (CTR 0%, position 41). Write 3 specific strategy adjustments: 1) what to do more of, 2) what to pause, 3) what new topic to test.", + "context_excerpt": "", + "attempted_solution": "", + "outcome": "unknown", + "reference_kind": "rubric", + "judge": {}, + "tags": [ + "research-cron-output" + ], + "source_sessions": [], + "split": "val" + } +] \ No newline at end of file diff --git a/plugins/openclaw/tests/wiki-tasks.json b/plugins/openclaw/tests/wiki-tasks.json new file mode 100644 index 0000000..544ed88 --- /dev/null +++ b/plugins/openclaw/tests/wiki-tasks.json @@ -0,0 +1,70 @@ +[ + { + "id": "wk-01", + "reference": "1. What GEO is and isn't (define vs SEO/AEO, dispel the 'just add FAQ' myth)\n2. The 3 citation mechanisms LLMs use (RAG, fine-tuning, in-context; weight each)\n3. The 2026 citation data (real statistics from Profound/Otterly/Peec; what % of queries get citations)\n4. The action framework (a 5-step audit-and-fix process, concrete)\n5. Measurement (which metrics actually predict citation lift; vanity vs real)", + "rubric": "Score 1.0 if 5 sections, in a logical order, each with a substantive (not generic) purpose, and the section content is GEO-specific (not generic SEO). Score 0.5 if 5 sections but 1-2 are generic. Score 0.0 if wrong number of sections or wrong order.", + "project": "wiki-canonical-guide", + "intent": "Wiki canonical guide: 'GEO 2026 Standards'. Audience: a mid-level SEO specialist who has heard of GEO but not done it. Tone: technical, evidence-driven, no fluff. Length target: 1500-2200 words. Outline the 5 sections that should appear in order. For each, give a 1-sentence sub-purpose.", + "context_excerpt": "", + "attempted_solution": "", + "outcome": "unknown", + "reference_kind": "rubric", + "judge": {}, + "tags": [ + "wiki-canonical-guide" + ], + "source_sessions": [], + "split": "val" + }, + { + "id": "wk-02", + "reference": "Yes, add inbound links. (1) geo-2026-standards.md \u2192 '## Action Framework' section, anchor: 'platform-specific citation rules' \u2014 natural since GEO standards reference ChatGPT/Perplexity behavior. (2) seo-2026-standards.md \u2192 '## AI Overviews' section, anchor: 'AI platform citations' \u2014 links to the mechanism guide. (3) content-strategy.md \u2192 '## Content Types' section, anchor: 'per-platform citation' \u2014 content strategy needs to know which platform favors which content.", + "rubric": "Score 1.0 if all 3 inbound links proposed with specific section + natural anchor text, demonstrating the link solves a real navigational gap (not just SEO-link-building). Score 0.5 if 2 of 3 are well-placed. Score 0.0 if generic anchors like 'click here' or no specific sections named.", + "project": "wiki-canonical-guide", + "intent": "Cross-link audit. The wiki page 'ai-platform-citation-guide.md' has 4 outbound links to other wiki pages, but no inbound links from: 'geo-2026-standards.md', 'seo-2026-standards.md', 'content-strategy.md'. Should we add inbound links? In which page should each inbound link go, and what anchor text would be natural?", + "context_excerpt": "", + "attempted_solution": "", + "outcome": "unknown", + "reference_kind": "rubric", + "judge": {}, + "tags": [ + "wiki-canonical-guide" + ], + "source_sessions": [], + "split": "val" + }, + { + "id": "wk-03", + "reference": "Priorities:\n1. Refresh 'geo-glossary.md' (last update 2026-04-12, 63 days) \u2014 add new terms like RAG, in-context citation, agentic SEO.\n2. Refresh 'competitor-pricing.md' (last update 2026-05-01, 44 days) \u2014 Profound raised enterprise tier.\n3. No structural fixes needed.\n\nTelegram: 'Wiki lint: 2 stale pages flagged (geo-glossary 63d, competitor-pricing 44d). No broken links. Both need refresh this week.'", + "rubric": "Score 1.0 if both stale pages correctly identified with specific (not generic) refresh notes, and Telegram summary is under 40 words with the right action. Score 0.5 if stale pages identified but refresh notes are vague. Score 0.0 if missing stale pages or Telegram over 40 words.", + "project": "wiki-canonical-guide", + "intent": "Wiki lint report. Today's scan: 14 wiki pages, 2 with 'Updated' dates > 30 days old ('geo-glossary.md' and 'competitor-pricing.md'), 0 broken internal links, 0 missing YAML frontmatter. Output: 1) prioritized action list, 2) Telegram summary (max 40 words).", + "context_excerpt": "", + "attempted_solution": "", + "outcome": "unknown", + "reference_kind": "rubric", + "judge": {}, + "tags": [ + "wiki-canonical-guide" + ], + "source_sessions": [], + "split": "train" + }, + { + "id": "wk-04", + "reference": "Index rebuilt: 14 wiki pages registered in _index.md (was 12 \u2014 added competitor-pricing-rev2 and citations-q2-2026).\nQuestion for Ethan: should 'competitor-pricing.md' and 'competitor-pricing-rev2.md' be merged? They're 78% similar in content.", + "rubric": "Score 1.0 if both sentences are accurate (count matches, names are plausible) and the question identifies a real consolidation opportunity (not a fabricated one). Score 0.5 if structure is right but content vague. Score 0.0 if wrong format or no question.", + "project": "wiki-canonical-guide", + "intent": "Index rebuild check. Run `python3 ~/agent-shared/scripts/update-index.py` (assume it works). After the run, the new wiki/_index.md should list all 14 pages. Generate a 2-sentence confirmation message + 1 question for Ethan to verify.", + "context_excerpt": "", + "attempted_solution": "", + "outcome": "unknown", + "reference_kind": "rubric", + "judge": {}, + "tags": [ + "wiki-canonical-guide" + ], + "source_sessions": [], + "split": "train" + } +] \ No newline at end of file diff --git a/plugins/run-sleep.sh b/plugins/run-sleep.sh new file mode 100755 index 0000000..310d8de --- /dev/null +++ b/plugins/run-sleep.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# SkillOpt-Sleep shared runner — used by all platform plugins (Claude Code, +# Codex, Copilot). Resolves the repo root (which contains the skillopt_sleep +# package), picks a Python >= 3.10, and execs the engine CLI. +# +# Usage: run-sleep.sh [args...] +set -euo pipefail + +# This script lives at /plugins/run-sleep.sh, so the repo root (which +# holds skillopt_sleep/) is one level up. CLAUDE_PLUGIN_ROOT (if set by Claude +# Code) points at the plugin dir; the engine is then two levels above it. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [ -d "$SCRIPT_DIR/../skillopt_sleep" ]; then + REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +elif [ -n "${CLAUDE_PLUGIN_ROOT:-}" ] && [ -d "$CLAUDE_PLUGIN_ROOT/../../skillopt_sleep" ]; then + REPO_ROOT="$(cd "$CLAUDE_PLUGIN_ROOT/../.." && pwd)" +elif [ -n "${SKILLOPT_SLEEP_REPO:-}" ] && [ -d "$SKILLOPT_SLEEP_REPO/skillopt_sleep" ]; then + REPO_ROOT="$SKILLOPT_SLEEP_REPO" +else + # last resort: search upward from CWD + d="$PWD" + while [ "$d" != "/" ]; do + [ -d "$d/skillopt_sleep" ] && { REPO_ROOT="$d"; break; } + d="$(dirname "$d")" + done +fi +if [ -z "${REPO_ROOT:-}" ]; then + echo "[sleep] ERROR: could not locate the skillopt_sleep package. Set SKILLOPT_SLEEP_REPO to the repo root." >&2 + exit 1 +fi + +PY="" +# Allow explicit Python override (useful on macOS with old system Python). +if [ -n "${SKILLOPT_SLEEP_PYTHON:-}" ]; then + PY="$SKILLOPT_SLEEP_PYTHON" +else + for cand in python3.12 python3.11 python3.10 python3; do + if command -v "$cand" >/dev/null 2>&1; then + ver="$("$cand" -c 'import sys; print("%d%d" % sys.version_info[:2])' 2>/dev/null || echo 0)" + if [ "${ver:-0}" -ge 310 ]; then PY="$cand"; break; fi + fi + done +fi +if [ -z "$PY" ]; then + echo "[sleep] ERROR: need Python >= 3.10 (found none)." >&2 + exit 1 +fi + +if [ "$#" -eq 0 ]; then set -- status; fi +cd "$REPO_ROOT" +exec "$PY" -m skillopt_sleep "$@" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..69abfbf --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,82 @@ +[build-system] +requires = ["setuptools>=68.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "skillopt" +version = "0.2.0" +description = "SkillOpt: Agentic Skill Optimization via Reflective Training Loops" +readme = "README.md" +license = {text = "MIT"} +requires-python = ">=3.10" +authors = [ + {name = "SkillOpt Team"}, +] +keywords = ["agent", "prompt-optimization", "skill-learning", "LLM", "agentic"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] +dependencies = [ + "openai>=1.30.0", + "pyyaml>=6.0", + "numpy>=1.24.0", + "openpyxl>=3.1.0", + "azure-identity>=1.15.0", + "azure-core>=1.30.0", + "httpx>=0.27.0", +] + +[project.optional-dependencies] +# Benchmark-specific dependencies +alfworld = ["alfworld>=0.4.0", "gymnasium>=0.29.0"] +# Claude model backend +claude = ["claude-agent-sdk>=0.1.0", "json_repair>=0.61.0"] +# Qwen local model backend (via vLLM) +qwen = ["vllm>=0.4.0", "json_repair>=0.61.0"] +# SearchQA data materialization +searchqa = ["datasets>=2.18.0"] +# Documentation site +docs = ["mkdocs-material>=9.5.0", "mkdocstrings[python]>=0.24.0"] +# WebUI dashboard +webui = ["gradio>=4.0.0"] +# Development tools +dev = ["ruff>=0.4.0", "pytest>=8.0.0"] +# All optional dependencies (except docs/dev/webui) +all = [ + "alfworld>=0.4.0", + "gymnasium>=0.29.0", + "claude-agent-sdk>=0.1.0", + "json_repair>=0.61.0", +] + +[project.scripts] +skillopt-train = "scripts.train:main" +skillopt-eval = "scripts.eval_only:main" +skillopt-sleep = "skillopt_sleep.__main__:main" + +[project.urls] +Homepage = "https://github.com/microsoft/SkillOpt" +Documentation = "https://microsoft.github.io/SkillOpt" +Repository = "https://github.com/microsoft/SkillOpt" +Issues = "https://github.com/microsoft/SkillOpt/issues" + +[tool.setuptools.packages.find] +# skillopt* = the research package +# skillopt_sleep = the open-source Sleep tool (decoupled, zero research dep) +# skillopt_webui = the Gradio dashboard (installed via the `webui` extra) +include = ["skillopt", "skillopt.*", "skillopt_sleep", "skillopt_sleep.*", "skillopt_webui", "skillopt_webui.*", "scripts*"] + +[tool.ruff] +line-length = 120 +target-version = "py310" + +[tool.ruff.lint] +select = ["E", "F", "I", "W"] +ignore = ["E501"] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5db9e70 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,31 @@ +# ── Core ────────────────────────────────────────── +openai>=1.30.0 +pyyaml>=6.0 +numpy>=1.24.0 +openpyxl>=3.1.0 +azure-identity>=1.15.0 +azure-core>=1.30.0 +httpx>=0.27.0 + +# ── Optional: ALFWorld benchmark ────────────────── +# alfworld>=0.4.0 +# gymnasium>=0.29.0 + +# ── Optional: Claude model backend ──────────────── +# claude-agent-sdk>=0.1.0 + +# ── Optional: Qwen local model (via vLLM) ──────── +# vllm>=0.4.0 + +# ── Optional: tolerant JSON repair for free-form output from non-OpenAI +# backends (Claude/Qwen). Without it extract_json() falls back safely and +# drops a malformed analyst edit instead of repairing it. Installed by the +# `claude`, `qwen`, and `all` extras in pyproject.toml. +# json_repair>=0.61.0 + +# ── Optional: WebUI dashboard ──────────────────── +# gradio>=4.0.0 + +# ── Optional: Documentation site ───────────────── +# mkdocs-material>=9.5.0 +# mkdocstrings[python]>=0.24.0 diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/eval_only.py b/scripts/eval_only.py new file mode 100644 index 0000000..79dfab0 --- /dev/null +++ b/scripts/eval_only.py @@ -0,0 +1,501 @@ +#!/usr/bin/env python3 +"""SkillOpt eval-only: run a single skill on a dataset without training. + +Usage +----- + python scripts/eval_only.py \ + --config configs/spreadsheetbench/default.yaml \ + --skill skillopt/envs/spreadsheetbench/skills/initial.md \ + --split_dir /path/to/split \ + --out_root outputs/eval_skill0 + +All YAML keys can be overridden from the CLI, same as train.py. +""" +from __future__ import annotations + +import argparse +import datetime +import json +import os +import sys + +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +_PROJECT_ROOT = os.path.dirname(_SCRIPT_DIR) +if _PROJECT_ROOT not in sys.path: + sys.path.insert(0, _PROJECT_ROOT) + +from skillopt.model import ( + configure_azure_openai, + configure_claude_code_exec, + configure_codex_exec, + configure_qwen_chat, + configure_minimax_chat, + set_reasoning_effort, + set_target_backend, + set_target_deployment, + set_optimizer_backend, + set_optimizer_deployment, +) +from skillopt.model.common import default_model_for_backend, normalize_backend_name + +_OPENAI_DEFAULT_MODEL_SENTINELS = {"gpt-5.4", "gpt-5.5"} +from skillopt.utils import compute_score + + +# ── Reuse registry from train.py ─────────────────────────────────────────── + +_ENV_REGISTRY: dict[str, type] = {} + + +def _register_builtins() -> None: + try: + from skillopt.envs.alfworld.adapter import ALFWorldAdapter + _ENV_REGISTRY["alfworld"] = ALFWorldAdapter + except ImportError: + pass + try: + from skillopt.envs.searchqa.adapter import SearchQAAdapter + _ENV_REGISTRY["searchqa"] = SearchQAAdapter + except ImportError: + pass + try: + from skillopt.envs.livemathematicianbench.adapter import LiveMathematicianBenchAdapter + _ENV_REGISTRY["livemathematicianbench"] = LiveMathematicianBenchAdapter + except ImportError: + pass + try: + from skillopt.envs.babyvision.adapter import BabyVisionAdapter + _ENV_REGISTRY["babyvision"] = BabyVisionAdapter + except ImportError: + pass + try: + from skillopt.envs.spreadsheetbench.adapter import SpreadsheetBenchAdapter + _ENV_REGISTRY["spreadsheetbench"] = SpreadsheetBenchAdapter + except ImportError: + pass + try: + from skillopt.envs.mmrb.adapter import MMRBAdapter + _ENV_REGISTRY["mmrb"] = MMRBAdapter + except ImportError: + pass + try: + from skillopt.envs.docvqa.adapter import DocVQAAdapter + _ENV_REGISTRY["docvqa"] = DocVQAAdapter + except ImportError: + pass + try: + from skillopt.envs.mathverse.adapter import MathVerseAdapter + _ENV_REGISTRY["mathverse"] = MathVerseAdapter + except ImportError: + pass + try: + from skillopt.envs.officeqa.adapter import OfficeQAAdapter + _ENV_REGISTRY["officeqa"] = OfficeQAAdapter + except ImportError: + pass + try: + from skillopt.envs.sealqa.adapter import SealQAAdapter + _ENV_REGISTRY["sealqa"] = SealQAAdapter + except ImportError: + pass + try: + from skillopt.envs.swebench.adapter import SWEBenchAdapter + _ENV_REGISTRY["swebench"] = SWEBenchAdapter + except ImportError: + pass + + +def get_adapter(cfg: dict): + _register_builtins() + env_name = cfg.get("env", "alfworld") + if env_name not in _ENV_REGISTRY: + raise ValueError( + f"Unknown environment '{env_name}'. " + f"Available: {list(_ENV_REGISTRY.keys())}" + ) + adapter_cls = _ENV_REGISTRY[env_name] + + import inspect + sig = inspect.signature(adapter_cls.__init__) + accepted = set(sig.parameters.keys()) - {"self"} + adapter_kwargs = {k: cfg[k] for k in accepted if k in cfg} + return adapter_cls(**adapter_kwargs) + + +# ── CLI ──────────────────────────────────────────────────────────────────── + +_BOOL = lambda x: str(x).lower() in ("true", "1", "yes") # noqa: E731 + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="SkillOpt eval-only") + p.add_argument("--config", type=str, required=True) + p.add_argument("--skill", type=str, required=True, + help="Path to skill .md file to evaluate") + p.add_argument("--split", type=str, default="all", + help="Which split to eval: train/valid_seen/valid_unseen/all (default: all)") + p.add_argument("--cfg-options", nargs="+", default=[], + help="Override config: section.key=value") + # Legacy flat overrides + p.add_argument("--env", type=str) + p.add_argument("--backend", type=str, + choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "minimax", "minimax_chat"]) + p.add_argument("--optimizer_model", type=str) + p.add_argument("--target_model", type=str) + p.add_argument("--optimizer_backend", type=str) + p.add_argument("--target_backend", type=str) + p.add_argument("--reasoning_effort", type=str, + choices=["", "low", "medium", "high", "xhigh", "max"]) + p.add_argument("--azure_endpoint", type=str) + p.add_argument("--azure_api_version", type=str) + p.add_argument("--azure_api_key", type=str) + p.add_argument("--azure_openai_endpoint", type=str) + p.add_argument("--azure_openai_api_version", type=str) + p.add_argument("--azure_openai_api_key", type=str) + p.add_argument("--azure_openai_auth_mode", type=str) + p.add_argument("--azure_openai_ad_scope", type=str) + p.add_argument("--azure_openai_managed_identity_client_id", type=str) + p.add_argument("--optimizer_azure_openai_endpoint", type=str) + p.add_argument("--optimizer_azure_openai_api_version", type=str) + p.add_argument("--optimizer_azure_openai_api_key", type=str) + p.add_argument("--optimizer_azure_openai_auth_mode", type=str) + p.add_argument("--optimizer_azure_openai_ad_scope", type=str) + p.add_argument("--optimizer_azure_openai_managed_identity_client_id", type=str) + p.add_argument("--target_azure_openai_endpoint", type=str) + p.add_argument("--target_azure_openai_api_version", type=str) + p.add_argument("--target_azure_openai_api_key", type=str) + p.add_argument("--target_azure_openai_auth_mode", type=str) + p.add_argument("--target_azure_openai_ad_scope", type=str) + p.add_argument("--target_azure_openai_managed_identity_client_id", type=str) + p.add_argument("--codex_exec_path", type=str) + p.add_argument("--codex_exec_sandbox", type=str) + p.add_argument("--codex_exec_profile", type=str) + p.add_argument("--codex_exec_full_auto", type=_BOOL) + p.add_argument("--codex_exec_reasoning_effort", type=str) + p.add_argument("--codex_exec_use_sdk", type=str) + p.add_argument("--codex_exec_network_access", type=_BOOL) + p.add_argument("--codex_exec_web_search", type=_BOOL) + p.add_argument("--codex_exec_approval_policy", type=str) + p.add_argument("--claude_code_exec_path", type=str) + p.add_argument("--claude_code_exec_profile", type=str) + p.add_argument("--claude_code_exec_use_sdk", type=str) + p.add_argument("--claude_code_exec_effort", type=str) + p.add_argument("--claude_code_exec_max_thinking_tokens", type=int) + p.add_argument("--minimax_base_url", type=str) + p.add_argument("--minimax_api_key", type=str) + p.add_argument("--minimax_model", type=str) + p.add_argument("--minimax_temperature", type=float) + p.add_argument("--minimax_max_tokens", type=int) + p.add_argument("--minimax_enable_thinking", type=_BOOL) + p.add_argument("--out_root", type=str) + p.add_argument("--data_path", type=str) + p.add_argument("--split_mode", type=str, + choices=["ratio", "split_dir"]) + p.add_argument("--split_ratio", type=str) + p.add_argument("--split_seed", type=int) + p.add_argument("--split_dir", type=str) + p.add_argument("--split_output_dir", type=str) + p.add_argument("--data_root", type=str) + p.add_argument("--max_turns", type=int) + p.add_argument("--workers", type=int) + p.add_argument("--max_api_workers", type=int) + p.add_argument("--seed", type=int) + p.add_argument("--test_env_num", type=int) + p.add_argument("--mode", type=str, + help="SpreadsheetBench: single/multi/react (default comes from config)") + return p.parse_args() + + +def main() -> None: + args = parse_args() + + from skillopt.config import load_config as _load, flatten_config, is_structured + + cfg = _load(args.config, overrides=args.cfg_options) + structured = is_structured(cfg) + + # Apply legacy --key value overrides + cli = {k: v for k, v in vars(args).items() + if v is not None and k not in ("config", "skill", "split", "cfg_options")} + if cli: + if structured: + from skillopt.config import apply_overrides + _MAP = { + "backend": "model.backend", + "optimizer_model": "model.optimizer", + "target_model": "model.target", + "optimizer_backend": "model.optimizer_backend", + "target_backend": "model.target_backend", + "reasoning_effort": "model.reasoning_effort", + "azure_endpoint": "model.azure_endpoint", + "azure_api_version": "model.azure_api_version", + "azure_api_key": "model.azure_api_key", + "azure_openai_endpoint": "model.azure_openai_endpoint", + "azure_openai_api_version": "model.azure_openai_api_version", + "azure_openai_api_key": "model.azure_openai_api_key", + "azure_openai_auth_mode": "model.azure_openai_auth_mode", + "azure_openai_ad_scope": "model.azure_openai_ad_scope", + "azure_openai_managed_identity_client_id": "model.azure_openai_managed_identity_client_id", + "optimizer_azure_openai_endpoint": "model.optimizer_azure_openai_endpoint", + "optimizer_azure_openai_api_version": "model.optimizer_azure_openai_api_version", + "optimizer_azure_openai_api_key": "model.optimizer_azure_openai_api_key", + "optimizer_azure_openai_auth_mode": "model.optimizer_azure_openai_auth_mode", + "optimizer_azure_openai_ad_scope": "model.optimizer_azure_openai_ad_scope", + "optimizer_azure_openai_managed_identity_client_id": "model.optimizer_azure_openai_managed_identity_client_id", + "target_azure_openai_endpoint": "model.target_azure_openai_endpoint", + "target_azure_openai_api_version": "model.target_azure_openai_api_version", + "target_azure_openai_api_key": "model.target_azure_openai_api_key", + "target_azure_openai_auth_mode": "model.target_azure_openai_auth_mode", + "target_azure_openai_ad_scope": "model.target_azure_openai_ad_scope", + "target_azure_openai_managed_identity_client_id": "model.target_azure_openai_managed_identity_client_id", + "codex_exec_path": "model.codex_exec_path", + "codex_exec_sandbox": "model.codex_exec_sandbox", + "codex_exec_profile": "model.codex_exec_profile", + "codex_exec_full_auto": "model.codex_exec_full_auto", + "codex_exec_reasoning_effort": "model.codex_exec_reasoning_effort", + "codex_exec_use_sdk": "model.codex_exec_use_sdk", + "codex_exec_network_access": "model.codex_exec_network_access", + "codex_exec_web_search": "model.codex_exec_web_search", + "codex_exec_approval_policy": "model.codex_exec_approval_policy", + "claude_code_exec_path": "model.claude_code_exec_path", + "claude_code_exec_profile": "model.claude_code_exec_profile", + "claude_code_exec_use_sdk": "model.claude_code_exec_use_sdk", + "claude_code_exec_effort": "model.claude_code_exec_effort", + "claude_code_exec_max_thinking_tokens": "model.claude_code_exec_max_thinking_tokens", + "minimax_base_url": "model.minimax_base_url", + "minimax_api_key": "model.minimax_api_key", + "minimax_model": "model.minimax_model", + "minimax_temperature": "model.minimax_temperature", + "minimax_max_tokens": "model.minimax_max_tokens", + "minimax_enable_thinking": "model.minimax_enable_thinking", + "seed": "train.seed", + "test_env_num": "evaluation.test_env_num", + "env": "env.name", + "out_root": "env.out_root", + } + mapped = [] + for k, v in cli.items(): + dotted = _MAP.get(k) + if dotted: + mapped.append(f"{dotted}={v}") + else: + mapped.append(f"env.{k}={v}") + apply_overrides(cfg, mapped) + else: + cfg.update(cli) + + cfg = flatten_config(cfg) if structured else cfg + + for new_key, old_key in ( + ("azure_openai_endpoint", "azure_endpoint"), + ("azure_openai_api_version", "azure_api_version"), + ("azure_openai_api_key", "azure_api_key"), + ): + if cfg.get(new_key) in (None, "") and cfg.get(old_key) not in (None, ""): + cfg[new_key] = cfg[old_key] + + explicit_backend = getattr(args, "backend", None) + if explicit_backend is None: + for option in args.cfg_options or []: + key = str(option).split("=", 1)[0].strip() + if key == "model.backend": + explicit_backend = str(option).split("=", 1)[1].strip() + break + + backend = normalize_backend_name(cfg.get("model_backend") or cfg.get("target_backend") or "azure_openai") + + def _has_model_override(dotted_key: str, legacy_key: str) -> bool: + if getattr(args, legacy_key, None) is not None: + return True + for option in args.cfg_options or []: + key = str(option).split("=", 1)[0].strip() + if key == dotted_key: + return True + return False + + if explicit_backend is not None: + backend = normalize_backend_name(explicit_backend) + cfg["model_backend"] = backend + if backend in {"claude", "claude_chat"}: + cfg.setdefault("optimizer_backend", "claude_chat") + cfg.setdefault("target_backend", "claude_chat") + elif backend in {"codex", "codex_exec"}: + cfg.setdefault("optimizer_backend", "openai_chat") + cfg.setdefault("target_backend", "codex_exec") + elif backend == "claude_code_exec": + cfg.setdefault("optimizer_backend", "openai_chat") + cfg.setdefault("target_backend", "claude_code_exec") + elif backend in {"minimax", "minimax_chat"}: + cfg.setdefault("optimizer_backend", "openai_chat") + cfg.setdefault("target_backend", "minimax_chat") + else: + cfg.setdefault("optimizer_backend", "openai_chat") + cfg.setdefault("target_backend", "openai_chat") + else: + cfg.setdefault("optimizer_backend", "openai_chat") + cfg.setdefault("target_backend", "openai_chat") + + if cfg.get("optimizer_backend") == "claude_chat": + if ( + str(cfg.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.optimizer", "optimizer_model") + ): + cfg["optimizer_model"] = default_model_for_backend("claude_chat") + if cfg.get("target_backend") == "claude_chat": + if ( + str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + cfg["target_model"] = default_model_for_backend("claude_chat") + if cfg.get("target_backend") == "claude_code_exec": + if ( + str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + cfg["target_model"] = default_model_for_backend("claude_chat") + if cfg.get("target_backend") == "minimax_chat": + if ( + str(cfg.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + cfg["target_model"] = ( + cfg.get("minimax_model") + or default_model_for_backend("minimax_chat") + ) + + if not cfg.get("out_root"): + env = cfg.get("env", "unknown") + model = cfg.get("target_model", "unknown").replace("/", "-") + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + cfg["out_root"] = os.path.join("outputs", f"eval_{env}_{model}_{ts}") + + cfg["out_root"] = os.path.abspath(cfg["out_root"]) + + out_root = cfg["out_root"] + os.makedirs(out_root, exist_ok=True) + + # Load skill + skill_path = os.path.abspath(args.skill) + with open(skill_path) as f: + skill_content = f.read() + print(f" [skill] {skill_path} ({len(skill_content)} chars)") + + # Configure models + configure_azure_openai( + endpoint=(cfg.get("azure_openai_endpoint") or cfg.get("azure_endpoint") or None), + api_version=(cfg.get("azure_openai_api_version") or cfg.get("azure_api_version") or None), + api_key=(cfg.get("azure_openai_api_key") or cfg.get("azure_api_key") or None), + auth_mode=cfg.get("azure_openai_auth_mode") or None, + ad_scope=cfg.get("azure_openai_ad_scope") or None, + managed_identity_client_id=cfg.get("azure_openai_managed_identity_client_id") or None, + optimizer_endpoint=cfg.get("optimizer_azure_openai_endpoint") or None, + optimizer_api_version=cfg.get("optimizer_azure_openai_api_version") or None, + optimizer_api_key=cfg.get("optimizer_azure_openai_api_key") or None, + optimizer_auth_mode=cfg.get("optimizer_azure_openai_auth_mode") or None, + optimizer_ad_scope=cfg.get("optimizer_azure_openai_ad_scope") or None, + optimizer_managed_identity_client_id=( + cfg.get("optimizer_azure_openai_managed_identity_client_id") or None + ), + target_endpoint=cfg.get("target_azure_openai_endpoint") or None, + target_api_version=cfg.get("target_azure_openai_api_version") or None, + target_api_key=cfg.get("target_azure_openai_api_key") or None, + target_auth_mode=cfg.get("target_azure_openai_auth_mode") or None, + target_ad_scope=cfg.get("target_azure_openai_ad_scope") or None, + target_managed_identity_client_id=( + cfg.get("target_azure_openai_managed_identity_client_id") or None + ), + ) + set_optimizer_backend(cfg.get("optimizer_backend", "openai_chat")) + set_target_backend(cfg.get("target_backend", "openai_chat")) + set_optimizer_deployment(cfg.get("optimizer_model", default_model_for_backend(backend))) + set_target_deployment(cfg.get("target_model", default_model_for_backend(backend))) + configure_codex_exec( + path=cfg.get("codex_exec_path", "codex"), + sandbox=cfg.get("codex_exec_sandbox", "workspace-write"), + profile=cfg.get("codex_exec_profile", ""), + full_auto=cfg.get("codex_exec_full_auto", False), + reasoning_effort=cfg.get("codex_exec_reasoning_effort", "none"), + use_sdk=cfg.get("codex_exec_use_sdk", None), + network_access=cfg.get("codex_exec_network_access", False), + web_search=cfg.get("codex_exec_web_search", False), + approval_policy=cfg.get("codex_exec_approval_policy", "never"), + ) + configure_claude_code_exec( + path=cfg.get("claude_code_exec_path", "claude"), + profile=cfg.get("claude_code_exec_profile", ""), + use_sdk=cfg.get("claude_code_exec_use_sdk", None), + effort=cfg.get("claude_code_exec_effort", cfg.get("reasoning_effort", "medium")), + max_thinking_tokens=cfg.get("claude_code_exec_max_thinking_tokens", 16384), + ) + configure_qwen_chat( + base_url=cfg.get("qwen_chat_base_url") or None, + api_key=cfg.get("qwen_chat_api_key") or None, + temperature=cfg.get("qwen_chat_temperature"), + timeout_seconds=cfg.get("qwen_chat_timeout_seconds"), + max_tokens=cfg.get("qwen_chat_max_tokens"), + enable_thinking=cfg.get("qwen_chat_enable_thinking"), + target_base_url=cfg.get("target_qwen_chat_base_url") or None, + target_api_key=cfg.get("target_qwen_chat_api_key") or None, + target_temperature=cfg.get("target_qwen_chat_temperature"), + target_timeout_seconds=cfg.get("target_qwen_chat_timeout_seconds"), + target_max_tokens=cfg.get("target_qwen_chat_max_tokens"), + target_enable_thinking=cfg.get("target_qwen_chat_enable_thinking"), + ) + configure_minimax_chat( + base_url=cfg.get("minimax_base_url") or None, + api_key=cfg.get("minimax_api_key") or None, + temperature=cfg.get("minimax_temperature"), + max_tokens=cfg.get("minimax_max_tokens"), + enable_thinking=cfg.get("minimax_enable_thinking"), + ) + minimax_model_cfg = cfg.get("minimax_model") + if minimax_model_cfg and cfg.get("target_backend") == "minimax_chat": + set_target_deployment(str(minimax_model_cfg)) + set_reasoning_effort(cfg.get("reasoning_effort", "") or None) + + # Build adapter + adapter = get_adapter(cfg) + adapter.setup(cfg) + + seed = cfg.get("seed", 42) + split = args.split or "all" + + if split == "all": + items = ( + adapter.build_eval_env(0, "train", seed) + + adapter.build_eval_env(0, "valid_seen", seed) + + adapter.build_eval_env(0, "valid_unseen", seed) + ) + else: + env_num = cfg.get("test_env_num", 0) + items = adapter.build_eval_env(env_num, split, seed) + + print(f"\n [eval] split={split} items={len(items)}") + print(f" [eval] out_root={out_root}") + print(f"{'='*60}") + + # Run rollout + results = adapter.rollout(items, skill_content, out_root) + + # Score + hard, soft = compute_score(results) + print(f"\n{'='*60}") + print(f" Results: hard={hard:.4f} soft={soft:.4f} (n={len(results)})") + print(f"{'='*60}") + + # Save summary + summary = { + "skill": skill_path, + "split": split, + "n_items": len(results), + "hard": hard, + "soft": soft, + } + with open(os.path.join(out_root, "eval_summary.json"), "w") as f: + json.dump(summary, f, indent=2, ensure_ascii=False) + + print(f" Saved to: {out_root}") + + +if __name__ == "__main__": + main() diff --git a/scripts/materialize_searchqa.py b/scripts/materialize_searchqa.py new file mode 100644 index 0000000..30838ac --- /dev/null +++ b/scripts/materialize_searchqa.py @@ -0,0 +1,148 @@ +"""Materialize runnable SearchQA splits from the released ID manifest.""" + +from __future__ import annotations + +import argparse +import json +from collections.abc import Iterable, Mapping +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent +SPLITS = ("train", "val", "test") +REQUIRED_FIELDS = ("question", "context", "answers") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--manifest-dir", + type=Path, + default=PROJECT_ROOT / "data" / "searchqa_id_split", + help="Directory containing train/val/test ID manifests.", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=PROJECT_ROOT / "data" / "searchqa_split", + help="Directory to write runnable train/val/test splits.", + ) + parser.add_argument( + "--dataset", + default="lucadiliello/searchqa", + help="Hugging Face dataset repository to load.", + ) + return parser.parse_args() + + +def load_manifest_ids(manifest_dir: Path) -> dict[str, list[str]]: + split_ids = {} + for split in SPLITS: + path = manifest_dir / split / "items.json" + with path.open(encoding="utf-8") as file: + items = json.load(file) + split_ids[split] = [str(item["id"]) for item in items] + return split_ids + + +def _iter_dataset_rows(dataset: Mapping[str, Iterable[dict]]) -> Iterable[dict]: + for source_split in dataset.values(): + yield from source_split + + +def _normalize_row(row: dict) -> dict: + try: + key = str(row["key"]) + except KeyError as exc: + raise ValueError("SearchQA source row is missing required field: key") from exc + + missing = [field for field in REQUIRED_FIELDS if field not in row] + if missing: + raise ValueError(f"SearchQA source row {key!r} is missing required fields: {', '.join(missing)}") + + return { + "id": key, + "question": row["question"], + "context": row["context"], + "answers": row["answers"], + } + + +def materialize_searchqa_splits( + manifest_dir: Path, + output_dir: Path, + dataset: Mapping[str, Iterable[dict]], + *, + dataset_name: str, +) -> dict[str, int]: + """Write runnable SearchQA train/val/test splits from a source dataset.""" + manifest_dir = manifest_dir.resolve() + output_dir = output_dir.resolve() + split_ids = load_manifest_ids(manifest_dir) + wanted_ids = {item_id for ids in split_ids.values() for item_id in ids} + + selected: dict[str, dict] = {} + duplicate_ids: set[str] = set() + for row in _iter_dataset_rows(dataset): + key = str(row.get("key", "")) + if key not in wanted_ids: + continue + if key in selected: + duplicate_ids.add(key) + continue + selected[key] = _normalize_row(row) + + if duplicate_ids: + preview = ", ".join(sorted(duplicate_ids)[:5]) + raise ValueError(f"SearchQA source dataset contains duplicate manifest IDs. First IDs: {preview}") + + missing = sorted(wanted_ids - selected.keys()) + if missing: + preview = ", ".join(missing[:5]) + raise RuntimeError(f"SearchQA source dataset is missing {len(missing)} manifest IDs. First IDs: {preview}") + + counts = {} + for split, ids in split_ids.items(): + items = [selected[item_id] for item_id in ids] + split_dir = output_dir / split + split_dir.mkdir(parents=True, exist_ok=True) + with (split_dir / "items.json").open("w", encoding="utf-8") as file: + json.dump(items, file, ensure_ascii=False, indent=2) + counts[split] = len(items) + + manifest = { + "source_manifest_dir": str(manifest_dir), + "source_dataset": dataset_name, + "counts": counts, + "item_fields": ["id", *REQUIRED_FIELDS], + } + with (output_dir / "split_manifest.json").open("w", encoding="utf-8") as file: + json.dump(manifest, file, ensure_ascii=False, indent=2) + + return counts + + +def main() -> None: + args = parse_args() + try: + from datasets import load_dataset + except ImportError as exc: + raise SystemExit( + "Missing dependency 'datasets'. Install it with:\n" + " python -m pip install 'skillopt[searchqa]'\n" + "or:\n" + " python -m pip install datasets" + ) from exc + + print(f"Loading {args.dataset}...") + dataset = load_dataset(args.dataset) + counts = materialize_searchqa_splits( + args.manifest_dir, + args.output_dir, + dataset, + dataset_name=args.dataset, + ) + print(f"Wrote SearchQA splits to {args.output_dir.resolve()}: {counts}") + + +if __name__ == "__main__": + main() diff --git a/scripts/run_alfworld.sh b/scripts/run_alfworld.sh new file mode 100755 index 0000000..05c5c93 --- /dev/null +++ b/scripts/run_alfworld.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# ────────────────────────────────────────────────────────────────────────────── +# SkillOpt — ALFWorld training launch script +# +# Prerequisites: +# pip install -e ".[alfworld]" +# pip install alfworld[full] && alfworld-download +# +# Usage: +# bash scripts/run_alfworld.sh +# bash scripts/run_alfworld.sh --num_epochs 2 --edit_budget 6 +# bash scripts/run_alfworld.sh --split_dir /path/to/alfworld_split +# ────────────────────────────────────────────────────────────────────────────── +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")" + +export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}" + +# ALFWorld data — uses ~/.cache/alfworld by default +export ALFWORLD_DATA="${ALFWORLD_DATA:-${HOME}/.cache/alfworld}" + +if [ ! -d "${ALFWORLD_DATA}/json_2.1.1" ]; then + echo "ERROR: ALFWorld data not found at ${ALFWORLD_DATA}/json_2.1.1" + echo "" + echo "To download ALFWorld data, run:" + echo " pip install alfworld[full]" + echo " alfworld-download" + echo "" + echo "Or set ALFWORLD_DATA to the directory containing json_2.1.1/" + exit 1 +fi + +OPTIMIZER_MODEL="${OPTIMIZER_MODEL:-gpt-5.5}" +TARGET_MODEL="${TARGET_MODEL:-gpt-5.5}" + +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +DEFAULT_OUT_ROOT="${PROJECT_ROOT}/outputs/skillopt_alfworld_${TARGET_MODEL}_${TIMESTAMP}" + +echo "============================================================" +echo " SkillOpt — ALFWorld Training" +echo "============================================================" +echo " Optimizer: ${OPTIMIZER_MODEL}" +echo " Target: ${TARGET_MODEL}" +echo " ALFWORLD_DATA: ${ALFWORLD_DATA}" +echo " Output: ${DEFAULT_OUT_ROOT}" +echo "============================================================" + +cd "${PROJECT_ROOT}" + +python scripts/train.py \ + --config configs/alfworld/default.yaml \ + --optimizer_model "${OPTIMIZER_MODEL}" \ + --target_model "${TARGET_MODEL}" \ + --out_root "${DEFAULT_OUT_ROOT}" \ + "$@" + +echo "" +echo "Done! Results saved to: ${DEFAULT_OUT_ROOT}" diff --git a/scripts/run_searchqa.sh b/scripts/run_searchqa.sh new file mode 100755 index 0000000..0f7a7cb --- /dev/null +++ b/scripts/run_searchqa.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# ────────────────────────────────────────────────────────────────────────────── +# SkillOpt — SearchQA training launch script +# +# Usage: +# bash scripts/run_searchqa.sh +# bash scripts/run_searchqa.sh --num_epochs 2 --edit_budget 6 +# bash scripts/run_searchqa.sh --split_dir /path/to/searchqa_split +# ────────────────────────────────────────────────────────────────────────────── +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")" + +export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}" + +OPTIMIZER_MODEL="${OPTIMIZER_MODEL:-gpt-5.5}" +TARGET_MODEL="${TARGET_MODEL:-gpt-5.5}" + +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +DEFAULT_OUT_ROOT="${PROJECT_ROOT}/outputs/skillopt_searchqa_${TARGET_MODEL}_${TIMESTAMP}" + +echo "============================================================" +echo " SkillOpt — SearchQA Training" +echo "============================================================" +echo " Optimizer: ${OPTIMIZER_MODEL}" +echo " Target: ${TARGET_MODEL}" +echo "============================================================" + +cd "${PROJECT_ROOT}" + +python scripts/train.py \ + --config configs/searchqa/default.yaml \ + --optimizer_model "${OPTIMIZER_MODEL}" \ + --target_model "${TARGET_MODEL}" \ + --out_root "${DEFAULT_OUT_ROOT}" \ + "$@" + +echo "" +echo "Done! Results saved to: ${DEFAULT_OUT_ROOT}" diff --git a/scripts/run_spreadsheetbench.sh b/scripts/run_spreadsheetbench.sh new file mode 100755 index 0000000..bcbb32c --- /dev/null +++ b/scripts/run_spreadsheetbench.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# ────────────────────────────────────────────────────────────────────────────── +# SkillOpt — SpreadsheetBench training launch script +# +# Usage: +# bash scripts/run_spreadsheetbench.sh --split_dir /path/to/split --data_root /path/to/data +# bash scripts/run_spreadsheetbench.sh --num_epochs 2 --edit_budget 6 +# ────────────────────────────────────────────────────────────────────────────── +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")" + +export PYTHONPATH="${PROJECT_ROOT}:${PYTHONPATH:-}" + +OPTIMIZER_MODEL="${OPTIMIZER_MODEL:-gpt-5.5}" +TARGET_MODEL="${TARGET_MODEL:-gpt-5.5}" + +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +DEFAULT_OUT_ROOT="${PROJECT_ROOT}/outputs/skillopt_spreadsheetbench_${TARGET_MODEL}_${TIMESTAMP}" + +echo "============================================================" +echo " SkillOpt — SpreadsheetBench Training" +echo "============================================================" +echo " Optimizer: ${OPTIMIZER_MODEL}" +echo " Target: ${TARGET_MODEL}" +echo "============================================================" + +cd "${PROJECT_ROOT}" + +python scripts/train.py \ + --config configs/spreadsheetbench/default.yaml \ + --optimizer_model "${OPTIMIZER_MODEL}" \ + --target_model "${TARGET_MODEL}" \ + --out_root "${DEFAULT_OUT_ROOT}" \ + "$@" + +echo "" +echo "Done! Results saved to: ${DEFAULT_OUT_ROOT}" diff --git a/scripts/train.py b/scripts/train.py new file mode 100644 index 0000000..5c0621a --- /dev/null +++ b/scripts/train.py @@ -0,0 +1,556 @@ +#!/usr/bin/env python3 +"""SkillOpt unified training entry point. + +Usage +----- + python scripts/train.py --config configs/alfworld/default.yaml + +Any YAML key can be overridden from the command line:: + + python scripts/train.py --config configs/alfworld/default.yaml \\ + --batch_size 40 --num_epochs 2 --seed 123 + +Run ``python scripts/train.py --help`` for a full list of options. +""" +from __future__ import annotations + +import argparse +import datetime +import os +import sys + +# Ensure the project root is on sys.path so ``import skillopt`` works +# regardless of where the script is invoked from. +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +_PROJECT_ROOT = os.path.dirname(_SCRIPT_DIR) +if _PROJECT_ROOT not in sys.path: + sys.path.insert(0, _PROJECT_ROOT) + +from skillopt.model.common import default_model_for_backend, normalize_backend_name + +_OPENAI_DEFAULT_MODEL_SENTINELS = {"gpt-5.4", "gpt-5.5"} + + +# ── Environment registry ──────────────────────────────────────────────────── + +_ENV_REGISTRY: dict[str, type] = {} + + +def _register_builtins() -> None: + """Lazy-import built-in adapters so we don't pull heavy deps at CLI parse time.""" + try: + from skillopt.envs.alfworld.adapter import ALFWorldAdapter + _ENV_REGISTRY["alfworld"] = ALFWorldAdapter + except ImportError: + pass # ALFWorld deps not installed — skip + try: + from skillopt.envs.searchqa.adapter import SearchQAAdapter + _ENV_REGISTRY["searchqa"] = SearchQAAdapter + except ImportError: + pass + try: + from skillopt.envs.livemathematicianbench.adapter import LiveMathematicianBenchAdapter + _ENV_REGISTRY["livemathematicianbench"] = LiveMathematicianBenchAdapter + except ImportError: + pass + try: + from skillopt.envs.babyvision.adapter import BabyVisionAdapter + _ENV_REGISTRY["babyvision"] = BabyVisionAdapter + except ImportError: + pass + try: + from skillopt.envs.spreadsheetbench.adapter import SpreadsheetBenchAdapter + _ENV_REGISTRY["spreadsheetbench"] = SpreadsheetBenchAdapter + except ImportError: + pass + try: + from skillopt.envs.mmrb.adapter import MMRBAdapter + _ENV_REGISTRY["mmrb"] = MMRBAdapter + except ImportError: + pass + try: + from skillopt.envs.docvqa.adapter import DocVQAAdapter + _ENV_REGISTRY["docvqa"] = DocVQAAdapter + except ImportError: + pass + try: + from skillopt.envs.mathverse.adapter import MathVerseAdapter + _ENV_REGISTRY["mathverse"] = MathVerseAdapter + except ImportError: + pass + try: + from skillopt.envs.officeqa.adapter import OfficeQAAdapter + _ENV_REGISTRY["officeqa"] = OfficeQAAdapter + except ImportError: + pass + try: + from skillopt.envs.sealqa.adapter import SealQAAdapter + _ENV_REGISTRY["sealqa"] = SealQAAdapter + except ImportError: + pass + try: + from skillopt.envs.swebench.adapter import SWEBenchAdapter + _ENV_REGISTRY["swebench"] = SWEBenchAdapter + except ImportError: + pass + + +def get_adapter(cfg: dict): + """Instantiate the environment adapter specified in ``cfg["env"]``.""" + _register_builtins() + env_name = cfg.get("env", "alfworld") + if env_name not in _ENV_REGISTRY: + raise ValueError( + f"Unknown environment '{env_name}'. " + f"Available: {list(_ENV_REGISTRY.keys())}" + ) + adapter_cls = _ENV_REGISTRY[env_name] + + # Inspect adapter __init__ signature and only pass accepted kwargs + import inspect + sig = inspect.signature(adapter_cls.__init__) + accepted = set(sig.parameters.keys()) - {"self"} + adapter_kwargs: dict = {} + for key in accepted: + if key in cfg: + adapter_kwargs[key] = cfg[key] + + return adapter_cls(**adapter_kwargs) + + +# ── CLI ────────────────────────────────────────────────────────────────────── + +_BOOL = lambda x: x.lower() in ("true", "1", "yes") # noqa: E731 + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser( + description="SkillOpt: Executive Strategy for Self-Evolving Agent Skills", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + p.add_argument("--config", type=str, required=True, + help="Path to YAML config file") + p.add_argument("--cfg-options", nargs="+", default=[], + help="Override config: section.key=value (e.g. train.batch_size=40)") + + # Legacy flat CLI overrides (still work, prefer --cfg-options for new usage) + p.add_argument("--env", type=str) + p.add_argument("--backend", type=str, + choices=["azure_openai", "codex", "codex_exec", "claude", "claude_chat", "claude_code_exec", "qwen", "qwen_chat", "minimax", "minimax_chat"]) + p.add_argument("--optimizer_model", type=str) + p.add_argument("--target_model", type=str) + p.add_argument("--optimizer_backend", type=str) + p.add_argument("--target_backend", type=str) + p.add_argument("--reasoning_effort", type=str, + choices=["", "low", "medium", "high", "xhigh", "max"]) + p.add_argument("--rewrite_reasoning_effort", type=str) + p.add_argument("--rewrite_max_completion_tokens", type=int) + p.add_argument("--azure_endpoint", type=str) + p.add_argument("--azure_api_version", type=str) + p.add_argument("--azure_api_key", type=str) + p.add_argument("--azure_openai_endpoint", type=str) + p.add_argument("--azure_openai_api_version", type=str) + p.add_argument("--azure_openai_api_key", type=str) + p.add_argument("--azure_openai_auth_mode", type=str) + p.add_argument("--azure_openai_ad_scope", type=str) + p.add_argument("--azure_openai_managed_identity_client_id", type=str) + p.add_argument("--optimizer_azure_openai_endpoint", type=str) + p.add_argument("--optimizer_azure_openai_api_version", type=str) + p.add_argument("--optimizer_azure_openai_api_key", type=str) + p.add_argument("--optimizer_azure_openai_auth_mode", type=str) + p.add_argument("--optimizer_azure_openai_ad_scope", type=str) + p.add_argument("--optimizer_azure_openai_managed_identity_client_id", type=str) + p.add_argument("--target_azure_openai_endpoint", type=str) + p.add_argument("--target_azure_openai_api_version", type=str) + p.add_argument("--target_azure_openai_api_key", type=str) + p.add_argument("--target_azure_openai_auth_mode", type=str) + p.add_argument("--target_azure_openai_ad_scope", type=str) + p.add_argument("--target_azure_openai_managed_identity_client_id", type=str) + p.add_argument("--qwen_chat_base_url", type=str) + p.add_argument("--qwen_chat_api_key", type=str) + p.add_argument("--qwen_chat_temperature", type=float) + p.add_argument("--qwen_chat_timeout_seconds", type=float) + p.add_argument("--qwen_chat_max_tokens", type=int) + p.add_argument("--qwen_chat_enable_thinking", type=_BOOL) + p.add_argument("--optimizer_qwen_chat_base_url", type=str) + p.add_argument("--optimizer_qwen_chat_api_key", type=str) + p.add_argument("--optimizer_qwen_chat_temperature", type=float) + p.add_argument("--optimizer_qwen_chat_timeout_seconds", type=float) + p.add_argument("--optimizer_qwen_chat_max_tokens", type=int) + p.add_argument("--optimizer_qwen_chat_enable_thinking", type=_BOOL) + p.add_argument("--target_qwen_chat_base_url", type=str) + p.add_argument("--target_qwen_chat_api_key", type=str) + p.add_argument("--target_qwen_chat_temperature", type=float) + p.add_argument("--target_qwen_chat_timeout_seconds", type=float) + p.add_argument("--target_qwen_chat_max_tokens", type=int) + p.add_argument("--target_qwen_chat_enable_thinking", type=_BOOL) + p.add_argument("--minimax_base_url", type=str) + p.add_argument("--minimax_api_key", type=str) + p.add_argument("--minimax_model", type=str) + p.add_argument("--minimax_temperature", type=float) + p.add_argument("--minimax_max_tokens", type=int) + p.add_argument("--minimax_enable_thinking", type=_BOOL) + p.add_argument("--codex_exec_path", type=str) + p.add_argument("--codex_exec_sandbox", type=str) + p.add_argument("--codex_exec_profile", type=str) + p.add_argument("--codex_exec_full_auto", type=_BOOL) + p.add_argument("--codex_exec_reasoning_effort", type=str) + p.add_argument("--codex_exec_use_sdk", type=str) + p.add_argument("--codex_exec_network_access", type=_BOOL) + p.add_argument("--codex_exec_web_search", type=_BOOL) + p.add_argument("--codex_exec_approval_policy", type=str) + p.add_argument("--claude_code_exec_path", type=str) + p.add_argument("--claude_code_exec_profile", type=str) + p.add_argument("--claude_code_exec_use_sdk", type=str) + p.add_argument("--claude_code_exec_effort", type=str) + p.add_argument("--claude_code_exec_max_thinking_tokens", type=int) + p.add_argument("--codex_trace_to_optimizer", type=_BOOL) + p.add_argument("--skill_init", type=str) + p.add_argument("--num_epochs", type=int) + p.add_argument("--train_size", type=int) + p.add_argument("--steps_per_epoch", type=int) + p.add_argument("--batch_size", type=int) + p.add_argument("--accumulation", type=int) + p.add_argument("--seed", type=int) + p.add_argument("--edit_budget", type=int) + p.add_argument("--min_edit_budget", type=int) + p.add_argument("--lr_scheduler", type=str, + choices=["constant", "linear", "cosine", "autonomous"]) + p.add_argument("--lr_control_mode", type=str, + choices=["fixed", "autonomous", "none"]) + p.add_argument("--merge_batch_size", type=int) + p.add_argument("--max_analyst_rounds", type=int) + p.add_argument("--sel_env_num", type=int) + p.add_argument("--test_env_num", type=int) + p.add_argument("--eval_test", type=_BOOL) + p.add_argument("--use_gate", type=_BOOL) + p.add_argument("--max_steps", type=int) + p.add_argument("--max_api_workers", type=int) + p.add_argument("--analyst_workers", type=int) + p.add_argument("--failure_only", type=_BOOL) + p.add_argument("--minibatch_size", type=int) + p.add_argument("--skill_update_mode", type=str, + choices=[ + "patch", + "rewrite_from_suggestions", + "rewrite", + "suggestions", + "full_rewrite", + "full_rewrite_minibatch", + "minibatch_full_rewrite", + ]) + p.add_argument("--use_slow_update", type=_BOOL) + p.add_argument("--slow_update_samples", type=int) + p.add_argument("--longitudinal_pair_policy", type=str, + choices=["mixed", "changed", "unchanged"]) + p.add_argument("--use_meta_skill", type=_BOOL) + p.add_argument("--use_skill_aware_reflection", type=_BOOL) + p.add_argument("--skill_aware_appendix_source", type=str, + choices=["both", "failure_only"]) + p.add_argument("--skill_aware_consolidate_threshold", type=int) + p.add_argument("--data_path", type=str) + p.add_argument("--split_mode", type=str, + choices=["ratio", "split_dir"]) + p.add_argument("--split_ratio", type=str) + p.add_argument("--split_seed", type=int) + p.add_argument("--split_dir", type=str) + p.add_argument("--split_output_dir", type=str) + p.add_argument("--data_root", type=str) + p.add_argument("--max_turns", type=int) + p.add_argument("--workers", type=int) + p.add_argument("--limit", type=int) + p.add_argument("--shuffle_choices", type=_BOOL) + p.add_argument("--use_theorem", type=_BOOL) + p.add_argument("--use_sketch", type=_BOOL) + p.add_argument("--image_detail", type=str) + p.add_argument("--judge_model", type=str) + p.add_argument("--judge_max_completion_tokens", type=int) + p.add_argument("--judge_retries", type=int) + p.add_argument("--out_root", type=str) + p.add_argument("--mode", type=str) + + return p.parse_args() + + +# ── Flat key → structured path mapping (for legacy CLI → structured config) ── + +_LEGACY_TO_STRUCTURED: dict[str, str] = { + "backend": "model.backend", + "optimizer_model": "model.optimizer", + "target_model": "model.target", + "optimizer_backend": "model.optimizer_backend", + "target_backend": "model.target_backend", + "reasoning_effort": "model.reasoning_effort", + "rewrite_reasoning_effort": "model.rewrite_reasoning_effort", + "rewrite_max_completion_tokens": "model.rewrite_max_completion_tokens", + "azure_endpoint": "model.azure_endpoint", + "azure_api_version": "model.azure_api_version", + "azure_api_key": "model.azure_api_key", + "azure_openai_endpoint": "model.azure_openai_endpoint", + "azure_openai_api_version": "model.azure_openai_api_version", + "azure_openai_api_key": "model.azure_openai_api_key", + "azure_openai_auth_mode": "model.azure_openai_auth_mode", + "azure_openai_ad_scope": "model.azure_openai_ad_scope", + "azure_openai_managed_identity_client_id": "model.azure_openai_managed_identity_client_id", + "optimizer_azure_openai_endpoint": "model.optimizer_azure_openai_endpoint", + "optimizer_azure_openai_api_version": "model.optimizer_azure_openai_api_version", + "optimizer_azure_openai_api_key": "model.optimizer_azure_openai_api_key", + "optimizer_azure_openai_auth_mode": "model.optimizer_azure_openai_auth_mode", + "optimizer_azure_openai_ad_scope": "model.optimizer_azure_openai_ad_scope", + "optimizer_azure_openai_managed_identity_client_id": "model.optimizer_azure_openai_managed_identity_client_id", + "target_azure_openai_endpoint": "model.target_azure_openai_endpoint", + "target_azure_openai_api_version": "model.target_azure_openai_api_version", + "target_azure_openai_api_key": "model.target_azure_openai_api_key", + "target_azure_openai_auth_mode": "model.target_azure_openai_auth_mode", + "target_azure_openai_ad_scope": "model.target_azure_openai_ad_scope", + "target_azure_openai_managed_identity_client_id": "model.target_azure_openai_managed_identity_client_id", + "qwen_chat_base_url": "model.qwen_chat_base_url", + "qwen_chat_api_key": "model.qwen_chat_api_key", + "qwen_chat_temperature": "model.qwen_chat_temperature", + "qwen_chat_timeout_seconds": "model.qwen_chat_timeout_seconds", + "qwen_chat_max_tokens": "model.qwen_chat_max_tokens", + "qwen_chat_enable_thinking": "model.qwen_chat_enable_thinking", + "optimizer_qwen_chat_base_url": "model.optimizer_qwen_chat_base_url", + "optimizer_qwen_chat_api_key": "model.optimizer_qwen_chat_api_key", + "optimizer_qwen_chat_temperature": "model.optimizer_qwen_chat_temperature", + "optimizer_qwen_chat_timeout_seconds": "model.optimizer_qwen_chat_timeout_seconds", + "optimizer_qwen_chat_max_tokens": "model.optimizer_qwen_chat_max_tokens", + "optimizer_qwen_chat_enable_thinking": "model.optimizer_qwen_chat_enable_thinking", + "target_qwen_chat_base_url": "model.target_qwen_chat_base_url", + "target_qwen_chat_api_key": "model.target_qwen_chat_api_key", + "target_qwen_chat_temperature": "model.target_qwen_chat_temperature", + "target_qwen_chat_timeout_seconds": "model.target_qwen_chat_timeout_seconds", + "target_qwen_chat_max_tokens": "model.target_qwen_chat_max_tokens", + "target_qwen_chat_enable_thinking": "model.target_qwen_chat_enable_thinking", + "minimax_base_url": "model.minimax_base_url", + "minimax_api_key": "model.minimax_api_key", + "minimax_model": "model.minimax_model", + "minimax_temperature": "model.minimax_temperature", + "minimax_max_tokens": "model.minimax_max_tokens", + "minimax_enable_thinking": "model.minimax_enable_thinking", + "codex_exec_path": "model.codex_exec_path", + "codex_exec_sandbox": "model.codex_exec_sandbox", + "codex_exec_profile": "model.codex_exec_profile", + "codex_exec_full_auto": "model.codex_exec_full_auto", + "codex_exec_reasoning_effort": "model.codex_exec_reasoning_effort", + "codex_exec_use_sdk": "model.codex_exec_use_sdk", + "codex_exec_network_access": "model.codex_exec_network_access", + "codex_exec_web_search": "model.codex_exec_web_search", + "codex_exec_approval_policy": "model.codex_exec_approval_policy", + "claude_code_exec_path": "model.claude_code_exec_path", + "claude_code_exec_profile": "model.claude_code_exec_profile", + "claude_code_exec_use_sdk": "model.claude_code_exec_use_sdk", + "claude_code_exec_effort": "model.claude_code_exec_effort", + "claude_code_exec_max_thinking_tokens": "model.claude_code_exec_max_thinking_tokens", + "codex_trace_to_optimizer": "model.codex_trace_to_optimizer", + "num_epochs": "train.num_epochs", + "train_size": "train.train_size", + "steps_per_epoch": "train.steps_per_epoch", + "batch_size": "train.batch_size", + "accumulation": "train.accumulation", + "seed": "train.seed", + "minibatch_size": "gradient.minibatch_size", + "merge_batch_size": "gradient.merge_batch_size", + "analyst_workers": "gradient.analyst_workers", + "max_analyst_rounds": "gradient.max_analyst_rounds", + "failure_only": "gradient.failure_only", + "edit_budget": "optimizer.learning_rate", + "min_edit_budget": "optimizer.min_learning_rate", + "lr_scheduler": "optimizer.lr_scheduler", + "lr_control_mode": "optimizer.lr_control_mode", + "skill_update_mode": "optimizer.skill_update_mode", + "use_slow_update": "optimizer.use_slow_update", + "slow_update_samples": "optimizer.slow_update_samples", + "longitudinal_pair_policy": "optimizer.longitudinal_pair_policy", + "use_meta_skill": "optimizer.use_meta_skill", + "use_skill_aware_reflection": "optimizer.use_skill_aware_reflection", + "skill_aware_appendix_source": "optimizer.skill_aware_appendix_source", + "skill_aware_consolidate_threshold": "optimizer.skill_aware_consolidate_threshold", + "use_gate": "evaluation.use_gate", + "sel_env_num": "evaluation.sel_env_num", + "test_env_num": "evaluation.test_env_num", + "eval_test": "evaluation.eval_test", + "env": "env.name", + "skill_init": "env.skill_init", + "out_root": "env.out_root", +} + + +def load_config(args: argparse.Namespace) -> dict: + """Load config with _base_ inheritance, then apply CLI overrides.""" + from skillopt.config import load_config as _load, flatten_config, is_structured + + cfg = _load(args.config, overrides=args.cfg_options) + structured = is_structured(cfg) + + # Apply legacy --key value overrides + cli = {k: v for k, v in vars(args).items() + if v is not None and k not in ("config", "cfg_options")} + if cli: + if structured: + from skillopt.config import apply_overrides + mapped = [] + for k, v in cli.items(): + dotted = _LEGACY_TO_STRUCTURED.get(k) + if dotted: + mapped.append(f"{dotted}={v}") + else: + mapped.append(f"env.{k}={v}") + apply_overrides(cfg, mapped) + else: + cfg.update(cli) + + # Flatten structured config → flat dict for trainer/adapter + flat = flatten_config(cfg) if structured else cfg + + for new_key, old_key in ( + ("azure_openai_endpoint", "azure_endpoint"), + ("azure_openai_api_version", "azure_api_version"), + ("azure_openai_api_key", "azure_api_key"), + ): + if flat.get(new_key) in (None, "") and flat.get(old_key) not in (None, ""): + flat[new_key] = flat[old_key] + + explicit_backend = getattr(args, "backend", None) + if explicit_backend is None: + for option in args.cfg_options or []: + key = str(option).split("=", 1)[0].strip() + if key == "model.backend": + explicit_backend = str(option).split("=", 1)[1].strip() + break + + backend = normalize_backend_name(flat.get("model_backend") or flat.get("target_backend") or "azure_openai") + + def _has_model_override(dotted_key: str, legacy_key: str) -> bool: + if getattr(args, legacy_key, None) is not None: + return True + for option in args.cfg_options or []: + key = str(option).split("=", 1)[0].strip() + if key == dotted_key: + return True + return False + + if explicit_backend is not None: + backend = normalize_backend_name(explicit_backend) + flat["model_backend"] = backend + if backend in {"claude", "claude_chat"}: + flat.setdefault("optimizer_backend", "claude_chat") + flat.setdefault("target_backend", "claude_chat") + elif backend in {"codex", "codex_exec"}: + flat.setdefault("optimizer_backend", "openai_chat") + flat.setdefault("target_backend", "codex_exec") + elif backend == "claude_code_exec": + flat.setdefault("optimizer_backend", "openai_chat") + flat.setdefault("target_backend", "claude_code_exec") + elif backend in {"qwen", "qwen_chat"}: + flat.setdefault("optimizer_backend", "openai_chat") + flat.setdefault("target_backend", "qwen_chat") + elif backend in {"minimax", "minimax_chat"}: + flat.setdefault("optimizer_backend", "openai_chat") + flat.setdefault("target_backend", "minimax_chat") + else: + flat.setdefault("optimizer_backend", "openai_chat") + flat.setdefault("target_backend", "openai_chat") + else: + flat.setdefault("optimizer_backend", "openai_chat") + flat.setdefault("target_backend", "openai_chat") + + if flat.get("optimizer_backend") == "claude_chat": + if ( + str(flat.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.optimizer", "optimizer_model") + ): + flat["optimizer_model"] = default_model_for_backend("claude_chat") + if flat.get("optimizer_backend") == "qwen_chat": + if ( + str(flat.get("optimizer_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.optimizer", "optimizer_model") + ): + flat["optimizer_model"] = default_model_for_backend("qwen_chat") + if flat.get("target_backend") == "claude_chat": + if ( + str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + flat["target_model"] = default_model_for_backend("claude_chat") + if flat.get("target_backend") == "claude_code_exec": + if ( + str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + flat["target_model"] = default_model_for_backend("claude_chat") + if flat.get("target_backend") == "qwen_chat": + if ( + str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + flat["target_model"] = default_model_for_backend("qwen_chat") + if flat.get("target_backend") == "minimax_chat": + if ( + str(flat.get("target_model", "") or "").strip() in _OPENAI_DEFAULT_MODEL_SENTINELS + and not _has_model_override("model.target", "target_model") + ): + flat["target_model"] = ( + flat.get("minimax_model") + or default_model_for_backend("minimax_chat") + ) + + # Auto-generate output root + if not flat.get("out_root"): + env = flat.get("env", "unknown") + model = flat.get("optimizer_model", "unknown").replace("/", "-") + ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + flat["out_root"] = os.path.join("outputs", f"skillopt_{env}_{model}_{ts}") + + flat["out_root"] = os.path.abspath(flat["out_root"]) + return flat + + +# ── Main ───────────────────────────────────────────────────────────────────── + +def main() -> None: + args = parse_args() + cfg = load_config(args) + + print(f"\n{'='*60}") + print(f" SkillOpt — Executive Strategy for Self-Evolving Agent Skills") + print(f"{'='*60}") + print(f" env: {cfg.get('env')}") + print(f" optimizer_model: {cfg.get('optimizer_model')}") + print(f" target_model: {cfg.get('target_model')}") + print(f" optimizer_backend:{cfg.get('optimizer_backend', 'openai_chat')}") + print(f" target_backend:{cfg.get('target_backend', 'openai_chat')}") + print(f" reasoning: {cfg.get('reasoning_effort') or 'off'}") + print(f" rewrite_effort: {cfg.get('rewrite_reasoning_effort') or 'off'}") + print(f" epochs: {cfg.get('num_epochs')}") + print(f" train_size: {cfg.get('train_size') or 'from dataset'}") + print(f" steps/epoch: auto") + print(f" batch_size: {cfg.get('batch_size')}") + print(f" edit_budget: {cfg.get('edit_budget')}") + print(f" lr_scheduler: {cfg.get('lr_scheduler', 'constant')}") + print(f" update_mode: {cfg.get('skill_update_mode', 'patch')}") + print(f" min_edit_budget:{cfg.get('min_edit_budget', 2)}") + print(f" minibatch_size: {cfg.get('minibatch_size')}") + print(f" seed: {cfg.get('seed')}") + print(f" meta_skill: {cfg.get('use_meta_skill', False)}") + print(f" skill_aware_reflection: {cfg.get('use_skill_aware_reflection', False)}") + print(f" slow_update: {cfg.get('use_slow_update', False)}") + print(f" out_root: {cfg.get('out_root')}") + print(f"{'='*60}\n") + + # Build adapter + adapter = get_adapter(cfg) + + # Build trainer and run + from skillopt.engine.trainer import ReflACTTrainer + trainer = ReflACTTrainer(cfg, adapter) + summary = trainer.train() + + print(f"\n Output saved to: {cfg['out_root']}") + if summary.get("test_hard") is not None: + print(f" Final test: {summary['test_hard']:.4f}") + + +if __name__ == "__main__": + main() diff --git a/skillopt-assets/arxiv-logomark-small.svg b/skillopt-assets/arxiv-logomark-small.svg new file mode 100644 index 0000000..91e027c --- /dev/null +++ b/skillopt-assets/arxiv-logomark-small.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/skillopt-assets/epoch-trends-1.png b/skillopt-assets/epoch-trends-1.png new file mode 100644 index 0000000..13cd46d Binary files /dev/null and b/skillopt-assets/epoch-trends-1.png differ diff --git a/skillopt-assets/openai.png b/skillopt-assets/openai.png new file mode 100644 index 0000000..bd7a119 Binary files /dev/null and b/skillopt-assets/openai.png differ diff --git a/skillopt-assets/pipeline-1.png b/skillopt-assets/pipeline-1.png new file mode 100644 index 0000000..7d56b4a Binary files /dev/null and b/skillopt-assets/pipeline-1.png differ diff --git a/skillopt-assets/qwen-color.png b/skillopt-assets/qwen-color.png new file mode 100644 index 0000000..2667528 Binary files /dev/null and b/skillopt-assets/qwen-color.png differ diff --git a/skillopt-assets/teaser-1.png b/skillopt-assets/teaser-1.png new file mode 100644 index 0000000..6a8cf15 Binary files /dev/null and b/skillopt-assets/teaser-1.png differ diff --git a/skillopt.html b/skillopt.html new file mode 100644 index 0000000..2be9a01 --- /dev/null +++ b/skillopt.html @@ -0,0 +1,2736 @@ + + + + + + SkillOpt | Executive Strategy for Self-Evolving Agent Skills + + + + + + + +
+
+
+ Text-space optimization for frozen agents +

SkillOpt

+

+ Executive Strategy for Self-Evolving Agent Skills. SkillOpt treats a compact + natural-language skill document as the trainable state of a frozen language + agent, then learns that document through rollouts, reflection, bounded edits, + and held-out validation gates. +

+
+ Core Idea + Method + View Results + + + Code Repo + + + + Paper + + + + Video + +
+ + + + Related project + SkillLens studies model-generated agent skills. + A companion project page from Microsoft Research. + + + +
+ + +
+
+ +
+
+
+ Project Video +
+

SkillOpt in motion.

+

+ A short visual overview of how SkillOpt treats natural-language skills + as trainable artifacts: roll out, reflect, edit, validate, and export. +

+
+
+
+ +
+

+ Promotional video for the SkillOpt project page. The static paper teaser is shown below for high-resolution inspection. +

+
+ +
+
+ Paper Teaser +
+

The core loop at a glance.

+

+ The teaser summarizes the SkillOpt training loop: rollout evidence, + optimizer-side reflection, bounded skill edits, validation gating, + and the exported reusable skill. +

+
+
+
+ SkillOpt teaser figure showing the target model, optimizer model, bounded edits, validation gate, and exported best skill. +
+

+ Figure from the SkillOpt paper. On small screens, the figure area scrolls horizontally to preserve the original details. +

+
+ +
+
+
01 / Core Idea
+
+

Train the procedure, not the weights.

+

+ SkillOpt makes the skill document itself the optimization target. The + target model, backend, and harness stay fixed; the procedure that guides + evidence gathering, tool use, verification, and output formatting evolves. +

+
+
+ +
+
+

A skill is external state for an agent.

+

+ Instead of fine-tuning a model or hand-maintaining prompts, SkillOpt runs + the frozen agent on scored batches, asks a separate optimizer model to + propose structured edits, and accepts a candidate only when validation + performance improves. +

+
+ Frozen target model + Optimizer model + Add / delete / replace edits + Held-out gate +
+
+ +
+
+ Rollout +

The target model executes tasks with the current skill and records scored trajectories.

+
+
+ Reflect +

The optimizer analyzes success and failure minibatches to find reusable procedures.

+
+
+ Edit +

Candidate add, delete, and replace operations are merged and ranked under a budget.

+
+
+ Gate +

The candidate skill is kept only if it improves held-out selection performance.

+
+
+
+
+ +
+
+
02 / Method
+
+

A training loop for natural-language skills.

+

+ The loop deliberately mirrors a learning algorithm: rollout evidence acts + like a forward pass, reflection acts like a language-level backward pass, + and the textual learning rate bounds how far the skill can move. +

+
+
+ +
+
+

Evidence

+

Rollout batches capture messages, tool calls, verifier feedback, task metadata, and final scores.

+
+
+

Minibatches

+

Failures and successes are reflected separately so edits correct recurring errors while preserving working behavior.

+
+
+

Bounded Edits

+

An edit budget functions as a textual learning rate, preventing useful rules from being overwritten by broad rewrites.

+
+
+

Memory

+

Rejected edits, slow update, and optimizer-side meta skill provide longer-horizon feedback without bloating deployment.

+
+
+ +
+ SkillOpt pipeline showing rollout, reflection, bounded edits, validation gate, slow update, and meta skill. +
+ SkillOpt pipeline from the paper. The frozen target model executes with the current skill; the optimizer model proposes bounded edits; held-out validation decides whether the candidate becomes the new current skill. +
+
+
+ +
+
+
03 / Main Results
+
+

SkillOpt improves GPT and Qwen target models.

+

+ The table reports main-result gains across target models and + execution harnesses, comparing no-skill execution with the final + SkillOpt skill on held-out test splits. +

+
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Target modelHarnessSearchQASheetOfficeDocVQALiveMathALFWorldAvg gain
OpenAI logoGPT-5.5Direct chat+9.6+38.9+39.0+12.4+29.3+11.9+23.5
OpenAI logoGPT-5.4Direct chat+6.2+21.1+12.8+13.6+7.2+15.6+12.8
OpenAI logoGPT-5.4-miniDirect chat+4.3+11.4+26.7+16.5+4.8+12.7+12.7
OpenAI logoGPT-5.4-nanoDirect chat+19.0+8.2+33.7+49.4+4.0+35.1+24.9
OpenAI logoGPT-5.2Direct chat+11.2+18.9+21.5+16.5+15.2+16.4+16.6
Qwen logoQwen3.5-4BDirect chat+3.1+14.6+15.2+2.1+29.6+50.7+19.2
Qwen logoQwen3.6-35B-A3BDirect chat+7.6+9.3+1.2+3.8+10.4+22.4+9.1
OpenAI logoGPT-5.5Codex+5.5+57.5+12.8+5.0+28.0N/A+21.8
OpenAI logoGPT-5.5Claude Code+4.0+58.3+13.9+3.5+13.3N/A+18.6
+
+ +
+
+
+ Method comparison +

SkillOpt clears the strongest baseline on every benchmark.

+
+
+
+
+
+ +
+ +
+
+
04 / Ablations
+
+

The controls are doing real work.

+

+ The paper isolates the optimizer components that keep skill learning stable: + enough evidence, bounded textual updates, rejected-edit feedback, slow + update, and optimizer-side memory. +

+
+
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ComponentSettingSearchQASpreadsheetLiveMath
Learning ratelr=4 default87.177.561.3
Learning ratewithout lr84.675.757.3
Rejected bufferwith buffer87.177.561.3
Rejected bufferwithout buffer85.572.958.9
Update memorymeta skill + slow update87.177.561.3
Update memorywithout both86.355.059.7
+
+ +
+

What the ablations say

+
+
+ Bounded + Textual learning rates prevent destructive rewrites while keeping enough plasticity to learn new procedures. +
+
+ Gated + Held-out selection turns reflection into propose-and-test optimization rather than unconditional self-editing. +
+
+ Buffered + Rejected edits become negative feedback, helping the optimizer avoid repeating harmful directions. +
+
+
+
+ +
+ Epoch checkpoint trends for SpreadsheetBench, SearchQA, and LiveMath. +
+ Epoch checkpoint trends from the paper. Selection-best checkpoints are compared with train rollout score and unseen test performance. +
+
+
+ +
+
+
05 / Skill Evolution
+
+

A typical run turns failures into concrete operating rules.

+

+ This ALFWorld run uses GPT-5.4-mini as the frozen target model and + GPT-5.5 as the optimizer model. The plot tracks train rollout and + held-out selection scores; hover or focus a point to inspect the + skill edit proposed at that stage. +

+
+
+ +
+
+
+ ALFWorld / train-sel evolution +
+ Train rollout + Selection gate +
+
+
+ + ALFWorld skill evolution scores + Selection score rises from 68.6 percent to 81.4 percent, while rejected edits are visible as downward candidate points. + + + + + + + + 85% + 80% + 75% + 70% + 65% + base + step 1 + step 2 + step 3 + slow + step 4 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ Accepted edits become the current skill only after held-out selection improves. + Step 3 is rescued by a slow update; Step 4 trains higher but fails selection. +
+
+ + +
+ +
+
+ Run setup + Target model: GPT-5.4-mini. Optimizer model: GPT-5.5. The skill starts from a compact ALFWorld instruction file and is edited in text space. +
+
+ Selection rule + Candidate edits are accepted only when held-out selection improves the current best score. +
+
+ Outcome + The selected skill improves final ALFWorld test hard score from 70.9% to 85.8%. +
+
+
+ +
+
+
06 / Transfer
+
+

The exported skill behaves like a reusable artifact.

+

+ SkillOpt exports a compact best_skill.md. The paper tests + whether that artifact transfers across model sizes, execution harnesses, + and nearby benchmarks without further target-side optimization. +

+
+
+ +
+
+ Cross-model + +15.2 +

GPT-5.4 LiveMath skill transferred to GPT-5.4-nano on LiveMathBench.

+
+
+ Cross-harness + +31.8 +

Codex-trained SpreadsheetBench skill transferred into Claude Code.

+
+
+ Self-optimizer + +10.4 +

GPT-5.4-nano used as its own optimizer improved SpreadsheetBench over baseline.

+
+
+ Deployment + 1 file +

The target model consumes only the final skill, not optimizer memory.

+
+
+ +
+ A stronger optimizer model gives the largest gains, but the loop is not merely + distillation from a stronger model. Even matched target-as-optimizer settings + can discover useful edits when the update is constrained, buffered, and + validated. +
+
+ +
+
+
07 / BibTeX
+
+

Citation.

+

+ If you find SkillOpt useful, please cite the arXiv preprint below. +

+
+
+ +
+ + 
@article{yang2026skillopt,
+  title={Skillopt: Executive strategy for self-evolving agent skills},
+  author={Yang, Yifan and Gong, Ziyang and Huang, Weiquan and Yang, Qihao and Zhou, Ziwei and Huang, Zisu and Li, Yan and Gao, Xuemei and Dai, Qi and Liu, Bei and others},
+  journal={arXiv preprint arXiv:2605.23904},
+  year={2026}
+}
+
+
+ +
+ SkillOpt: Executive Strategy for Self-Evolving Agent Skills + Code / Citation +
+
+ + + diff --git a/skillopt/__init__.py b/skillopt/__init__.py new file mode 100644 index 0000000..d370c6e --- /dev/null +++ b/skillopt/__init__.py @@ -0,0 +1,28 @@ +"""ReflACT: Reflective Agent Tuning. + +A general-purpose framework for iteratively optimizing LLM agent skills +through structured reflection and self-improvement. + +Pipeline stages: + 1. Rollout — execute episodes with current skill + 2. Reflect — analyze trajectories, generate patches + 3. Aggregate — hierarchical merge of patches + 4. Select — rank and select top edits + 5. Update — apply edits to skill document + 6. Evaluate — validate candidate skill, accept/reject +""" + +__version__ = "0.2.0" + +from skillopt.types import ( # noqa: F401 + BatchSpec, + Edit, + EditOp, + FailureSummaryEntry, + GateAction, + GateResult, + Patch, + RawPatch, + RolloutResult, + SlowUpdateResult, +) diff --git a/skillopt/config.py b/skillopt/config.py new file mode 100644 index 0000000..2c1c4e9 --- /dev/null +++ b/skillopt/config.py @@ -0,0 +1,285 @@ +"""ReflACT config loading engine — structured YAML with inheritance. + +Supports two config formats: + 1. **Structured** (new): sections like ``model``, ``train``, ``gradient``, + ``optimizer``, ``evaluation``, ``env`` — with ``_base_`` inheritance. + 2. **Flat** (legacy): all keys at top level — fully backward compatible. + +Usage:: + + from skillopt.config import load_config, flatten_config + + cfg = load_config("configs/searchqa_default.yaml") + flat = flatten_config(cfg) # always returns flat dict for trainer +""" +from __future__ import annotations + +import copy +import os +from typing import Any + +import yaml + +# ── Section names that indicate a structured config ────────────────────── + +_STRUCTURED_SECTIONS = frozenset({ + "model", "train", "gradient", "optimizer", "evaluation", "env", +}) + +# ── Structured → flat key mapping ──────────────────────────────────────── + +_FLATTEN_MAP: dict[str, str] = { + "model.backend": "model_backend", + "model.optimizer": "optimizer_model", + "model.target": "target_model", + "model.optimizer_backend": "optimizer_backend", + "model.target_backend": "target_backend", + "model.reasoning_effort": "reasoning_effort", + "model.rewrite_reasoning_effort": "rewrite_reasoning_effort", + "model.rewrite_max_completion_tokens": "rewrite_max_completion_tokens", + "model.codex_exec_path": "codex_exec_path", + "model.codex_exec_sandbox": "codex_exec_sandbox", + "model.codex_exec_profile": "codex_exec_profile", + "model.codex_exec_full_auto": "codex_exec_full_auto", + "model.codex_exec_reasoning_effort": "codex_exec_reasoning_effort", + "model.codex_exec_use_sdk": "codex_exec_use_sdk", + "model.codex_exec_network_access": "codex_exec_network_access", + "model.codex_exec_web_search": "codex_exec_web_search", + "model.codex_exec_approval_policy": "codex_exec_approval_policy", + "model.claude_code_exec_path": "claude_code_exec_path", + "model.claude_code_exec_profile": "claude_code_exec_profile", + "model.claude_code_exec_use_sdk": "claude_code_exec_use_sdk", + "model.claude_code_exec_effort": "claude_code_exec_effort", + "model.claude_code_exec_max_thinking_tokens": "claude_code_exec_max_thinking_tokens", + "model.codex_trace_to_optimizer": "codex_trace_to_optimizer", + "model.azure_endpoint": "azure_endpoint", + "model.azure_api_version": "azure_api_version", + "model.azure_api_key": "azure_api_key", + "model.azure_openai_endpoint": "azure_openai_endpoint", + "model.azure_openai_api_version": "azure_openai_api_version", + "model.azure_openai_api_key": "azure_openai_api_key", + "model.azure_openai_auth_mode": "azure_openai_auth_mode", + "model.azure_openai_ad_scope": "azure_openai_ad_scope", + "model.azure_openai_managed_identity_client_id": "azure_openai_managed_identity_client_id", + "model.optimizer_azure_openai_endpoint": "optimizer_azure_openai_endpoint", + "model.optimizer_azure_openai_api_version": "optimizer_azure_openai_api_version", + "model.optimizer_azure_openai_api_key": "optimizer_azure_openai_api_key", + "model.optimizer_azure_openai_auth_mode": "optimizer_azure_openai_auth_mode", + "model.optimizer_azure_openai_ad_scope": "optimizer_azure_openai_ad_scope", + "model.optimizer_azure_openai_managed_identity_client_id": "optimizer_azure_openai_managed_identity_client_id", + "model.target_azure_openai_endpoint": "target_azure_openai_endpoint", + "model.target_azure_openai_api_version": "target_azure_openai_api_version", + "model.target_azure_openai_api_key": "target_azure_openai_api_key", + "model.target_azure_openai_auth_mode": "target_azure_openai_auth_mode", + "model.target_azure_openai_ad_scope": "target_azure_openai_ad_scope", + "model.target_azure_openai_managed_identity_client_id": "target_azure_openai_managed_identity_client_id", + "model.qwen_chat_base_url": "qwen_chat_base_url", + "model.qwen_chat_api_key": "qwen_chat_api_key", + "model.qwen_chat_temperature": "qwen_chat_temperature", + "model.qwen_chat_timeout_seconds": "qwen_chat_timeout_seconds", + "model.qwen_chat_max_tokens": "qwen_chat_max_tokens", + "model.qwen_chat_enable_thinking": "qwen_chat_enable_thinking", + "model.optimizer_qwen_chat_base_url": "optimizer_qwen_chat_base_url", + "model.optimizer_qwen_chat_api_key": "optimizer_qwen_chat_api_key", + "model.optimizer_qwen_chat_temperature": "optimizer_qwen_chat_temperature", + "model.optimizer_qwen_chat_timeout_seconds": "optimizer_qwen_chat_timeout_seconds", + "model.optimizer_qwen_chat_max_tokens": "optimizer_qwen_chat_max_tokens", + "model.optimizer_qwen_chat_enable_thinking": "optimizer_qwen_chat_enable_thinking", + "model.target_qwen_chat_base_url": "target_qwen_chat_base_url", + "model.target_qwen_chat_api_key": "target_qwen_chat_api_key", + "model.target_qwen_chat_temperature": "target_qwen_chat_temperature", + "model.target_qwen_chat_timeout_seconds": "target_qwen_chat_timeout_seconds", + "model.target_qwen_chat_max_tokens": "target_qwen_chat_max_tokens", + "model.target_qwen_chat_enable_thinking": "target_qwen_chat_enable_thinking", + "model.minimax_base_url": "minimax_base_url", + "model.minimax_api_key": "minimax_api_key", + "model.minimax_model": "minimax_model", + "model.minimax_temperature": "minimax_temperature", + "model.minimax_max_tokens": "minimax_max_tokens", + "model.minimax_enable_thinking": "minimax_enable_thinking", + "train.num_epochs": "num_epochs", + "train.train_size": "train_size", + "train.steps_per_epoch": "steps_per_epoch", + "train.batch_size": "batch_size", + "train.accumulation": "accumulation", + "train.seed": "seed", + "gradient.minibatch_size": "minibatch_size", + "gradient.merge_batch_size": "merge_batch_size", + "gradient.analyst_workers": "analyst_workers", + "gradient.failure_only": "failure_only", + "gradient.max_analyst_rounds": "max_analyst_rounds", + "optimizer.learning_rate": "edit_budget", + "optimizer.min_learning_rate": "min_edit_budget", + "optimizer.lr_scheduler": "lr_scheduler", + "optimizer.lr_control_mode": "lr_control_mode", + "optimizer.skill_update_mode": "skill_update_mode", + "optimizer.meta_learning_rate": "meta_edit_budget", + "optimizer.use_slow_update": "use_slow_update", + "optimizer.slow_update_samples": "slow_update_samples", + "optimizer.slow_update_gate_with_selection": "slow_update_gate_with_selection", + "optimizer.longitudinal_pair_policy": "longitudinal_pair_policy", + "optimizer.use_meta_skill": "use_meta_skill", + "optimizer.use_skill_aware_reflection": "use_skill_aware_reflection", + "optimizer.skill_aware_appendix_source": "skill_aware_appendix_source", + "optimizer.skill_aware_consolidate_threshold": "skill_aware_consolidate_threshold", + "evaluation.use_gate": "use_gate", + "evaluation.gate_metric": "gate_metric", + "evaluation.gate_mixed_weight": "gate_mixed_weight", + "evaluation.use_semantic_density": "use_semantic_density", + "evaluation.semantic_density_weight": "semantic_density_weight", + "evaluation.leading_words": "leading_words", + "evaluation.sel_env_num": "sel_env_num", + "evaluation.test_env_num": "test_env_num", + "evaluation.eval_test": "eval_test", + "env.name": "env", + "env.skill_init": "skill_init", + "env.out_root": "out_root", +} + + +# ── Deep merge ─────────────────────────────────────────────────────────── + +def _deep_merge(base: dict, override: dict) -> dict: + """Recursively merge *override* into *base* (returns new dict).""" + result = copy.deepcopy(base) + for key, val in override.items(): + if key in result and isinstance(result[key], dict) and isinstance(val, dict): + result[key] = _deep_merge(result[key], val) + else: + result[key] = copy.deepcopy(val) + return result + + +# ── YAML loading with _base_ inheritance ───────────────────────────────── + +def _load_yaml(path: str, _visited: set[str] | None = None) -> dict: + """Load a YAML file, resolving ``_base_`` inheritance recursively.""" + abs_path = os.path.abspath(path) + if _visited is None: + _visited = set() + if abs_path in _visited: + raise ValueError(f"Circular _base_ inheritance: {abs_path}") + _visited.add(abs_path) + + with open(abs_path, encoding="utf-8") as f: + cfg = yaml.safe_load(f) or {} + + base_ref = cfg.pop("_base_", None) + if base_ref: + base_path = os.path.join(os.path.dirname(abs_path), base_ref) + base_cfg = _load_yaml(base_path, _visited) + cfg = _deep_merge(base_cfg, cfg) + + return cfg + + +# ── Format detection ───────────────────────────────────────────────────── + +def is_structured(cfg: dict) -> bool: + """Return True if *cfg* uses the new structured section format.""" + return any( + key in _STRUCTURED_SECTIONS and isinstance(cfg.get(key), dict) + for key in cfg + ) + + +# ── Flatten ────────────────────────────────────────────────────────────── + +def flatten_config(cfg: dict) -> dict: + """Convert a structured config to the flat dict expected by the trainer. + + If *cfg* is already flat, returns a shallow copy unchanged. + """ + if not is_structured(cfg): + return dict(cfg) + + flat: dict[str, Any] = {} + + # Apply the explicit mapping + for dotted, flat_key in _FLATTEN_MAP.items(): + section, key = dotted.split(".", 1) + section_dict = cfg.get(section, {}) + if isinstance(section_dict, dict) and key in section_dict: + flat[flat_key] = section_dict[key] + + # Pass through env-specific keys not in the explicit mapping + env_section = cfg.get("env", {}) + if isinstance(env_section, dict): + mapped_env_keys = { + k.split(".", 1)[1] + for k in _FLATTEN_MAP + if k.startswith("env.") + } + for key, val in env_section.items(): + if key not in mapped_env_keys: + flat[key] = val + + return flat + + +# ── Override application ───────────────────────────────────────────────── + +def _cast_value(val_str: str) -> Any: + """Auto-cast a CLI string value to int / float / bool / str.""" + if val_str.lower() in ("true", "yes"): + return True + if val_str.lower() in ("false", "no"): + return False + try: + return int(val_str) + except ValueError: + pass + try: + return float(val_str) + except ValueError: + pass + return val_str + + +def apply_overrides(cfg: dict, overrides: list[str]) -> None: + """Apply ``key=value`` overrides to a structured config (in place). + + Supports both ``section.key=value`` (for structured configs) and + ``key=value`` (for flat configs or flat keys in env section). + """ + for item in overrides: + if "=" not in item: + raise ValueError(f"Invalid override (expected key=value): {item!r}") + key, val_str = item.split("=", 1) + val = _cast_value(val_str) + + if "." in key: + section, subkey = key.split(".", 1) + if section in cfg and isinstance(cfg[section], dict): + cfg[section][subkey] = val + else: + cfg.setdefault(section, {})[subkey] = val + else: + # Flat key — apply to top level (for legacy compat) + cfg[key] = val + + +# ── Public API ─────────────────────────────────────────────────────────── + +def load_config( + path: str, + overrides: list[str] | None = None, +) -> dict: + """Load a config file with ``_base_`` inheritance and optional overrides. + + Parameters + ---------- + path : str + Path to the YAML config file. + overrides : list[str] | None + ``key=value`` strings from ``--cfg-options``. + + Returns + ------- + dict + The merged config (structured or flat depending on the YAML). + """ + cfg = _load_yaml(path) + if overrides: + apply_overrides(cfg, overrides) + return cfg diff --git a/skillopt/datasets/__init__.py b/skillopt/datasets/__init__.py new file mode 100644 index 0000000..3aa2eb8 --- /dev/null +++ b/skillopt/datasets/__init__.py @@ -0,0 +1,7 @@ +"""ReflACT Datasets -- task batch planning and data loading. + +Analogous to the datasets and dataloaders in neural network training: +provides batch sampling, epoch planning, and data management for the +ReflACT training pipeline. +""" +from skillopt.datasets.base import BaseDataLoader, BatchSpec, SplitDataLoader # noqa: F401 diff --git a/skillopt/datasets/base.py b/skillopt/datasets/base.py new file mode 100644 index 0000000..668f201 --- /dev/null +++ b/skillopt/datasets/base.py @@ -0,0 +1,512 @@ +"""Generic task dataloader abstractions for ReflACT. + +ReflACT does not train model parameters directly. Instead, it iterates over +task batches, rolls out the current skill, reflects on failures/successes, +and updates the skill document. Because of that, the "dataloader" abstraction +here is closer to a batch sampler / episode planner than a tensor loader. + +Class hierarchy:: + + BaseDataLoader # abstract — simulator-backed envs (e.g. ALFWorld) + └── SplitDataLoader # abstract — dataset-backed envs with split_dir + +SplitDataLoader supports two dataset entry modes: + +1. ``split_mode="split_dir"``: consume an existing split directory. +2. ``split_mode="ratio"``: build a deterministic split directory from a raw + dataset path using an explicit train:val:test ratio. + +In either case, the standardised split layout is: + + split_dir/ + ├── train/ # training items + ├── val/ # validation / selection items (gate) + └── test/ # held-out test items + +Each subdirectory's contents are benchmark-specific. Subclasses only need +to implement ``load_split_items(split_path)`` to teach the loader how to +read items from one of those directories. +""" +from __future__ import annotations + +import glob +import json +import os +import random +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(slots=True) +class BatchSpec: + """A concrete batch request consumed by the training loop. + + Parameters + ---------- + phase : str + ``"train"`` or ``"eval"``. + split : str + Dataset split name, typically ``"train"`` or an eval split. + seed : int + Random seed used to construct the batch deterministically. + batch_size : int + Requested number of items / episodes in this batch. + payload : object | None + Environment-specific batch payload. For dataset-backed environments + this is often a list of sampled items; for simulator-backed + environments this may be ``None`` and the seed alone can define the + batch. + metadata : dict[str, Any] + Optional structured metadata for logging, resume, or curriculum logic. + """ + + phase: str + split: str + seed: int + batch_size: int + payload: object | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + +class BaseDataLoader(ABC): + """Abstract base class for task batch planning in ReflACT. + + Subclasses are responsible for defining how a train or eval batch is + sampled. The default implementation here provides deterministic epoch seed + planning so all loaders share the same reproducibility behavior. + """ + + def setup(self, cfg: dict) -> None: + """Optional one-time initialization with the full trainer config.""" + + def set_out_root(self, out_root: str) -> None: + """Optional hook for loaders that persist split files or state.""" + + def state_dict(self) -> dict[str, Any]: + """Return serializable loader state for resume support.""" + return {} + + def load_state_dict(self, state: dict[str, Any]) -> None: + """Restore loader state from :meth:`state_dict` output.""" + + def get_train_size(self) -> int | None: + """Return the size of the training pool when known.""" + return None + + @staticmethod + def make_base_seeds(steps_per_epoch: int, accumulation: int, seed: int) -> list[int]: + """Return the deterministic seed pool used to define train batches.""" + batches_per_epoch = steps_per_epoch * accumulation + return [seed + i + 1 for i in range(batches_per_epoch)] + + @staticmethod + def shuffle_epoch_seeds(base_seeds: list[int], epoch: int, seed: int) -> list[int]: + """Return the per-epoch deterministic shuffle of *base_seeds*.""" + epoch_rng = random.Random(seed + epoch * 1000) + shuffled = list(base_seeds) + epoch_rng.shuffle(shuffled) + return shuffled + + def plan_train_epoch( + self, + *, + epoch: int, + steps_per_epoch: int, + accumulation: int, + batch_size: int, + seed: int, + **kwargs, + ) -> list[BatchSpec]: + """Build the full list of training batches for one epoch.""" + base_seeds = self.make_base_seeds( + steps_per_epoch=steps_per_epoch, + accumulation=accumulation, + seed=seed, + ) + shuffled_seeds = self.shuffle_epoch_seeds(base_seeds, epoch=epoch, seed=seed) + return [ + self.build_train_batch(batch_size=batch_size, seed=batch_seed, **kwargs) + for batch_seed in shuffled_seeds + ] + + @abstractmethod + def build_train_batch(self, batch_size: int, seed: int, **kwargs) -> BatchSpec: + """Construct one training batch specification.""" + + @abstractmethod + def build_eval_batch( + self, + env_num: int, + split: str, + seed: int, + **kwargs, + ) -> BatchSpec: + """Construct one evaluation batch specification.""" + + +# ── Split-based dataloader for dataset-backed environments ────────────── + +# Canonical split names expected under split_dir/ +SPLIT_NAMES = ("train", "val", "test") + +# Maps legacy / trainer split names → canonical directory names +_SPLIT_ALIAS: dict[str, str] = { + "train": "train", + "valid_seen": "val", + "selection": "val", + "val": "val", + "valid_unseen": "test", + "test": "test", +} + + +def _load_json_or_jsonl(path: str) -> list[dict]: + """Load a list of items from a JSON or JSONL file.""" + with open(path, encoding="utf-8") as f: + content = f.read().strip() + if not content: + return [] + + try: + data = json.loads(content) + except json.JSONDecodeError: + data = None + + if isinstance(data, list): + return data + if isinstance(data, dict): + nested = data.get("data") + if isinstance(nested, list): + return nested + return list(data.values()) + + items: list[dict] = [] + for line in content.splitlines(): + line = line.strip() + if line: + items.append(json.loads(line)) + return items + + +def _parse_split_ratio(text: str) -> tuple[int, int, int]: + parts = [part.strip() for part in str(text or "").split(":") if part.strip()] + if len(parts) != 3: + raise ValueError( + f"split_ratio must be in train:val:test form, got {text!r}" + ) + try: + train, val, test = (int(part) for part in parts) + except ValueError as exc: + raise ValueError( + f"split_ratio must contain integers, got {text!r}" + ) from exc + if min(train, val, test) <= 0: + raise ValueError(f"split_ratio parts must be positive, got {text!r}") + return train, val, test + + +def _compute_split_counts(total: int, ratio: tuple[int, int, int]) -> tuple[int, int, int]: + weights = list(ratio) + denom = sum(weights) + raw = [total * weight / denom for weight in weights] + counts = [int(value) for value in raw] + remaining = total - sum(counts) + order = sorted( + range(len(raw)), + key=lambda idx: (raw[idx] - counts[idx], weights[idx]), + reverse=True, + ) + for idx in order[:remaining]: + counts[idx] += 1 + return counts[0], counts[1], counts[2] + + +class SplitDataLoader(BaseDataLoader): + """Base class for dataset-backed environments. + + Supported modes: + + - ``split_mode="split_dir"``: load an existing ``train/``, ``val/``, + ``test/`` directory tree. + - ``split_mode="ratio"``: load raw items from ``data_path`` and materialize + a deterministic split directory with the requested ratio. + """ + + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "ratio", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + seed: int = 42, + limit: int = 0, + **kwargs, + ) -> None: + self.split_dir = split_dir + self.data_path = data_path + self.split_mode = split_mode + self.split_ratio = split_ratio + self.split_seed = int(split_seed) + self.split_output_dir = split_output_dir + self.seed = seed + self.limit = limit + self._splits: dict[str, list[dict]] = {} + + # ── Setup ──────────────────────────────────────────────────────────── + + def setup(self, cfg: dict) -> None: + if not self.split_mode: + self.split_mode = str(cfg.get("split_mode", "ratio") or "ratio") + if not self.split_dir: + self.split_dir = cfg.get("split_dir", "") + if not self.data_path: + self.data_path = cfg.get("data_path", "") + if not self.split_output_dir: + self.split_output_dir = cfg.get("split_output_dir", "") + if "split_seed" in cfg and not self.split_seed: + self.split_seed = int(cfg.get("split_seed", 0) or 0) + if not self.split_seed: + self.split_seed = self.seed + if not self.split_ratio: + self.split_ratio = str(cfg.get("split_ratio", "2:1:7") or "2:1:7") + + mode = str(self.split_mode or "ratio").strip().lower() + if mode not in {"ratio", "split_dir"}: + raise ValueError( + f"{type(self).__name__} split_mode must be 'ratio' or 'split_dir', " + f"got {self.split_mode!r}" + ) + self.split_mode = mode + + if self.split_mode == "ratio": + self.split_dir = self._materialize_ratio_split(cfg) + if not self.split_dir: + raise ValueError( + f"{type(self).__name__} requires either " + "`split_mode=ratio` with `data_path`, or `split_mode=split_dir` " + f"with `split_dir` pointing to {'/'.join(SPLIT_NAMES)}/." + ) + self._load_all_splits() + + def _resolve_split_output_dir(self, cfg: dict) -> str: + if self.split_output_dir: + return os.path.abspath(self.split_output_dir) + out_root = os.path.abspath(str(cfg.get("out_root") or os.getcwd())) + env_name = str(cfg.get("env") or type(self).__name__.replace("DataLoader", "").lower()) + ratio_tag = str(self.split_ratio or "2:1:7").replace(":", "-") + return os.path.join(out_root, "_generated_splits", f"{env_name}_{ratio_tag}_seed{self.split_seed}") + + def load_raw_items(self, data_path: str) -> list[dict]: + """Load raw items from a dataset path before ratio splitting. + + Subclasses can override when the raw dataset is not a single JSON/JSONL + file or when directory layouts require custom normalization. + """ + if os.path.isdir(data_path): + if any(os.path.isdir(os.path.join(data_path, name)) for name in SPLIT_NAMES): + raise ValueError( + f"{type(self).__name__} got a split directory as data_path. " + "Use split_mode=split_dir and pass it as split_dir instead." + ) + candidates = sorted(glob.glob(os.path.join(data_path, "*.json"))) + candidates += sorted(glob.glob(os.path.join(data_path, "*.jsonl"))) + if len(candidates) != 1: + raise ValueError( + f"{type(self).__name__} expected data_path to be one JSON/JSONL file " + f"or a directory containing exactly one such file, got: {data_path}" + ) + return _load_json_or_jsonl(candidates[0]) + return _load_json_or_jsonl(data_path) + + def write_split_items(self, split_path: str, items: list[dict]) -> None: + os.makedirs(split_path, exist_ok=True) + out_path = os.path.join(split_path, "items.json") + with open(out_path, "w", encoding="utf-8") as f: + json.dump(items, f, ensure_ascii=False, indent=2) + + def _materialize_ratio_split(self, cfg: dict) -> str: + data_path = os.path.abspath(str(self.data_path or "").strip()) + if not data_path: + raise ValueError( + f"{type(self).__name__} requires data_path when split_mode=ratio." + ) + + ratio = _parse_split_ratio(self.split_ratio) + items = self.load_raw_items(data_path) + if not isinstance(items, list) or not items: + raise ValueError(f"No raw items available for ratio split from {data_path}") + + shuffled = list(items) + rng = random.Random(self.split_seed) + rng.shuffle(shuffled) + + train_n, val_n, test_n = _compute_split_counts(len(shuffled), ratio) + train_items = shuffled[:train_n] + val_items = shuffled[train_n: train_n + val_n] + test_items = shuffled[train_n + val_n: train_n + val_n + test_n] + + split_dir = self._resolve_split_output_dir(cfg) + manifest = { + "source_data_path": data_path, + "split_mode": "ratio", + "split_ratio": self.split_ratio, + "split_seed": self.split_seed, + "counts": { + "train": len(train_items), + "val": len(val_items), + "test": len(test_items), + }, + } + os.makedirs(split_dir, exist_ok=True) + self.write_split_items(os.path.join(split_dir, "train"), train_items) + self.write_split_items(os.path.join(split_dir, "val"), val_items) + self.write_split_items(os.path.join(split_dir, "test"), test_items) + with open(os.path.join(split_dir, "split_manifest.json"), "w", encoding="utf-8") as f: + json.dump(manifest, f, ensure_ascii=False, indent=2) + print( + f" [{type(self).__name__}] generated ratio split {self.split_ratio} " + f"at {split_dir} from {data_path}" + ) + return split_dir + + def _load_all_splits(self) -> None: + for name in SPLIT_NAMES: + split_path = os.path.join(self.split_dir, name) + if not os.path.isdir(split_path): + raise ValueError( + f"Missing '{name}/' subdirectory in split_dir: {self.split_dir}" + ) + items = self.load_split_items(split_path) + if self.limit: + items = items[: self.limit] + self._splits[name] = items + + counts = " ".join(f"{k}={len(v)}" for k, v in self._splits.items()) + print(f" [{type(self).__name__}] {counts} (from {self.split_dir})") + + def load_split_items(self, split_path: str) -> list[dict]: + """Load items from one split directory (e.g. ``split_dir/train/``). + + Default: finds the first ``.json`` file in the directory and loads it + as a JSON array. Subclasses can override for custom formats. + """ + json_files = sorted(glob.glob(os.path.join(split_path, "*.json"))) + if not json_files: + raise FileNotFoundError( + f"No .json file found in {split_path}" + ) + with open(json_files[0], encoding="utf-8") as f: + items = json.load(f) + if not isinstance(items, list): + raise ValueError( + f"Expected JSON array in {json_files[0]}, got {type(items).__name__}" + ) + return items + + # ── Accessors ──────────────────────────────────────────────────────── + + @property + def train_items(self) -> list[dict]: + return self._splits.get("train", []) + + @property + def val_items(self) -> list[dict]: + return self._splits.get("val", []) + + @property + def test_items(self) -> list[dict]: + return self._splits.get("test", []) + + def get_split_items(self, split: str) -> list[dict]: + """Resolve a split name (including legacy aliases) to its item list.""" + canonical = _SPLIT_ALIAS.get(split, split) + return list(self._splits.get(canonical, self.val_items)) + + def get_train_size(self) -> int: + return len(self.train_items) + + def plan_train_epoch( + self, + *, + epoch: int, + steps_per_epoch: int, + accumulation: int, + batch_size: int, + seed: int, + **kwargs, + ) -> list[BatchSpec]: + """Build one full epoch that covers the train split in shuffled order. + + For split-backed datasets, an epoch should correspond to one pass over + the available training items rather than repeated independent sampling. + """ + epoch_rng = random.Random(seed + epoch * 1000) + items = list(self.train_items) + epoch_rng.shuffle(items) + + total_batches = steps_per_epoch * accumulation + if total_batches <= 0: + return [] + + batches: list[BatchSpec] = [] + cursor = 0 + for batch_idx in range(total_batches): + batch_items = items[cursor: cursor + batch_size] + cursor += len(batch_items) + + # Extremely small datasets can leave trailing empty microbatches + # when accumulation > 1. Reuse the shuffled prefix in that case so + # the trainer still receives the expected batch count. + if not batch_items and items: + refill_rng = random.Random(seed + epoch * 1000 + batch_idx + 1) + batch_items = list(items) + refill_rng.shuffle(batch_items) + batch_items = batch_items[:batch_size] + + batches.append( + BatchSpec( + phase="train", + split="train", + seed=seed + epoch * 1000 + batch_idx + 1, + batch_size=len(batch_items), + payload=batch_items, + ) + ) + + return batches + + # ── Batch construction ─────────────────────────────────────────────── + + def build_train_batch(self, batch_size: int, seed: int, **kwargs) -> BatchSpec: + rng = random.Random(seed) + items = list(self.train_items) + rng.shuffle(items) + items = items[:batch_size] + return BatchSpec( + phase="train", + split="train", + seed=seed, + batch_size=len(items), + payload=items, + ) + + def build_eval_batch( + self, + env_num: int, + split: str, + seed: int, + **kwargs, + ) -> BatchSpec: + items = self.get_split_items(split) + if env_num and env_num < len(items): + items = items[:env_num] + return BatchSpec( + phase="eval", + split=split, + seed=seed, + batch_size=len(items), + payload=items, + ) diff --git a/skillopt/engine/__init__.py b/skillopt/engine/__init__.py new file mode 100644 index 0000000..b876e70 --- /dev/null +++ b/skillopt/engine/__init__.py @@ -0,0 +1,9 @@ +"""ReflACT Engine -- the training runner. + +Analogous to the Runner in mmengine: orchestrates the full training pipeline +including rollout, gradient computation, aggregation, optimization, and +evaluation. +""" +from skillopt.engine.trainer import ReflACTTrainer # noqa: F401 + +__all__ = ["ReflACTTrainer"] diff --git a/skillopt/engine/trainer.py b/skillopt/engine/trainer.py new file mode 100644 index 0000000..8312cdb --- /dev/null +++ b/skillopt/engine/trainer.py @@ -0,0 +1,2406 @@ +"""ReflACT Trainer — the main training loop. + +Orchestrates the 6-stage ReflACT pipeline: + 1. Rollout — execute episodes with current skill + 2. Reflect — analyze trajectories, generate patches + 3. Aggregate — hierarchical merge of patches + 4. Select — rank and select top edits + 5. Update — apply edits to skill document + 6. Evaluate — validate candidate skill, accept/reject + +The trainer is environment-agnostic; all environment-specific logic is +delegated to an :class:`~skillopt.envs.base.EnvAdapter` instance. +""" +from __future__ import annotations + +import glob +import json +import math +import os +import random +import re +import time +from collections import defaultdict + +from skillopt.datasets.base import BatchSpec +from skillopt.envs.base import EnvAdapter +from skillopt.evaluation.gate import GateResult, evaluate_gate, select_gate_score +from skillopt.gradient.aggregate import merge_patches +from skillopt.optimizer.meta_skill import run_meta_skill +from skillopt.optimizer.clip import rank_and_select +from skillopt.optimizer.lr_autonomous import decide_autonomous_learning_rate +from skillopt.optimizer.rewrite import rewrite_skill_from_suggestions +from skillopt.optimizer.scheduler import build_scheduler +from skillopt.optimizer.skill import apply_patch_with_report +from skillopt.optimizer.appendix import ( + append_to_appendix_field, + extract_appendix_notes as extract_appendix_notes_from_skill, + inject_empty_appendix_field, + _strip_all_appendix_fields, +) +from skillopt.optimizer.skill_aware import ( + configure_skill_aware_reflection, + consolidate_appendix_notes, + extract_appendix_notes as extract_appendix_notes_from_result, +) +from skillopt.optimizer.slow_update import ( + build_comparison_pairs, + extract_slow_update_field, + inject_empty_slow_update_field, + replace_slow_update_field, + run_slow_update, + save_comparison_pairs, +) +from skillopt.optimizer.update_modes import ( + get_payload_items, + is_full_rewrite_minibatch_mode, + normalize_update_mode, + payload_label, + short_item_summary, +) +from skillopt.model import ( + chat_optimizer, + configure_azure_openai, + configure_claude_code_exec, + configure_codex_exec, + configure_minimax_chat, + configure_qwen_chat, + get_token_summary, + reset_token_tracker, + set_reasoning_effort, + set_target_backend, + set_target_deployment, + set_optimizer_backend, + set_optimizer_deployment, +) +from skillopt.utils import compute_score, skill_hash + + +# ── Skill-aware reflection: appendix flush ─────────────────────────────────── + +def _flush_skill_aware_appendix( + current_skill: str, + all_raw_patches: list, + step_rec: dict, + step_dir: str, + cfg: dict, +) -> str: + """Append this step's EXECUTION_LAPSE notes into the protected appendix. + + Returns the (possibly) updated skill. Must be called on BOTH the normal + update path and the skip branches: a lapse-only step yields no body + patches by design (analysts return ``edits: []`` carriers), so the skip + paths would otherwise silently drop every note of the step. + """ + step_appendix_notes: list[str] = [] + for rp in all_raw_patches: + if isinstance(rp, dict): + step_appendix_notes.extend(extract_appendix_notes_from_result(rp)) + if not step_appendix_notes: + return current_skill + + before_notes = extract_appendix_notes_from_skill(current_skill) + current_skill = append_to_appendix_field( + current_skill, step_appendix_notes, + ) + after_notes = extract_appendix_notes_from_skill(current_skill) + n_added = len(after_notes) - len(before_notes) + step_rec["n_execution_lapse_notes"] = len(step_appendix_notes) + step_rec["n_appendix_notes_added"] = n_added + step_rec["n_appendix_notes_total"] = len(after_notes) + with open(os.path.join(step_dir, "appendix_notes.json"), "w") as f: + json.dump( + { + "step_notes": step_appendix_notes, + "appendix_after": after_notes, + }, + f, indent=2, ensure_ascii=False, + ) + print( + f" [skill-aware] +{n_added} appendix note(s) " + f"(total {len(after_notes)}) from {len(step_appendix_notes)} lapse signal(s)" + ) + # Threshold-gated LLM consolidation (paper Eq.11): when the + # appendix grows past N notes, compact it with one optimizer + # call (dedupe / merge / shorten). 0 disables it. Any failure + # leaves the appendix unchanged. + consolidate_threshold = int( + cfg.get("skill_aware_consolidate_threshold", 0) or 0 + ) + if consolidate_threshold > 0 and len(after_notes) > consolidate_threshold: + compacted = consolidate_appendix_notes( + after_notes, chat_fn=chat_optimizer, + ) + if compacted and len(compacted) < len(after_notes): + current_skill = append_to_appendix_field( + _strip_all_appendix_fields(current_skill), compacted, + ) + step_rec["n_appendix_notes_consolidated"] = len(compacted) + step_rec["n_appendix_notes_total"] = len(compacted) + print( + f" [skill-aware] consolidated appendix " + f"{len(after_notes)} -> {len(compacted)} notes" + ) + return current_skill + + +# ── Patch normalization ─────────────────────────────────────────────────────── + +def _normalise_patches( + raw_patches: list[dict | None], + update_mode: str = "patch", +) -> tuple[list[dict], list[dict]]: + """Extract inner 'patch' sub-dict, split into failure/success lists. + + Each element is expected to conform to :class:`~skillopt.types.RawPatch`. + """ + mode = normalize_update_mode(update_mode) + failure: list[dict] = [] + success: list[dict] = [] + for p in raw_patches: + if not isinstance(p, dict): + continue + inner = p.get("patch", p) + if not isinstance(inner, dict): + continue + items = get_payload_items(inner, mode) + if not items: + continue + support = max(int(p.get("batch_size", 0) or 0), 1) + for item in items: + if isinstance(item, dict): + item.setdefault("source_type", p.get("source_type", "failure")) + item.setdefault("support_count", support) + if p.get("source_type", "failure") == "success": + success.append(inner) + else: + failure.append(inner) + return failure, success + + +def _normalise_longitudinal_pair_policy(policy: str | None) -> str: + raw = str(policy or "mixed").strip().lower() + aliases = { + "mixed": "mixed", + "default": "mixed", + "random": "mixed", + "all": "mixed", + "changed": "changed", + "change": "changed", + "delta": "changed", + "10_01": "changed", + "01_10": "changed", + "unchanged": "unchanged", + "stable": "unchanged", + "same": "unchanged", + "00_11": "unchanged", + } + if raw not in aliases: + raise ValueError( + "optimizer.longitudinal_pair_policy must be one of " + "mixed, changed, unchanged" + ) + return aliases[raw] + + +def _normalise_lr_control_mode(mode: str | None) -> str: + raw = str(mode or "fixed").strip().lower() + aliases = { + "fixed": "fixed", + "manual": "fixed", + "scheduler": "fixed", + "scheduled": "fixed", + "autonomous": "autonomous", + "auto": "autonomous", + "optimizer": "autonomous", + "none": "none", + "off": "none", + "no_lr": "none", + } + if raw not in aliases: + raise ValueError("optimizer.lr_control_mode must be one of fixed, autonomous, none") + return aliases[raw] + + +def _filter_longitudinal_pairs(pairs: list[dict], policy: str) -> list[dict]: + if policy == "mixed": + return pairs + if policy == "changed": + keep = {"improved", "regressed"} + elif policy == "unchanged": + keep = {"persistent_fail", "stable_success"} + else: + raise ValueError(f"Unknown longitudinal pair policy: {policy}") + return [p for p in pairs if p.get("category") in keep] + + +def _pair_category_counts(pairs: list[dict]) -> dict[str, int]: + counts = { + "improved": 0, + "regressed": 0, + "persistent_fail": 0, + "stable_success": 0, + } + for pair in pairs: + cat = str(pair.get("category", "")) + counts[cat] = counts.get(cat, 0) + 1 + return counts + + +def _safe_pair_id(value: str) -> str: + safe = re.sub(r"[^A-Za-z0-9_.-]+", "_", str(value)).strip("_") + return safe[:80] or "item" + + +def _build_longitudinal_pairs( + *, + adapter: EnvAdapter, + dataloader, + prev_skill: str, + curr_skill: str, + initial_items: list[dict], + initial_prev_results: list[dict], + initial_curr_results: list[dict], + prev_rollout_dir: str, + curr_rollout_dir: str, + policy: str, + target_n: int, + seed: int, + out_root: str, +) -> tuple[list[dict], list[dict]]: + """Build longitudinal pairs, optionally filtering by change category. + + ``mixed`` preserves the legacy behavior exactly. ``changed`` keeps only + 10/01 pairs and attempts to top up to ``target_n`` by scanning the train + split once. ``unchanged`` keeps only 00/11 pairs and does not top up. + """ + all_pairs = build_comparison_pairs( + initial_prev_results, + initial_curr_results, + initial_items, + prev_rollout_dir=prev_rollout_dir, + curr_rollout_dir=curr_rollout_dir, + ) + selected_pairs = _filter_longitudinal_pairs(all_pairs, policy) + if policy != "changed" or len(selected_pairs) >= target_n or dataloader is None: + return selected_pairs, all_pairs + + train_items = list(getattr(dataloader, "train_items", []) or []) + if not train_items: + return selected_pairs, all_pairs + + seen_ids = {str(p.get("id", "")) for p in all_pairs} + rng = random.Random(seed) + candidates = list(train_items) + rng.shuffle(candidates) + candidates = [item for item in candidates if str(item.get("id", "")) not in seen_ids] + + for idx, item in enumerate(candidates): + if len(selected_pairs) >= target_n: + break + item_id = _safe_pair_id(str(item.get("id", f"item_{idx}"))) + batch = BatchSpec( + phase="train", + split="train", + seed=seed + idx + 1, + batch_size=1, + payload=[item], + ) + env = adapter.build_env_from_batch(batch, out_root=out_root) + prev_dir = os.path.join(prev_rollout_dir, "topup", item_id) + curr_dir = os.path.join(curr_rollout_dir, "topup", item_id) + prev_results = adapter.rollout(env, prev_skill, prev_dir) + curr_results = adapter.rollout(env, curr_skill, curr_dir) + pair = build_comparison_pairs( + prev_results, + curr_results, + [item], + prev_rollout_dir=prev_dir, + curr_rollout_dir=curr_dir, + ) + all_pairs.extend(pair) + selected_pairs.extend(_filter_longitudinal_pairs(pair, policy)) + + return selected_pairs[:target_n], all_pairs + + +# ── History / persistence helpers ───────────────────────────────────────────── + +_SECRET_KEYS = { + "azure_api_key", + "api_key", + "openai_api_key", +} + + +def _redact_value(val: str) -> str: + if len(val) <= 8: + return "*" * len(val) + return f"{val[:4]}...{val[-4:]}" + + +def _redact_cfg(cfg: dict) -> dict: + redacted = dict(cfg) + for key in list(redacted): + if key.lower() in _SECRET_KEYS and redacted.get(key): + redacted[key] = _redact_value(str(redacted[key])) + return redacted + +def _load_history(out_root: str) -> list[dict]: + path = os.path.join(out_root, "history.json") + if os.path.exists(path): + with open(path) as f: + return json.load(f) + return [] + + +def _save_history(out_root: str, history: list[dict]) -> None: + path = os.path.join(out_root, "history.json") + with open(path, "w") as f: + json.dump(history, f, ensure_ascii=False, indent=2) + + +def _save_skill(out_root: str, step: int, content: str) -> None: + skills_dir = os.path.join(out_root, "skills") + os.makedirs(skills_dir, exist_ok=True) + with open(os.path.join(skills_dir, f"skill_v{step:04d}.md"), "w") as f: + f.write(content) + + +def _load_skill(out_root: str, step: int) -> str: + path = os.path.join(out_root, "skills", f"skill_v{step:04d}.md") + with open(path) as f: + return f.read() + + +def _load_meta_skill_content(out_root: str, epoch: int) -> str: + if epoch <= 0: + return "" + path = os.path.join( + out_root, "meta_skill", f"epoch_{epoch:02d}", "meta_skill_result.json", + ) + if not os.path.exists(path): + return "" + try: + with open(path) as f: + result = json.load(f) + return str(result.get("meta_skill_content", "")).strip() + except Exception: + return "" + + +def _load_runtime_state(out_root: str) -> dict | None: + path = os.path.join(out_root, "runtime_state.json") + if not os.path.exists(path): + return None + try: + with open(path) as f: + state = json.load(f) + return state if isinstance(state, dict) else None + except Exception: + return None + + +def _save_runtime_state(out_root: str, state: dict) -> None: + path = os.path.join(out_root, "runtime_state.json") + with open(path, "w") as f: + json.dump(state, f, ensure_ascii=False, indent=2) + + +def _resolve_train_size(cfg: dict, dataloader) -> int: + configured = int(cfg.get("train_size", 0) or 0) + inferred: int | None = None + + if dataloader is not None: + getter = getattr(dataloader, "get_train_size", None) + if callable(getter): + try: + value = getter() + except Exception: + value = None + if value is not None: + inferred = int(value) + elif hasattr(dataloader, "train_items"): + try: + inferred = len(getattr(dataloader, "train_items")) + except Exception: + inferred = None + + if inferred is not None and inferred <= 0: + inferred = None + + if configured > 0 and inferred is not None and configured != inferred: + raise ValueError( + f"Configured train_size={configured} does not match loaded train split " + f"size={inferred}. Fix the config or the dataset split." + ) + + train_size = configured if configured > 0 else inferred + if train_size is None or train_size <= 0: + raise ValueError( + "Unable to determine train_size automatically. " + "Provide train.train_size in the config for this environment." + ) + return int(train_size) + + +def _compute_task_type_buckets(results: list[dict], task_types: list[str]) -> dict[str, dict]: + """Compute per-task-type success rates.""" + buckets: dict[str, dict] = {} + for task in task_types + ["overall"]: + buckets[task] = {"total": 0, "hard": 0, "soft": 0.0} + for r in results: + tt = r.get("task_type", "other") + for key in [tt, "overall"]: + if key not in buckets: + buckets[key] = {"total": 0, "hard": 0, "soft": 0.0} + buckets[key]["total"] += 1 + buckets[key]["hard"] += float(r.get("hard", 0)) + buckets[key]["soft"] += float(r.get("soft", 0.0)) + return buckets + + +def _format_rejection_buffer(buffer: list[dict]) -> str: + """**DEPRECATED** — kept for backward compat; use _format_step_buffer.""" + return _format_step_buffer(buffer) + + +def _extract_failure_patterns( + rollout_results: list[dict], + step_dir: str, +) -> list[dict]: + """Extract compact failure patterns from rollout results. + + Uses analyst ``failure_summary`` from minibatch patches when available, + otherwise falls back to ``fail_reason`` prefix grouping. + """ + failures = [r for r in rollout_results if not r.get("hard") or float(r.get("hard", 0)) < 1e-9] + if not failures: + return [] + + # Group by fail_reason prefix + groups: dict[str, list[dict]] = defaultdict(list) + for r in failures: + reason = r.get("fail_reason", "unknown") + prefix = reason.split(":")[0].strip() if ":" in reason else reason + groups[prefix].append(r) + + # Try richer descriptions from analyst patches + analyst_descs: list[str] = [] + patch_globs = [ + os.path.join(step_dir, "patches", "minibatch_fail_*.json"), + os.path.join(step_dir, "batch_*", "patches", "minibatch_fail_*.json"), + ] + seen_patch_files: set[str] = set() + for pattern in patch_globs: + for fname in sorted(glob.glob(pattern)): + if fname in seen_patch_files: + continue + seen_patch_files.add(fname) + try: + with open(fname) as f: + patch = json.load(f) + for fs in patch.get("failure_summary", []): + ft = fs.get("failure_type", "") + sd = fs.get("description", "") + analyst_descs.append(f"{ft}: {sd}" if sd else ft) + except Exception: + pass + + patterns = [] + desc_iter = iter(analyst_descs) + for prefix, items in groups.items(): + desc = next(desc_iter, None) or prefix + patterns.append({ + "pattern": desc, + "count": len(items), + "task_ids": [str(r.get("id", "?")) for r in items], + }) + return patterns + + +def _format_step_buffer(buffer: list[dict]) -> str: + """Format the unified step buffer into a single context block. + + Each entry captures what happened at a previous step: failure patterns + observed during rollout, and — when the step was rejected — the specific + edits that were tried and the resulting score drop. + + Returns empty string when *buffer* is empty. + """ + if not buffer: + return "" + + parts = [ + "Below is a summary of previous steps in this epoch. " + "Use it to avoid repeating ineffective edits and to prioritise " + "failure patterns that remain unsolved.\n" + ] + + for entry in buffer: + step = entry["step"] + action = entry["action"] + n_fail = entry.get("n_fail", 0) + n_total = entry.get("n_total", "?") + + parts.append(f"### Step {step} — {action.upper()} ({n_fail}/{n_total} failed)") + + # Failure patterns + for p in entry.get("failure_patterns", []): + ids = ", ".join(p["task_ids"]) + parts.append(f' - "{p["pattern"]}" (×{p["count"]}, tasks: {ids})') + + # Rejected edits (only present on reject) + rejected = entry.get("rejected_edits", []) + if rejected: + score_before = entry.get("score_before", "?") + score_after = entry.get("score_after", "?") + parts.append( + f" Rejected edits (score {score_before} → {score_after}):" + ) + for i, e in enumerate(rejected, 1): + if e.get("op") is not None: + op = e.get("op", "?") + content = e.get("content", "") + target = e.get("target", "") + if target: + parts.append(f' {i}. [{op}] target="{target}" → "{content}"') + else: + parts.append(f' {i}. [{op}] "{content}"') + else: + kind = e.get("type", "?") + title = e.get("title", "") + instruction = e.get("instruction", "") + parts.append(f' {i}. [{kind}] "{title}" → "{instruction}"') + + return "\n".join(parts) + + +# ── Trainer ────────────────────────────────────────────────────────────────── + +class ReflACTTrainer: + """Main ReflACT training loop. + + Parameters + ---------- + cfg : dict + Configuration dictionary. See ``configs/alfworld_default.yaml`` + for the full list of keys. + adapter : EnvAdapter + Environment adapter instance. + """ + + def __init__(self, cfg: dict, adapter: EnvAdapter) -> None: + self.cfg = cfg + self.adapter = adapter + + def train(self) -> dict: + """Execute the full ReflACT training loop. Returns summary dict.""" + cfg = self.cfg + adapter = self.adapter + out_root = cfg["out_root"] + os.makedirs(out_root, exist_ok=True) + + # ── Adapter setup (one-time init) ──────────────────────────── + adapter.setup(cfg) + dataloader = adapter.get_dataloader() + + def _build_train_env(batch: BatchSpec): + env_manager = adapter.build_env_from_batch(batch, out_root=out_root) + return env_manager, batch.batch_size, batch.seed + + def _build_eval_env(split: str, env_num: int, seed: int): + if dataloader is None: + env_manager = adapter.build_eval_env( + env_num=env_num, + split=split, + seed=seed, + out_root=out_root, + ) + actual_n = len(env_manager) if hasattr(env_manager, "__len__") else env_num + return env_manager, actual_n + + batch = dataloader.build_eval_batch( + env_num=env_num, + split=split, + seed=seed, + out_root=out_root, + ) + env_manager = adapter.build_env_from_batch(batch, out_root=out_root) + return env_manager, batch.batch_size + + # ── Configure models ───────────────────────────────────────────── + backend = cfg.get("model_backend", "azure_openai") + configure_azure_openai( + endpoint=( + cfg.get("azure_openai_endpoint") + or cfg.get("azure_endpoint") + or None + ), + api_version=( + cfg.get("azure_openai_api_version") + or cfg.get("azure_api_version") + or None + ), + api_key=( + cfg.get("azure_openai_api_key") + or cfg.get("azure_api_key") + or None + ), + auth_mode=cfg.get("azure_openai_auth_mode") or None, + ad_scope=cfg.get("azure_openai_ad_scope") or None, + managed_identity_client_id=cfg.get("azure_openai_managed_identity_client_id") or None, + optimizer_endpoint=cfg.get("optimizer_azure_openai_endpoint") or None, + optimizer_api_version=cfg.get("optimizer_azure_openai_api_version") or None, + optimizer_api_key=cfg.get("optimizer_azure_openai_api_key") or None, + optimizer_auth_mode=cfg.get("optimizer_azure_openai_auth_mode") or None, + optimizer_ad_scope=cfg.get("optimizer_azure_openai_ad_scope") or None, + optimizer_managed_identity_client_id=( + cfg.get("optimizer_azure_openai_managed_identity_client_id") or None + ), + target_endpoint=cfg.get("target_azure_openai_endpoint") or None, + target_api_version=cfg.get("target_azure_openai_api_version") or None, + target_api_key=cfg.get("target_azure_openai_api_key") or None, + target_auth_mode=cfg.get("target_azure_openai_auth_mode") or None, + target_ad_scope=cfg.get("target_azure_openai_ad_scope") or None, + target_managed_identity_client_id=( + cfg.get("target_azure_openai_managed_identity_client_id") or None + ), + ) + optimizer_backend = cfg.get("optimizer_backend") + target_backend = cfg.get("target_backend") + if not optimizer_backend or not target_backend: + if backend in {"claude", "claude_chat"}: + optimizer_backend = optimizer_backend or "claude_chat" + target_backend = target_backend or "claude_chat" + elif backend in {"codex", "codex_exec"}: + optimizer_backend = optimizer_backend or "openai_chat" + target_backend = target_backend or "codex_exec" + elif backend == "claude_code_exec": + optimizer_backend = optimizer_backend or "openai_chat" + target_backend = target_backend or "claude_code_exec" + elif backend in {"qwen", "qwen_chat"}: + optimizer_backend = optimizer_backend or "openai_chat" + target_backend = target_backend or "qwen_chat" + else: + optimizer_backend = optimizer_backend or "openai_chat" + target_backend = target_backend or "openai_chat" + cfg["optimizer_backend"] = optimizer_backend + cfg["target_backend"] = target_backend + set_optimizer_backend(optimizer_backend) + set_target_backend(target_backend) + set_optimizer_deployment(cfg["optimizer_model"]) + set_target_deployment(cfg["target_model"]) + configure_codex_exec( + path=cfg.get("codex_exec_path", "codex"), + sandbox=cfg.get("codex_exec_sandbox", "workspace-write"), + profile=cfg.get("codex_exec_profile", ""), + full_auto=cfg.get("codex_exec_full_auto", False), + reasoning_effort=cfg.get("codex_exec_reasoning_effort", "none"), + use_sdk=cfg.get("codex_exec_use_sdk", None), + network_access=cfg.get("codex_exec_network_access", False), + web_search=cfg.get("codex_exec_web_search", False), + approval_policy=cfg.get("codex_exec_approval_policy", "never"), + ) + configure_claude_code_exec( + path=cfg.get("claude_code_exec_path", "claude"), + profile=cfg.get("claude_code_exec_profile", ""), + use_sdk=cfg.get("claude_code_exec_use_sdk", None), + effort=cfg.get("claude_code_exec_effort", cfg.get("reasoning_effort", "medium")), + max_thinking_tokens=cfg.get("claude_code_exec_max_thinking_tokens", 16384), + ) + configure_qwen_chat( + base_url=cfg.get("qwen_chat_base_url") or None, + api_key=cfg.get("qwen_chat_api_key") or None, + temperature=cfg.get("qwen_chat_temperature"), + timeout_seconds=cfg.get("qwen_chat_timeout_seconds"), + max_tokens=cfg.get("qwen_chat_max_tokens"), + enable_thinking=cfg.get("qwen_chat_enable_thinking"), + optimizer_base_url=cfg.get("optimizer_qwen_chat_base_url") or None, + optimizer_api_key=cfg.get("optimizer_qwen_chat_api_key") or None, + optimizer_temperature=cfg.get("optimizer_qwen_chat_temperature"), + optimizer_timeout_seconds=cfg.get("optimizer_qwen_chat_timeout_seconds"), + optimizer_max_tokens=cfg.get("optimizer_qwen_chat_max_tokens"), + optimizer_enable_thinking=cfg.get("optimizer_qwen_chat_enable_thinking"), + target_base_url=cfg.get("target_qwen_chat_base_url") or None, + target_api_key=cfg.get("target_qwen_chat_api_key") or None, + target_temperature=cfg.get("target_qwen_chat_temperature"), + target_timeout_seconds=cfg.get("target_qwen_chat_timeout_seconds"), + target_max_tokens=cfg.get("target_qwen_chat_max_tokens"), + target_enable_thinking=cfg.get("target_qwen_chat_enable_thinking"), + ) + configure_minimax_chat( + base_url=cfg.get("minimax_base_url") or None, + api_key=cfg.get("minimax_api_key") or None, + temperature=cfg.get("minimax_temperature"), + max_tokens=cfg.get("minimax_max_tokens"), + enable_thinking=cfg.get("minimax_enable_thinking"), + ) + minimax_model_cfg = cfg.get("minimax_model") + if minimax_model_cfg and cfg.get("target_backend") == "minimax_chat": + set_target_deployment(str(minimax_model_cfg)) + os.environ["REFLACT_CODEX_TRACE_TO_OPTIMIZER"] = ( + "1" + if target_backend == "codex_exec" and cfg.get("codex_trace_to_optimizer", False) + else "0" + ) + reasoning = cfg.get("reasoning_effort", "") or None + set_reasoning_effort(reasoning) + print( + f" [model config] backend={backend} " + f"optimizer={cfg['optimizer_model']} ({optimizer_backend}) " + f"target={cfg['target_model']} ({target_backend}) " + f"reasoning={reasoning or 'off'}" + ) + + # ── Initialize Ray ─────────────────────────────────────────────── + if adapter.requires_ray(): + try: + import ray + except ImportError as e: + raise ImportError( + "This environment requires ray, but ray is not installed." + ) from e + + if not ray.is_initialized(): + ray.init(num_gpus=0) + + # ── Load initial skill ─────────────────────────────────────────── + skill_init_path = os.path.abspath(cfg["skill_init"]) + if os.path.exists(skill_init_path): + with open(skill_init_path) as f: + skill_init = f.read() + print(f" [initial skill] {skill_init_path} ({len(skill_init)} chars)") + else: + skill_init = "" + print(" [initial skill] no initial skill file — starting from blank") + + # ── Training parameters ────────────────────────────────────────── + batch_size = cfg["batch_size"] + num_epochs = cfg["num_epochs"] + accumulation = cfg["accumulation"] + seed = cfg["seed"] + merge_bs = cfg["merge_batch_size"] + max_analyst_rounds = int(cfg.get("max_analyst_rounds", 3) or 3) + update_mode = normalize_update_mode(cfg.get("skill_update_mode", "patch")) + lr_control_mode = _normalise_lr_control_mode(cfg.get("lr_control_mode", "fixed")) + if is_full_rewrite_minibatch_mode(update_mode): + lr_control_mode = "none" + longitudinal_pair_policy = _normalise_longitudinal_pair_policy( + cfg.get("longitudinal_pair_policy", "mixed") + ) + rewrite_reasoning_effort = cfg.get("rewrite_reasoning_effort", "high") + if rewrite_reasoning_effort == "": + rewrite_reasoning_effort = None + rewrite_max_completion_tokens = int(cfg.get("rewrite_max_completion_tokens", 64000)) + if batch_size <= 0: + raise ValueError(f"batch_size must be positive, got {batch_size}") + if accumulation <= 0: + raise ValueError(f"accumulation must be positive, got {accumulation}") + + train_size = _resolve_train_size(cfg, dataloader) + steps_per_epoch = math.ceil(train_size / (batch_size * accumulation)) + batches_per_epoch = steps_per_epoch * accumulation + total_steps = num_epochs * steps_per_epoch + + # Persist resolved derived fields so config.json / summary.json match + # the actual runtime recipe. + cfg["train_size"] = train_size + cfg["steps_per_epoch"] = steps_per_epoch + cfg["batches_per_epoch"] = batches_per_epoch + cfg["samples_per_epoch"] = train_size + cfg["skill_update_mode"] = update_mode + cfg["lr_control_mode"] = lr_control_mode + + # Save config after deriving runtime values. + with open(os.path.join(out_root, "config.json"), "w") as f: + json.dump(_redact_cfg(cfg), f, indent=2, ensure_ascii=False) + + train_pool_size = train_size + + scheduler = build_scheduler( + mode=cfg.get("lr_scheduler", "constant"), + max_lr=cfg["edit_budget"], + min_lr=cfg.get("min_edit_budget", 2), + total_steps=total_steps, + ) + + # Fixed training pool: base seeds (each seed = one deterministic batch) + if dataloader is not None: + base_seeds = dataloader.make_base_seeds( + steps_per_epoch=steps_per_epoch, + accumulation=accumulation, + seed=seed, + ) + else: + base_seeds = [seed + i + 1 for i in range(batches_per_epoch)] + + print(f"\n [config] epochs={num_epochs} steps/epoch={steps_per_epoch} " + f"(auto) accum={accumulation} batch_size={batch_size}") + print(f" [config] train_size={train_size}") + print(f" [config] batches/epoch={batches_per_epoch} " + f"total_steps={total_steps} " + f"games/epoch={train_pool_size}") + print(f" [config] lr_scheduler={cfg.get('lr_scheduler', 'constant')} " + f"edit_budget={cfg['edit_budget']} " + f"min_edit_budget={cfg.get('min_edit_budget', 2)}") + print(f" [config] skill_update_mode={update_mode} " + f"lr_control_mode={lr_control_mode} " + f"rewrite_reasoning_effort={rewrite_reasoning_effort or 'off'} " + f"rewrite_max_completion_tokens={rewrite_max_completion_tokens} " + f"max_analyst_rounds={max_analyst_rounds}") + print(f" [config] longitudinal_pair_policy={longitudinal_pair_policy}") + print(f" [config] base_seeds={base_seeds}") + + # ── Resume check ───────────────────────────────────────────────── + history = _load_history(out_root) + runtime_state = _load_runtime_state(out_root) + if runtime_state: + last_step = int(runtime_state.get("last_completed_step", 0) or 0) + current_skill_path = runtime_state.get("current_skill_path") or os.path.join( + out_root, "skills", f"skill_v{last_step:04d}.md", + ) + with open(current_skill_path) as f: + current_skill = f.read() + best_skill_path = runtime_state.get("best_skill_path") or os.path.join( + out_root, "best_skill.md", + ) + if os.path.exists(best_skill_path): + with open(best_skill_path) as f: + best_skill = f.read() + else: + best_skill = current_skill + current_score = float(runtime_state.get("current_score", -1.0) or -1.0) + best_score = float(runtime_state.get("best_score", current_score) or current_score) + best_step = runtime_state.get("best_step", last_step) + current_origin = str( + runtime_state.get("current_origin") + or (f"step_{last_step:04d}" if last_step > 0 else "initial_skill") + ) + best_origin = str(runtime_state.get("best_origin") or current_origin) + resume_from = last_step + 1 + scheduler.load_state_dict({"current_step": last_step}) + print( + f" [resume] from step {resume_from} " + f"current={current_score:.4f} best={best_score:.4f} " + f"(origin={current_origin})" + ) + elif history: + last_step = history[-1]["step"] + current_skill = _load_skill(out_root, last_step) + best_rec = max(history, key=lambda h: h.get("best_score", 0.0)) + best_score = best_rec["best_score"] + best_step = best_rec["best_step"] + best_skill_path = os.path.join(out_root, "best_skill.md") + if os.path.exists(best_skill_path): + with open(best_skill_path) as f: + best_skill = f.read() + else: + best_skill = _load_skill(out_root, best_step) + current_score = history[-1].get("current_score", best_score) + current_origin = f"step_{last_step:04d}" + best_origin = f"step_{int(best_step):04d}" if isinstance(best_step, int) else str(best_step) + resume_from = last_step + 1 + scheduler.load_state_dict({"current_step": last_step}) + print( + f" [resume] from step {resume_from} " + f"current={current_score:.4f} best={best_score:.4f}" + ) + else: + current_skill = skill_init + best_skill = skill_init + best_score = -1.0 + current_score = -1.0 + best_step = 0 + current_origin = "initial_skill" + best_origin = "initial_skill" + resume_from = 1 + + _save_skill(out_root, 0, skill_init) + + use_skill_aware = cfg.get("use_skill_aware_reflection", False) + # Publish the toggle process-wide so run_minibatch_reflect resolves it + # from config for EVERY env adapter — no per-benchmark wiring needed. + configure_skill_aware_reflection( + use_skill_aware, + cfg.get("skill_aware_appendix_source", "both"), + ) + if use_skill_aware: + current_skill = inject_empty_appendix_field(current_skill) + + def _persist_runtime_state(last_completed_step: int) -> None: + _save_runtime_state( + out_root, + { + "last_completed_step": last_completed_step, + "current_skill_path": os.path.join( + out_root, "skills", f"skill_v{last_completed_step:04d}.md", + ), + "current_score": current_score, + "current_origin": current_origin, + "best_skill_path": os.path.join(out_root, "best_skill.md"), + "best_score": best_score, + "best_step": best_step, + "best_origin": best_origin, + }, + ) + + # ── Selection cache ────────────────────────────────────────────── + sel_cache: dict[str, tuple[float, float]] = {} + for rec in history: + sh = rec.get("candidate_hash", "") + if sh and rec.get("selection_hard") is not None: + sel_cache[sh] = (rec["selection_hard"], rec["selection_soft"]) + + # ── Baseline evaluation on selection set ───────────────────────── + # `use_gate=False` keeps validation running (selection rollout + + # scoring are unconditional below) but force-accepts every candidate + # instead of gating it; final skill is chosen manually afterwards. + use_gate = cfg.get("use_gate", True) is not False + gate_metric = str(cfg.get("gate_metric", "hard")).strip().lower() + if gate_metric not in {"hard", "soft", "mixed"}: + raise ValueError( + f"evaluation.gate_metric must be 'hard' | 'soft' | 'mixed', " + f"got {gate_metric!r}" + ) + gate_mixed_weight = float(cfg.get("gate_mixed_weight", 0.5)) + use_semantic_density = bool(cfg.get("use_semantic_density", False)) + semantic_density_weight = float(cfg.get("semantic_density_weight", 0.05)) + leading_words_raw = cfg.get("leading_words", None) + leading_words = None + if leading_words_raw is not None: + if isinstance(leading_words_raw, str): + leading_words = [w.strip() for w in leading_words_raw.split(",") if w.strip()] + else: + leading_words = list(leading_words_raw) + if not 0.0 <= gate_mixed_weight <= 1.0: + raise ValueError( + f"evaluation.gate_mixed_weight must be in [0, 1], " + f"got {gate_mixed_weight}" + ) + print( + f" [gate] metric={gate_metric}" + + ( + f" mixed_weight={gate_mixed_weight}" + if gate_metric == "mixed" + else "" + ) + + ("" if use_gate + else " (DISABLED → validation runs, candidates force-accepted)") + ) + slow_gate_with_selection = bool( + cfg.get("slow_update_gate_with_selection", False) + ) + print( + " [slow update] acceptance=" + + ("gated (selection-set validation)" + if slow_gate_with_selection + else "force-accept (unconditional)") + ) + if current_score < 0: + print(f"\n{'='*60}") + print(" BASELINE — evaluate initial skill on Selection set (valid_seen)") + print(f"{'='*60}") + sel_env, sel_n = _build_eval_env( + split="valid_seen", + env_num=cfg["sel_env_num"], + seed=seed, + ) + print(f" Selection items: {sel_n}") + baseline_dir = os.path.join(out_root, "selection_eval_baseline") + baseline_results = adapter.rollout(sel_env, skill_init, baseline_dir) + baseline_hard, baseline_soft = compute_score(baseline_results) + current_score = select_gate_score( + baseline_hard, baseline_soft, gate_metric, gate_mixed_weight, + skill_content=skill_init, + use_semantic_density=use_semantic_density, + semantic_density_weight=semantic_density_weight, + leading_words=leading_words, + ) + best_score = current_score + sh = skill_hash(skill_init) + sel_cache[sh] = (baseline_hard, baseline_soft) + current_origin = "initial_skill" + best_origin = "initial_skill" + _persist_runtime_state(0) + print( + f" [baseline result] selection hard={baseline_hard:.4f} " + f"soft={baseline_soft:.4f} " + f"gate[{gate_metric}]={current_score:.4f}" + ) + + # ── Training loop ──────────────────────────────────────────────── + t_loop_start = time.time() + + if resume_from > total_steps: + print(f"\n [skip] all {total_steps} steps complete — jumping to evaluation") + + global_step = 0 + for epoch in range(1, num_epochs + 1): + if dataloader is not None: + epoch_batches = dataloader.plan_train_epoch( + epoch=epoch, + steps_per_epoch=steps_per_epoch, + accumulation=accumulation, + batch_size=batch_size, + seed=seed, + out_root=out_root, + ) + shuffled_seeds = [batch.seed for batch in epoch_batches] + else: + epoch_batches = [] + epoch_rng = random.Random(seed + epoch * 1000) + shuffled_seeds = base_seeds.copy() + epoch_rng.shuffle(shuffled_seeds) + + # Step buffer: accumulates per-step context (failure patterns + + # rejected edits) within this epoch so optimizers see full history. + step_buffer: list[dict] = [] + active_meta_skill = ( + _load_meta_skill_content(out_root, epoch - 1) + if cfg.get("use_meta_skill", False) + else "" + ) + + print( + f"\n [EPOCH {epoch}/{num_epochs}] " + f"shuffled_seeds={shuffled_seeds}" + ) + if active_meta_skill: + print( + f" [meta skill] loaded from epoch {epoch - 1} " + f"({len(active_meta_skill)} chars)" + ) + + for step_in_epoch in range(steps_per_epoch): + global_step += 1 + if global_step < resume_from: + continue + + step_t0 = time.time() + step_dir = os.path.join(out_root, "steps", f"step_{global_step:04d}") + os.makedirs(step_dir, exist_ok=True) + + tokens_before = get_token_summary() + + print( + f"\n [STEP {global_step}/{total_steps}] " + f"epoch={epoch} step_in_epoch={step_in_epoch} " + f"{'='*30}" + ) + + step_rec: dict = { + "step": global_step, + "epoch": epoch, + "step_in_epoch": step_in_epoch, + "timing": {}, + "tokens": {}, + } + + # ── Accumulation: Rollout + Reflect ────────────────────── + all_failure_patches: list[dict] = [] + all_success_patches: list[dict] = [] + all_raw_patches: list[dict | None] = [] + all_rollout_results: list[dict] = [] + accum_rollout_stats: list[dict] = [] + total_rollout_time = 0.0 + total_reflect_time = 0.0 + + for a in range(accumulation): + batch_idx = step_in_epoch * accumulation + a + if dataloader is not None: + batch_spec = epoch_batches[batch_idx] + train_env, train_n, batch_seed = _build_train_env(batch_spec) + else: + batch_seed = shuffled_seeds[batch_idx] + train_env = adapter.build_train_env( + batch_size=batch_size, + seed=batch_seed, + out_root=out_root, + ) + train_n = len(train_env) if hasattr(train_env, "__len__") else batch_size + + # Directory routing + if accumulation > 1: + batch_dir = os.path.join(step_dir, f"batch_{a}") + else: + batch_dir = step_dir + + rollout_dir = os.path.join(batch_dir, "rollout") + patches_dir = os.path.join(batch_dir, "patches") + + # ① ROLLOUT ──────────────────────────────────────────── + t_phase = time.time() + print(f" [1/6 ROLLOUT] train items={train_n} (from pool, batch_seed={batch_seed})") + rollout_results = adapter.rollout( + train_env, current_skill, rollout_dir, + use_eval_feedback=True, + ) + r_hard, r_soft = compute_score(rollout_results) + total_rollout_time += time.time() - t_phase + all_rollout_results.extend(rollout_results) + print(f" [1/6 done] hard={r_hard:.4f} soft={r_soft:.4f}") + + # ② REFLECT ──────────────────────────────────────────── + t_phase = time.time() + pred_dir = os.path.join(rollout_dir, "predictions") + + # Build step context from buffer + step_buffer_context = _format_step_buffer(step_buffer) + + raw_patches = adapter.reflect( + rollout_results, current_skill, batch_dir, + prediction_dir=pred_dir, patches_dir=patches_dir, + random_seed=batch_seed, + step_buffer_context=step_buffer_context, + meta_skill_context=active_meta_skill, + ) + failure_patches, success_patches = _normalise_patches( + raw_patches, + update_mode=update_mode, + ) + all_failure_patches.extend(failure_patches) + all_success_patches.extend(success_patches) + all_raw_patches.extend(raw_patches) + total_reflect_time += time.time() - t_phase + + print( + f" [2/6 done] failure_patches={len(failure_patches)} " + f"success_patches={len(success_patches)}" + ) + + # Track per-batch stats + accum_rollout_stats.append({ + "batch_idx": a, + "batch_seed": batch_seed, + "n_envs": len(rollout_results), + "hard": r_hard, + "soft": r_soft, + "n_failure_patches": len(failure_patches), + "n_success_patches": len(success_patches), + }) + + # ── End of accumulation loop ───────────────────────────── + + # Aggregate rollout stats across batches + total_n = sum(b["n_envs"] for b in accum_rollout_stats) + agg_hard = sum(b["hard"] * b["n_envs"] for b in accum_rollout_stats) / max(total_n, 1) + agg_soft = sum(b["soft"] * b["n_envs"] for b in accum_rollout_stats) / max(total_n, 1) + + step_rec["rollout_hard"] = round(agg_hard, 6) + step_rec["rollout_soft"] = round(agg_soft, 6) + step_rec["rollout_n"] = total_n + step_rec["accumulation_batches"] = accum_rollout_stats + step_rec["timing"]["rollout_s"] = round(total_rollout_time, 1) + step_rec["timing"]["reflect_s"] = round(total_reflect_time, 1) + + n_total_patches = len(all_failure_patches) + len(all_success_patches) + step_rec["n_patches"] = n_total_patches + step_rec["n_failure_patches"] = len(all_failure_patches) + step_rec["n_success_patches"] = len(all_success_patches) + + if accumulation > 1: + print( + f" [accum done] total: failure={len(all_failure_patches)} " + f"success={len(all_success_patches)} " + f"from {accumulation} batches" + ) + + # ── No patches? Skip ───────────────────────────────────── + if not all_failure_patches and not all_success_patches: + # Skill-aware: a lapse-only step has no body patches but + # may still carry appendix notes — flush them BEFORE + # skipping, or they would be silently dropped. + if use_skill_aware: + current_skill = _flush_skill_aware_appendix( + current_skill, all_raw_patches, step_rec, step_dir, cfg, + ) + step_rec["action"] = "skip_no_patches" + step_rec["current_score"] = current_score + step_rec["best_score"] = best_score + step_rec["best_step"] = best_step + step_rec["skill_len"] = len(current_skill) + step_rec["wall_time_s"] = round(time.time() - step_t0, 1) + history.append(step_rec) + _save_history(out_root, history) + _save_skill(out_root, global_step, current_skill) + _persist_runtime_state(global_step) + with open(os.path.join(step_dir, "step_record.json"), "w") as f: + json.dump(step_rec, f, indent=2, ensure_ascii=False) + print(" [skip] no usable patches — skill unchanged") + continue + + # ③ AGGREGATE ────────────────────────────────────────────── + t_phase = time.time() + merged_patch = merge_patches( + current_skill, all_failure_patches, all_success_patches, + batch_size=merge_bs, verbose=True, + workers=cfg["analyst_workers"], + update_mode=update_mode, + meta_skill_context=active_meta_skill, + ) + with open(os.path.join(step_dir, "merged_patch.json"), "w") as f: + json.dump(merged_patch, f, ensure_ascii=False, indent=2) + + merged_items = get_payload_items(merged_patch, update_mode) + n_edits_merged = len(merged_items) + step_rec["n_edits_merged"] = n_edits_merged + step_rec["timing"]["aggregate_s"] = round(time.time() - t_phase, 1) + print(f" [3/6 done] merged {n_edits_merged} {payload_label(update_mode)}") + + # ④ SELECT ───────────────────────────────────────────────── + t_phase = time.time() + lr_decision = None + if is_full_rewrite_minibatch_mode(update_mode): + edit_budget = None + ranked_patch = merged_patch + ranked_items = merged_items + n_edits_ranked = len(ranked_items) + step_rec["n_edits_ranked"] = n_edits_ranked + step_rec["edit_budget"] = None + step_rec["lr_control_mode"] = "none" + with open(os.path.join(step_dir, "ranked_edits.json"), "w") as f: + json.dump(ranked_patch, f, ensure_ascii=False, indent=2) + else: + if lr_control_mode == "autonomous": + lr_decision = decide_autonomous_learning_rate( + skill_content=current_skill, + merged_patch=merged_patch, + update_mode=update_mode, + rollout_hard=agg_hard, + rollout_soft=agg_soft, + rollout_n=total_n, + step_buffer_context=step_buffer_context, + meta_skill_context=active_meta_skill, + ) + edit_budget = int(lr_decision["learning_rate"]) + with open(os.path.join(step_dir, "lr_decision.json"), "w") as f: + json.dump(lr_decision, f, ensure_ascii=False, indent=2) + with open(os.path.join(out_root, "lr_history.jsonl"), "a") as f: + f.write(json.dumps({ + "step": global_step, + "epoch": epoch, + **lr_decision, + }, ensure_ascii=False) + "\n") + else: + edit_budget = scheduler.step() + ranked_patch = rank_and_select( + current_skill, merged_patch, + max_edits=edit_budget, + update_mode=update_mode, + meta_skill_context=active_meta_skill, + ) + with open(os.path.join(step_dir, "ranked_edits.json"), "w") as f: + json.dump(ranked_patch, f, ensure_ascii=False, indent=2) + + ranked_items = get_payload_items(ranked_patch, update_mode) + n_edits_ranked = len(ranked_items) + step_rec["n_edits_ranked"] = n_edits_ranked + step_rec["edit_budget"] = edit_budget + step_rec["lr_control_mode"] = lr_control_mode + if lr_decision is not None: + step_rec["lr_decision"] = lr_decision + step_rec["timing"]["select_s"] = round(time.time() - t_phase, 1) + + support_counts = [ + item.get("support_count", 0) for item in ranked_items if isinstance(item, dict) + ] + step_rec["support_counts"] = support_counts + if is_full_rewrite_minibatch_mode(update_mode): + print( + f" [4/6 SELECT] skipped LR/select; " + f"using {n_edits_ranked} merged {payload_label(update_mode)}" + ) + else: + print( + f" [4/6 SELECT] " + f"{n_edits_merged} -> {n_edits_ranked} {payload_label(update_mode)} " + f"(budget={edit_budget}, lr_control={lr_control_mode})" + ) + + # ⑤ UPDATE ───────────────────────────────────────────────── + t_phase = time.time() + rewrite_result = None + if update_mode == "rewrite_from_suggestions": + rewrite_result = rewrite_skill_from_suggestions( + current_skill, + ranked_patch, + step_buffer_context=step_buffer_context, + env=cfg.get("env"), + reasoning_effort=rewrite_reasoning_effort, + max_completion_tokens=rewrite_max_completion_tokens, + ) + if rewrite_result and rewrite_result.get("new_skill"): + candidate_skill = rewrite_result["new_skill"] + apply_report = [] + with open(os.path.join(step_dir, "rewrite_result.json"), "w") as f: + json.dump(rewrite_result, f, ensure_ascii=False, indent=2) + else: + candidate_skill = current_skill + apply_report = [] + elif is_full_rewrite_minibatch_mode(update_mode): + skill_candidates = get_payload_items(ranked_patch, update_mode) + selected_candidate = next( + ( + item for item in skill_candidates + if isinstance(item, dict) and str(item.get("new_skill", "")).strip() + ), + None, + ) + if selected_candidate: + candidate_skill = str(selected_candidate["new_skill"]).rstrip() + "\n" + apply_report = [] + rewrite_result = { + "reasoning": ranked_patch.get("reasoning", ""), + "change_summary": selected_candidate.get("change_summary", []), + "title": selected_candidate.get("title", ""), + "source_type": selected_candidate.get("source_type", ""), + } + with open(os.path.join(step_dir, "full_rewrite_result.json"), "w") as f: + json.dump( + { + "selected_candidate": selected_candidate, + "merged_patch": ranked_patch, + }, + f, + ensure_ascii=False, + indent=2, + ) + else: + candidate_skill = current_skill + apply_report = [] + else: + candidate_skill, apply_report = apply_patch_with_report(current_skill, ranked_patch) + with open(os.path.join(step_dir, "candidate_skill.md"), "w") as f: + f.write(candidate_skill) + if apply_report: + with open(os.path.join(step_dir, "edit_apply_report.json"), "w") as f: + json.dump(apply_report, f, indent=2, ensure_ascii=False) + + cand_hash = skill_hash(candidate_skill) + step_rec["candidate_hash"] = cand_hash + step_rec["candidate_skill_len"] = len(candidate_skill) + if rewrite_result: + step_rec["rewrite_change_summary"] = rewrite_result.get("change_summary", []) + if apply_report: + step_rec["edit_apply_summary"] = { + "total": len(apply_report), + "applied": sum( + 1 for row in apply_report if str(row.get("status", "")).startswith("applied") + ), + "skipped": sum( + 1 for row in apply_report if str(row.get("status", "")).startswith("skipped") + ), + "errors": sum( + 1 for row in apply_report if row.get("status") == "error" + ), + } + step_rec["timing"]["update_s"] = round(time.time() - t_phase, 1) + if ( + update_mode == "rewrite_from_suggestions" + and rewrite_result is None + ) or ( + is_full_rewrite_minibatch_mode(update_mode) + and rewrite_result is None + ): + # Skill-aware: flush appendix notes before skipping (see + # the skip_no_patches branch above). + if use_skill_aware: + current_skill = _flush_skill_aware_appendix( + current_skill, all_raw_patches, step_rec, step_dir, cfg, + ) + step_rec["action"] = "skip_no_rewrite" + step_rec["current_score"] = current_score + step_rec["best_score"] = best_score + step_rec["best_step"] = best_step + step_rec["skill_len"] = len(current_skill) + step_rec["wall_time_s"] = round(time.time() - step_t0, 1) + history.append(step_rec) + _save_history(out_root, history) + _save_skill(out_root, global_step, current_skill) + _persist_runtime_state(global_step) + with open(os.path.join(step_dir, "step_record.json"), "w") as f: + json.dump(step_rec, f, indent=2, ensure_ascii=False) + print(" [skip] no usable rewrite generated — skill unchanged") + continue + print( + f" [5/6 UPDATE] " + f"skill_len {len(current_skill)} -> {len(candidate_skill)}" + ) + + # ⑥ EVALUATE ─────────────────────────────────────────────── + t_phase = time.time() + if cand_hash in sel_cache: + cand_hard, cand_soft = sel_cache[cand_hash] + print( + f" [6/6 EVALUATE] " + f"cache hit {cand_hash}: hard={cand_hard:.4f}" + ) + else: + sel_env, sel_n = _build_eval_env( + split="valid_seen", + env_num=cfg["sel_env_num"], + seed=seed, + ) + print(f" [6/6 EVALUATE] selection items={sel_n}") + sel_eval_dir = os.path.join(step_dir, "selection_eval") + sel_results = adapter.rollout(sel_env, candidate_skill, sel_eval_dir) + cand_hard, cand_soft = compute_score(sel_results) + sel_cache[cand_hash] = (cand_hard, cand_soft) + + step_rec["selection_hard"] = cand_hard + step_rec["selection_soft"] = cand_soft + + gate = evaluate_gate( + candidate_skill=candidate_skill, + cand_hard=cand_hard, + current_skill=current_skill, + current_score=current_score, + best_skill=best_skill, + best_score=best_score, + best_step=best_step, + global_step=global_step, + cand_soft=cand_soft, + metric=gate_metric, + mixed_weight=gate_mixed_weight, + use_semantic_density=use_semantic_density, + semantic_density_weight=semantic_density_weight, + leading_words=leading_words, + ) if use_gate else None + cand_gate_score = select_gate_score( + cand_hard, cand_soft, gate_metric, gate_mixed_weight, + skill_content=candidate_skill, + use_semantic_density=use_semantic_density, + semantic_density_weight=semantic_density_weight, + leading_words=leading_words, + ) + if not use_gate: + # Validation ran (scores recorded above) but the gate is + # disabled: force-accept the candidate as the new current + # skill. Best-so-far is still tracked for convenience; the + # final skill is selected manually from the trajectory. + if cand_gate_score > best_score: + fa_best_skill = candidate_skill + fa_best_score = cand_gate_score + fa_best_step = global_step + else: + fa_best_skill = best_skill + fa_best_score = best_score + fa_best_step = best_step + gate = GateResult( + action="force_accept", + current_skill=candidate_skill, + current_score=cand_gate_score, + best_skill=fa_best_skill, + best_score=fa_best_score, + best_step=fa_best_step, + ) + step_rec["gate_metric"] = gate_metric + step_rec["candidate_gate_score"] = cand_gate_score + step_rec["action"] = gate.action + prev_current = current_score + prev_best = best_score + current_skill = gate.current_skill + current_score = gate.current_score + best_skill = gate.best_skill + best_score = gate.best_score + best_step = gate.best_step + if gate.action in {"accept", "accept_new_best", "force_accept"}: + current_origin = f"step_{global_step:04d}" + if gate.action == "accept_new_best" or ( + gate.action == "force_accept" and best_step == global_step + ): + best_origin = current_origin + + if use_skill_aware: + current_skill = _flush_skill_aware_appendix( + current_skill, all_raw_patches, step_rec, step_dir, cfg, + ) + + if gate_metric == "hard": + score_label = f"hard={cand_hard:.4f}" + elif gate_metric == "soft": + score_label = f"soft={cand_soft:.4f}" + else: + score_label = ( + f"mixed[w={gate_mixed_weight}]={cand_gate_score:.4f} " + f"(hard={cand_hard:.4f} soft={cand_soft:.4f})" + ) + if gate.action == "accept_new_best": + print( + f" [6/6 EVALUATE] ACCEPT (new best) " + f"{score_label} > prev best {prev_best:.4f}" + ) + elif gate.action == "accept": + print( + f" [6/6 EVALUATE] ACCEPT " + f"{score_label} > current={prev_current:.4f}" + ) + elif gate.action == "force_accept": + print( + f" [6/6 EVALUATE] FORCE-ACCEPT (gate disabled) " + f"{score_label}" + ) + else: + print( + f" [6/6 EVALUATE] REJECT " + f"{score_label} <= current={current_score:.4f}" + ) + + step_rec["timing"]["evaluate_s"] = round(time.time() - t_phase, 1) + + # ── Step buffer: unified failure patterns + rejected edits ─ + action = step_rec.get("action", "unknown") + n_total = len(all_rollout_results) or 1 + n_fail = sum(1 for r in all_rollout_results if not r.get("hard") or float(r.get("hard", 0)) < 1e-9) + failure_patterns = _extract_failure_patterns( + all_rollout_results, step_dir, + ) + + buf_entry: dict = { + "step": global_step, + "action": action, + "n_total": n_total, + "n_fail": n_fail, + "failure_patterns": failure_patterns, + } + + # Attach rejected edits when the step was rejected + if "reject" in action and ranked_patch: + rejected_edits = [ + short_item_summary(item, update_mode) + for item in ranked_items + if isinstance(item, dict) + ] + buf_entry["score_before"] = current_score + buf_entry["score_after"] = cand_gate_score + buf_entry["rejected_edits"] = rejected_edits + + step_buffer.append(buf_entry) + + # Persist step digest for step buffer context + digest_path = os.path.join(step_dir, "trajectory_digest.json") + with open(digest_path, "w") as f: + json.dump(buf_entry, f, indent=2, ensure_ascii=False) + + # ── Token snapshot ─────────────────────────────────────── + tokens_after = get_token_summary() + step_tokens: dict = {} + for stage in tokens_after: + if stage == "_total": + continue + after = tokens_after[stage] + before = tokens_before.get(stage, {}) + step_tokens[stage] = { + "calls": after.get("calls", 0) - before.get("calls", 0), + "prompt_tokens": after.get("prompt_tokens", 0) + - before.get("prompt_tokens", 0), + "completion_tokens": after.get("completion_tokens", 0) + - before.get("completion_tokens", 0), + } + step_rec["tokens"] = step_tokens + + # ── Save state ─────────────────────────────────────────── + step_rec["current_score"] = current_score + step_rec["best_score"] = best_score + step_rec["best_step"] = best_step + step_rec["current_origin"] = current_origin + step_rec["best_origin"] = best_origin + step_rec["skill_len"] = len(current_skill) + step_rec["wall_time_s"] = round(time.time() - step_t0, 1) + + _save_skill(out_root, global_step, current_skill) + with open(os.path.join(out_root, "best_skill.md"), "w") as f: + f.write(best_skill) + history.append(step_rec) + _save_history(out_root, history) + _persist_runtime_state(global_step) + with open(os.path.join(step_dir, "step_record.json"), "w") as f: + json.dump(step_rec, f, indent=2, ensure_ascii=False) + + timing = step_rec["timing"] + print( + f"\n [STEP {global_step} done] " + f"epoch={epoch} action={step_rec['action']} " + f"current={current_score:.4f} best={best_score:.4f} " + f"dt={step_rec['wall_time_s']}s\n" + f" timing: rollout={timing.get('rollout_s',0)}s " + f"reflect={timing.get('reflect_s',0)}s " + f"aggregate={timing.get('aggregate_s',0)}s " + f"select={timing.get('select_s',0)}s " + f"evaluate={timing.get('evaluate_s',0)}s" + ) + + epoch_last_step_skill = current_skill + epoch_comparison_pairs: list[dict] | None = None + + # ── SLOW UPDATE (end of epoch) ────────────────────────────── + use_slow = cfg.get("use_slow_update", False) + if use_slow: + slow_dir = os.path.join(out_root, "slow_update", f"epoch_{epoch:02d}") + slow_done_path = os.path.join(slow_dir, "slow_result.json") + + if os.path.exists(slow_done_path): + # Resume support + print( + f"\n [SLOW UPDATE epoch {epoch}] " + f"resumed — already done" + ) + with open(slow_done_path) as f: + slow_saved = json.load(f) + comparison_path = os.path.join(slow_dir, "comparison_pairs.json") + if os.path.exists(comparison_path): + try: + with open(comparison_path) as f: + epoch_comparison_pairs = json.load(f) + except Exception: + epoch_comparison_pairs = None + if ( + slow_saved.get("slow_update_content") + and epoch >= 2 + ): + action = slow_saved.get("action") + if slow_gate_with_selection: + # Gated mode (follow SkillReflection): re-apply the + # guidance to current_skill only when it was accepted. + if action in {"accept", "accept_new_best"}: + current_skill = replace_slow_update_field( + current_skill, + slow_saved["slow_update_content"], + ) + elif action in { + "accept", "accept_new_best", "force_accept", + }: + # Force-accept mode: re-apply guidance to + # current_skill only. best_skill must remain a + # faithful snapshot of the val-best step and must + # NOT receive force-injected slow-update content. + current_skill = replace_slow_update_field( + current_skill, slow_saved["slow_update_content"], + ) + elif epoch == 1: + # Epoch 1: inject empty placeholder + os.makedirs(slow_dir, exist_ok=True) + current_skill = inject_empty_slow_update_field(current_skill) + current_origin = f"slow_update_placeholder_epoch_{epoch:02d}" + _save_skill(out_root, global_step, current_skill) + with open(os.path.join(out_root, "best_skill.md"), "w") as f: + f.write(best_skill) + with open(slow_done_path, "w") as f: + json.dump({"action": "inject_placeholder", "epoch": epoch}, f, indent=2) + _persist_runtime_state(global_step) + print( + f"\n [SLOW UPDATE epoch {epoch}] " + f"injected empty placeholder" + ) + else: + # Epoch 2+: longitudinal comparison + os.makedirs(slow_dir, exist_ok=True) + print( + f"\n {'='*60}\n" + f" SLOW UPDATE — Epoch {epoch} " + f"(comparing epoch {epoch-1} vs {epoch})\n" + f" {'='*60}" + ) + + # 1. Get skill from last step of previous epoch + prev_epoch_records = [ + h for h in history if h.get("epoch") == epoch - 1 + ] + prev_epoch_last_step = prev_epoch_records[-1]["step"] + prev_skill = _load_skill(out_root, prev_epoch_last_step) + + # 2. Sample items from train set + slow_n = cfg.get("slow_update_samples", 20) + slow_seed = seed + epoch * 2000 + if dataloader is not None: + slow_batch = dataloader.build_train_batch( + batch_size=slow_n, + seed=slow_seed, + out_root=out_root, + ) + slow_env = adapter.build_env_from_batch( + slow_batch, out_root=out_root, + ) + else: + slow_env = adapter.build_train_env( + batch_size=slow_n, + seed=slow_seed, + out_root=out_root, + ) + slow_items = list(slow_env) if hasattr(slow_env, "__iter__") else slow_env + print(f" [slow update] sampled {len(slow_items)} train items (seed={slow_seed})") + + # 3. Rollout with both skills + t_slow = time.time() + prev_rollout_dir = os.path.join(slow_dir, "rollout_prev") + curr_rollout_dir = os.path.join(slow_dir, "rollout_curr") + results_prev = adapter.rollout(slow_env, prev_skill, prev_rollout_dir) + results_curr = adapter.rollout(slow_env, current_skill, curr_rollout_dir) + + prev_hard, _ = compute_score(results_prev) + curr_hard, _ = compute_score(results_curr) + print( + f" [slow update] prev epoch hard={prev_hard:.4f} " + f"curr epoch hard={curr_hard:.4f}" + ) + + # 4. Build and save structured comparison pairs + comparison_pairs, all_comparison_pairs = _build_longitudinal_pairs( + adapter=adapter, + dataloader=dataloader, + prev_skill=prev_skill, + curr_skill=current_skill, + initial_items=slow_items, + initial_prev_results=results_prev, + initial_curr_results=results_curr, + prev_rollout_dir=prev_rollout_dir, + curr_rollout_dir=curr_rollout_dir, + policy=longitudinal_pair_policy, + target_n=slow_n, + seed=slow_seed, + out_root=out_root, + ) + epoch_comparison_pairs = comparison_pairs + if all_comparison_pairs is not comparison_pairs: + save_comparison_pairs( + all_comparison_pairs, + os.path.join(slow_dir, "comparison_pairs_all.json"), + ) + save_comparison_pairs( + comparison_pairs, + os.path.join(slow_dir, "comparison_pairs.json"), + ) + n_regressed = sum(1 for p in comparison_pairs if p["category"] == "regressed") + n_improved = sum(1 for p in comparison_pairs if p["category"] == "improved") + n_persist = sum(1 for p in comparison_pairs if p["category"] == "persistent_fail") + n_stable = sum(1 for p in comparison_pairs if p["category"] == "stable_success") + print( + f" [slow update] comparison: " + f"regressed={n_regressed} improved={n_improved} " + f"persistent_fail={n_persist} stable_success={n_stable} " + f"policy={longitudinal_pair_policy} " + f"kept={len(comparison_pairs)}/{len(all_comparison_pairs)}" + ) + + # 5. Extract previous slow update guidance for reflection + existing_guidance = extract_slow_update_field(current_skill) + + # 6. Optimizer analysis (with reflection on previous guidance) + slow_result = run_slow_update( + current_skill, + results_prev, + results_curr, + slow_items, + prev_skill=prev_skill, + prev_slow_update_content=existing_guidance, + prev_rollout_dir=prev_rollout_dir, + curr_rollout_dir=curr_rollout_dir, + comparison_pairs=comparison_pairs, + ) + slow_time = round(time.time() - t_slow, 1) + + if slow_result and slow_result.get("slow_update_content"): + slow_candidate = replace_slow_update_field( + current_skill, slow_result["slow_update_content"], + ) + slow_candidate_hash = skill_hash(slow_candidate) + with open(os.path.join(slow_dir, "candidate_skill.md"), "w") as f: + f.write(slow_candidate) + slow_result["time_s"] = slow_time + slow_result["prev_hard"] = prev_hard + slow_result["curr_hard"] = curr_hard + slow_result["candidate_hash"] = slow_candidate_hash + slow_result["update_origin"] = "slow_update_momentum" + slow_result["update_target"] = ( + "Address longitudinal regressions and persistent failures " + "observed across adjacent epochs." + ) + + # Slow update acceptance — two modes selected via + # `optimizer.slow_update_gate_with_selection`. + if slow_gate_with_selection: + # ── Gated mode (follow SkillReflection) ────────── + # Evaluate the slow-update candidate on the + # selection set and accept/reject via the same + # validation gate used for step-level updates. + if slow_candidate_hash in sel_cache: + slow_sel_hard, slow_sel_soft = sel_cache[ + slow_candidate_hash + ] + print( + f" [slow gate] cache hit: " + f"hard={slow_sel_hard:.4f}" + ) + else: + sel_env, sel_n = _build_eval_env( + split="valid_seen", + env_num=cfg["sel_env_num"], + seed=seed, + ) + print(f" [slow gate] selection items={sel_n}") + slow_eval_dir = os.path.join( + slow_dir, "selection_eval", + ) + slow_eval_results = adapter.rollout( + sel_env, slow_candidate, slow_eval_dir, + ) + slow_sel_hard, slow_sel_soft = compute_score( + slow_eval_results + ) + sel_cache[slow_candidate_hash] = ( + slow_sel_hard, slow_sel_soft, + ) + + slow_gate = evaluate_gate( + candidate_skill=slow_candidate, + cand_hard=slow_sel_hard, + current_skill=current_skill, + current_score=current_score, + best_skill=best_skill, + best_score=best_score, + best_step=best_step, + global_step=global_step, + cand_soft=slow_sel_soft, + metric=gate_metric, + mixed_weight=gate_mixed_weight, + use_semantic_density=use_semantic_density, + semantic_density_weight=semantic_density_weight, + leading_words=leading_words, + ) + slow_result["selection_hard"] = slow_sel_hard + slow_result["selection_soft"] = slow_sel_soft + slow_result["action"] = slow_gate.action + prev_current = current_score + prev_best = best_score + current_skill = slow_gate.current_skill + current_score = slow_gate.current_score + best_skill = slow_gate.best_skill + best_score = slow_gate.best_score + best_step = slow_gate.best_step + if slow_gate.action in {"accept", "accept_new_best"}: + current_origin = ( + f"slow_update_epoch_{epoch:02d}" + ) + if slow_gate.action == "accept_new_best": + best_origin = current_origin + print( + f" [slow gate] ACCEPT (new best) " + f"hard={slow_sel_hard:.4f} > " + f"prev best {prev_best:.4f}" + ) + elif slow_gate.action == "accept": + print( + f" [slow gate] ACCEPT " + f"hard={slow_sel_hard:.4f} > " + f"current={prev_current:.4f}" + ) + else: + print( + f" [slow gate] REJECT " + f"hard={slow_sel_hard:.4f} <= " + f"current={current_score:.4f}" + ) + print( + f" [slow update] guidance written " + f"({len(slow_result['slow_update_content'])} " + f"chars), {slow_time}s" + ) + else: + # ── Force-accept mode (default) ────────────────── + # The epoch-level longitudinal guidance is injected + # into current_skill ONLY, so training continues + # with the accumulated slow memory. best_skill is + # left untouched: it must remain a faithful snapshot + # of the val-best step (which may be a pre-slow step + # such as S_0 carrying no slow_update field at all). + slow_content = slow_result["slow_update_content"] + current_skill = replace_slow_update_field( + current_skill, slow_content, + ) + # Update caches so downstream steps use the + # slow-update-injected skill for hashing. + slow_candidate_hash = skill_hash(current_skill) + sel_cache[slow_candidate_hash] = (current_score, 0.0) + + slow_result["action"] = "force_accept" + current_origin = f"slow_update_epoch_{epoch:02d}" + + print( + f" [slow update] force-injected into " + f"current only " + f"({len(slow_content)} chars), " + f"{slow_time}s" + ) + else: + slow_result = slow_result or {} + slow_result["action"] = "no_content" + slow_result["time_s"] = slow_time + print( + f" [slow update] no guidance produced, " + f"{slow_time}s" + ) + + # 5. Save + with open(slow_done_path, "w") as f: + json.dump(slow_result, f, indent=2, ensure_ascii=False) + _save_skill(out_root, global_step, current_skill) + with open(os.path.join(out_root, "best_skill.md"), "w") as f: + f.write(best_skill) + _persist_runtime_state(global_step) + + print( + f"\n [SLOW UPDATE epoch {epoch} done] " + f"current={current_score:.4f} best={best_score:.4f}" + ) + + # ── META SKILL (end of epoch, optimizer-side memory) ───────── + use_meta_skill = cfg.get("use_meta_skill", False) + if use_meta_skill: + meta_skill_dir = os.path.join(out_root, "meta_skill", f"epoch_{epoch:02d}") + meta_skill_done_path = os.path.join(meta_skill_dir, "meta_skill_result.json") + os.makedirs(meta_skill_dir, exist_ok=True) + + if os.path.exists(meta_skill_done_path): + print(f"\n [META SKILL epoch {epoch}] resumed — already done") + elif epoch == 1: + with open(meta_skill_done_path, "w") as f: + json.dump( + {"action": "skip_first_epoch", "epoch": epoch}, + f, indent=2, ensure_ascii=False, + ) + print(f"\n [META SKILL epoch {epoch}] skipped — first epoch") + else: + print( + f"\n {'='*60}\n" + f" META SKILL — Epoch {epoch} " + f"(optimizer memory from epoch {epoch-1} vs {epoch})\n" + f" {'='*60}" + ) + + prev_epoch_records = [h for h in history if h.get("epoch") == epoch - 1] + prev_epoch_last_step = prev_epoch_records[-1]["step"] + prev_skill = _load_skill(out_root, prev_epoch_last_step) + prev_meta_skill = _load_meta_skill_content(out_root, epoch - 1) + + if epoch_comparison_pairs is None: + meta_n = cfg.get("slow_update_samples", 20) + meta_seed = seed + epoch * 2000 + if dataloader is not None: + meta_batch = dataloader.build_train_batch( + batch_size=meta_n, + seed=meta_seed, + out_root=out_root, + ) + meta_env = adapter.build_env_from_batch( + meta_batch, out_root=out_root, + ) + else: + meta_env = adapter.build_train_env( + batch_size=meta_n, + seed=meta_seed, + out_root=out_root, + ) + meta_items = list(meta_env) if hasattr(meta_env, "__iter__") else meta_env + prev_rollout_dir = os.path.join(meta_skill_dir, "rollout_prev") + curr_rollout_dir = os.path.join(meta_skill_dir, "rollout_curr") + results_prev = adapter.rollout(meta_env, prev_skill, prev_rollout_dir) + results_curr = adapter.rollout(meta_env, epoch_last_step_skill, curr_rollout_dir) + epoch_comparison_pairs, all_meta_comparison_pairs = _build_longitudinal_pairs( + adapter=adapter, + dataloader=dataloader, + prev_skill=prev_skill, + curr_skill=epoch_last_step_skill, + initial_items=meta_items, + initial_prev_results=results_prev, + initial_curr_results=results_curr, + prev_rollout_dir=prev_rollout_dir, + curr_rollout_dir=curr_rollout_dir, + policy=longitudinal_pair_policy, + target_n=meta_n, + seed=meta_seed, + out_root=out_root, + ) + if all_meta_comparison_pairs is not epoch_comparison_pairs: + save_comparison_pairs( + all_meta_comparison_pairs, + os.path.join(meta_skill_dir, "comparison_pairs_all.json"), + ) + save_comparison_pairs( + epoch_comparison_pairs, + os.path.join(meta_skill_dir, "comparison_pairs.json"), + ) + meta_counts = _pair_category_counts(epoch_comparison_pairs) + print( + f" [meta skill] comparison: " + f"regressed={meta_counts.get('regressed', 0)} " + f"improved={meta_counts.get('improved', 0)} " + f"persistent_fail={meta_counts.get('persistent_fail', 0)} " + f"stable_success={meta_counts.get('stable_success', 0)} " + f"policy={longitudinal_pair_policy} " + f"kept={len(epoch_comparison_pairs)}/{len(all_meta_comparison_pairs)}" + ) + + t_meta_skill = time.time() + meta_skill_result = run_meta_skill( + prev_skill=prev_skill, + curr_skill=epoch_last_step_skill, + comparison_pairs=epoch_comparison_pairs or [], + prev_meta_skill_content=prev_meta_skill, + ) + meta_skill_time = round(time.time() - t_meta_skill, 1) + + if meta_skill_result and meta_skill_result.get("meta_skill_content"): + meta_skill_result["time_s"] = meta_skill_time + meta_skill_result["action"] = "write_meta_skill" + print( + f" [meta skill] memory written " + f"({len(meta_skill_result['meta_skill_content'])} chars), " + f"{meta_skill_time}s" + ) + else: + meta_skill_result = meta_skill_result or {} + meta_skill_result["time_s"] = meta_skill_time + meta_skill_result["action"] = "no_content" + print(f" [meta skill] no memory produced, {meta_skill_time}s") + + with open(meta_skill_done_path, "w") as f: + json.dump(meta_skill_result, f, indent=2, ensure_ascii=False) + + # ── Save best skill ────────────────────────────────────────────── + with open(os.path.join(out_root, "best_skill.md"), "w") as f: + f.write(best_skill) + _persist_runtime_state(global_step) + print( + f"\n [done] best skill from step {best_step}, " + f"score={best_score:.4f}" + ) + + # ── Final test evaluation (valid_unseen) ───────────────────────── + baseline_test_hard = None + baseline_test_soft = None + test_hard = None + test_soft = None + final_test_hard = None + final_test_soft = None + final_selection_hard = None + final_selection_soft = None + + if cfg["eval_test"]: + task_types = adapter.get_task_types() + + # ── Final skill validation (valid_seen) + best promotion ───── + # The final (last) skill may carry an epoch-end slow_update that + # was force-injected WITHOUT a val pass (use_gate=false or + # slow_update_gate_with_selection=false), so it never competed for + # best. Run one real val on the final skill; if its gate score + # beats the incumbent best, PROMOTE it to best so that best is the + # true val-argmax over all skills (including the final slow_update). + # When final == best, reuse the existing val score (no rollout). + try: + if skill_hash(current_skill) == skill_hash(best_skill): + final_selection_hard, final_selection_soft = best_score, None + print( + "\n [final skill == best skill] " + f"final_selection_hard={best_score:.4f} (reused)" + ) + else: + fval_env, fval_n = _build_eval_env( + split="valid_seen", + env_num=cfg["sel_env_num"], + seed=seed, + ) + fval_dir = os.path.join(out_root, "final_selection_eval") + fval_results = adapter.rollout(fval_env, current_skill, fval_dir) + final_selection_hard, final_selection_soft = compute_score(fval_results) + final_gate_score = select_gate_score( + final_selection_hard, final_selection_soft, + gate_metric, gate_mixed_weight, + skill_content=current_skill, + use_semantic_density=use_semantic_density, + semantic_density_weight=semantic_density_weight, + leading_words=leading_words, + ) + print( + f"\n [final skill val] items={fval_n} " + f"final_selection_hard={final_selection_hard:.4f} " + f"gate={final_gate_score:.4f} " + f"(best={best_score:.4f})" + ) + if final_gate_score > best_score: + # Promote: the final (slow-updated) skill is val-better + # than the incumbent best. Make it the new best so the + # subsequent BEST-skill test rollout evaluates it and + # best/final test scores coincide. + print( + f" [promote] final {final_gate_score:.4f} > " + f"best {best_score:.4f} → final becomes new best " + f"(step {global_step}, origin {current_origin})" + ) + best_skill = current_skill + best_score = final_gate_score + best_step = global_step + best_origin = current_origin + with open(os.path.join(out_root, "best_skill.md"), "w") as f: + f.write(best_skill) + _persist_runtime_state(global_step) + except Exception as _e: # noqa: BLE001 + final_selection_hard = None + final_selection_soft = None + print(f"\n [final skill val FAILED: {_e!r}]") + + # Baseline: S_0 on test set (valid_unseen) + print(f"\n{'='*60}") + print(" BASELINE TEST — evaluate initial skill on Test set (valid_unseen)") + print(f"{'='*60}") + test_env, test_n = _build_eval_env( + split="valid_unseen", + env_num=cfg["test_env_num"], + seed=seed, + ) + print(f" Test items: {test_n}") + baseline_test_dir = os.path.join(out_root, "test_eval_baseline") + os.makedirs(baseline_test_dir, exist_ok=True) + baseline_test_results = adapter.rollout(test_env, skill_init, baseline_test_dir) + baseline_test_hard, baseline_test_soft = compute_score(baseline_test_results) + baseline_buckets = _compute_task_type_buckets(baseline_test_results, task_types) + print("\n === Baseline Test Results (S_0) ===") + for task_type in task_types + ["overall"]: + b = baseline_buckets.get(task_type, {"total": 0, "hard": 0}) + t = max(b["total"], 1) + print( + f" {task_type:<40s}: " + f"hard={b['hard']}/{b['total']}={b['hard']/t:.4f}" + ) + with open(os.path.join(baseline_test_dir, "summary.json"), "w") as f: + json.dump( + { + k: { + "total": b["total"], + "hard_acc": b["hard"] / max(b["total"], 1), + } + for k, b in baseline_buckets.items() + }, + f, indent=2, ensure_ascii=False, + ) + + # Best skill on test set + print(f"\n{'='*60}") + print(" BEST SKILL TEST — evaluate best skill on Test set (valid_unseen)") + print(f"{'='*60}") + test_env2, test_n2 = _build_eval_env( + split="valid_unseen", + env_num=cfg["test_env_num"], + seed=seed, + ) + print(f" Test items: {test_n2}") + test_dir = os.path.join(out_root, "test_eval") + os.makedirs(test_dir, exist_ok=True) + test_results = adapter.rollout(test_env2, best_skill, test_dir) + test_hard, test_soft = compute_score(test_results) + best_buckets = _compute_task_type_buckets(test_results, task_types) + print("\n === Best Skill Test Results ===") + for task_type in task_types + ["overall"]: + b = best_buckets.get(task_type, {"total": 0, "hard": 0}) + t = max(b["total"], 1) + print( + f" {task_type:<40s}: " + f"hard={b['hard']}/{b['total']}={b['hard']/t:.4f}" + ) + with open(os.path.join(test_dir, "summary.json"), "w") as f: + json.dump( + { + k: { + "total": b["total"], + "hard_acc": b["hard"] / max(b["total"], 1), + } + for k, b in best_buckets.items() + }, + f, indent=2, ensure_ascii=False, + ) + + # Final skill (last skill in trajectory) on test set. + # Distinct from best_skill: with use_gate=False every candidate is + # force-accepted so the final skill is whatever the last step + # produced; with use_gate=True it is the last accepted skill, which + # may differ from the best-on-val skill. We always evaluate it so + # every run reports baseline / best-on-val / final on test. + # Guarded so a failure here never prevents summary.json from being + # written (the orchestrator's post-hoc safety net fills it in). + try: + if skill_hash(current_skill) == skill_hash(best_skill): + # Final == best: reuse results, skip a redundant rollout. + final_test_hard, final_test_soft = test_hard, test_soft + final_test_dir = os.path.join(out_root, "test_eval_final") + os.makedirs(final_test_dir, exist_ok=True) + with open(os.path.join(final_test_dir, "summary.json"), "w") as f: + json.dump( + { + k: { + "total": b["total"], + "hard_acc": b["hard"] / max(b["total"], 1), + } + for k, b in best_buckets.items() + }, + f, indent=2, ensure_ascii=False, + ) + print( + "\n [final skill == best skill] " + f"final_test_hard={final_test_hard:.4f} (reused)" + ) + else: + print(f"\n{'='*60}") + print(" FINAL SKILL TEST — evaluate last skill on Test set (valid_unseen)") + print(f"{'='*60}") + test_env3, test_n3 = _build_eval_env( + split="valid_unseen", + env_num=cfg["test_env_num"], + seed=seed, + ) + print(f" Test items: {test_n3}") + final_test_dir = os.path.join(out_root, "test_eval_final") + os.makedirs(final_test_dir, exist_ok=True) + final_test_results = adapter.rollout(test_env3, current_skill, final_test_dir) + final_test_hard, final_test_soft = compute_score(final_test_results) + final_buckets = _compute_task_type_buckets(final_test_results, task_types) + print("\n === Final Skill Test Results ===") + for task_type in task_types + ["overall"]: + b = final_buckets.get(task_type, {"total": 0, "hard": 0}) + t = max(b["total"], 1) + print( + f" {task_type:<40s}: " + f"hard={b['hard']}/{b['total']}={b['hard']/t:.4f}" + ) + with open(os.path.join(final_test_dir, "summary.json"), "w") as f: + json.dump( + { + k: { + "total": b["total"], + "hard_acc": b["hard"] / max(b["total"], 1), + } + for k, b in final_buckets.items() + }, + f, indent=2, ensure_ascii=False, + ) + except Exception as _e: # noqa: BLE001 + final_test_hard = None + final_test_soft = None + print(f"\n [final skill test FAILED: {_e!r}] " + "— will be filled by post-hoc eval") + + # Comparison + delta_hard = (test_hard or 0) - (baseline_test_hard or 0) + print(f"\n === Improvement vs baseline (init S_0) ===") + print( + f" [2] best-on-val hard: {baseline_test_hard:.4f} -> {test_hard:.4f} " + f"(delta={delta_hard:+.4f})" + ) + if final_test_hard is not None: + final_delta_hard = (final_test_hard or 0) - (baseline_test_hard or 0) + print( + f" [3] final/last hard: {baseline_test_hard:.4f} -> {final_test_hard:.4f} " + f"(delta={final_delta_hard:+.4f})" + ) + + # ── Global summary ─────────────────────────────────────────────── + total_wall = time.time() - t_loop_start + n_accept = sum(1 for h in history if "accept" in h.get("action", "")) + n_reject = sum(1 for h in history if h.get("action") == "reject") + n_skip = sum(1 for h in history if h.get("action") == "skip_no_patches") + + token_summary = get_token_summary() + + # Epoch-level statistics + epoch_stats = [] + for e in range(1, num_epochs + 1): + epoch_records = [h for h in history if h.get("epoch") == e] + if epoch_records: + epoch_stats.append({ + "epoch": e, + "steps": [h["step"] for h in epoch_records], + "accepts": sum(1 for h in epoch_records if "accept" in h.get("action", "")), + "rejects": sum(1 for h in epoch_records if h.get("action") == "reject"), + "skips": sum(1 for h in epoch_records if h.get("action") == "skip_no_patches"), + "best_score_at_epoch_end": epoch_records[-1].get("best_score", 0.0), + "current_score_at_epoch_end": epoch_records[-1].get("current_score", 0.0), + }) + + summary = { + "version": "skillopt-0.1.0", + "config": _redact_cfg(cfg), + "baseline_selection_hard": sel_cache.get( + skill_hash(skill_init), (None, None), + )[0], + "best_selection_hard": best_score, + "final_selection_hard": final_selection_hard, + "final_selection_soft": final_selection_soft, + "best_step": best_step, + "current_origin": current_origin, + "best_origin": best_origin, + "total_steps": len(history), + "total_accepts": n_accept, + "total_rejects": n_reject, + "total_skips": n_skip, + "epoch_stats": epoch_stats, + "baseline_test_hard": baseline_test_hard, + "baseline_test_soft": baseline_test_soft, + "test_hard": test_hard, + "test_soft": test_soft, + "final_test_hard": final_test_hard, + "final_test_soft": final_test_soft, + "test_delta_hard": ( + (test_hard or 0) - (baseline_test_hard or 0) + if test_hard is not None + else None + ), + "final_test_delta_hard": ( + (final_test_hard or 0) - (baseline_test_hard or 0) + if final_test_hard is not None + else None + ), + "total_wall_time_s": round(total_wall, 1), + "token_summary": token_summary, + } + with open(os.path.join(out_root, "summary.json"), "w") as f: + json.dump(summary, f, indent=2, ensure_ascii=False) + + print(f"\n{'='*60}") + print(" Final Summary") + print(f"{'='*60}") + print( + f" steps={len(history)} accept={n_accept} " + f"reject={n_reject} skip={n_skip}" + ) + print(f" best_score={best_score:.4f} (step {best_step}) wall={total_wall:.0f}s") + if epoch_stats: + for es in epoch_stats: + print( + f" epoch {es['epoch']}: accept={es['accepts']} reject={es['rejects']} " + f"best={es['best_score_at_epoch_end']:.4f}" + ) + if baseline_test_hard is not None: + print("\n === TEST scores (3 skills, split=valid_unseen) ===") + print( + f" [1] init/baseline (S_0) : " + f"test_hard={baseline_test_hard:.4f}" + ) + if test_hard is not None: + print( + f" [2] best-on-val (step {best_step})".ljust(37) + + f": test_hard={test_hard:.4f} test_soft={test_soft:.4f}" + ) + if final_test_hard is not None: + print( + f" [3] final/last skill : " + f"test_hard={final_test_hard:.4f} test_soft={final_test_soft:.4f}" + ) + if token_summary.get("_total"): + t = token_summary["_total"] + print( + f" total tokens: {t['total_tokens']:,} " + f"(prompt={t['prompt_tokens']:,} " + f"completion={t['completion_tokens']:,} " + f"calls={t['calls']})" + ) + + return summary diff --git a/skillopt/envs/__init__.py b/skillopt/envs/__init__.py new file mode 100644 index 0000000..ecd0aaa --- /dev/null +++ b/skillopt/envs/__init__.py @@ -0,0 +1 @@ +"""ReflACT environment adapters.""" diff --git a/skillopt/envs/_template/README.md b/skillopt/envs/_template/README.md new file mode 100644 index 0000000..2057445 --- /dev/null +++ b/skillopt/envs/_template/README.md @@ -0,0 +1,43 @@ +# Benchmark Template + +This directory provides scaffold files for adding a new benchmark to SkillOpt. + +## Files + +- `env_template.py` — Environment adapter template (subclasses + `EnvAdapter`; implements the 4 abstract methods so the file is + instantiable out of the box — `reflect` is inherited). +- `loader_template.py` — Data loader template (subclasses + `SplitDataLoader`; implements `load_split_items` for `.json`/`.jsonl`). +- `config_template.yaml` — Config file template. + +## Usage + +1. **Copy the directory:** + ```bash + cp -r skillopt/envs/_template skillopt/envs/your_benchmark + ``` +2. **Rename the files** (drop the `_template` suffix): + ```bash + cd skillopt/envs/your_benchmark + mv env_template.py adapter.py + mv loader_template.py dataloader.py + ``` + …and inside each file rename the classes + (`TemplateBenchmarkEnv → YourBenchmarkAdapter`, + `TemplateBenchmarkLoader → YourBenchmarkLoader`) + and fix the cross-import in `adapter.py`. +3. **Implement the TODO blocks** inside `adapter.py:rollout` and the + `_normalize_item` helper in `dataloader.py`. (`reflect` is inherited from + `EnvAdapter`; override it only for custom reflection logic.) +4. **Register** the adapter — add a `try / except ImportError` block in + `scripts/train.py`'s `_register_builtins()` mapping the registry key + to your `YourBenchmarkAdapter` class. There is no + `BENCHMARK_REGISTRY` dict in `skillopt/envs/__init__.py`; the live + registry is `_ENV_REGISTRY` in `scripts/train.py`. +5. **Create the config** at `configs/your_benchmark/default.yaml` + (start from `config_template.yaml`). `_base_` is a **string path**, + not a list. + +See the [Add a New Benchmark guide](../../../docs/guide/new-benchmark.md) +for the full step-by-step with a worked `docfaithful` example. diff --git a/skillopt/envs/_template/config_template.yaml b/skillopt/envs/_template/config_template.yaml new file mode 100644 index 0000000..b482cc7 --- /dev/null +++ b/skillopt/envs/_template/config_template.yaml @@ -0,0 +1,55 @@ +# ────────────────────────────────────────────────── +# SkillOpt Config Template — +# ────────────────────────────────────────────────── +# Copy this file to configs//default.yaml +# and customize the values below. + +# Inherit global defaults. +# NOTE: `_base_` is a string path, not a list. +_base_: ../_base_/default.yaml + +# ── Environment ────────────────────────────────── +env: + name: your_benchmark # Must match the key registered in scripts/train.py + # Optional: a seed skill document. Create this file yourself before the + # first run, or omit the key to start from an empty skill. + # skill_init: skillopt/envs/your_benchmark/skills/initial.md + data_path: data/your_benchmark # Path to your data (for split_mode: ratio) + split_dir: "" # Set this and use split_mode: split_dir for pre-split data + split_mode: ratio # "ratio" or "split_dir" + split_ratio: "2:1:7" # train:val:test (used when split_mode: ratio) + workers: 4 # Parallel rollout workers + max_completion_tokens: 4096 # Cap per target-model call + limit: 0 # 0 = no limit; small int = debug sample + +# ── Training ───────────────────────────────────── +train: + num_epochs: 4 + batch_size: 40 + accumulation: 1 + seed: 42 + +# ── Gradient (Reflection) ─────────────────────── +gradient: + analyst_workers: 16 # Parallel reflection workers + minibatch_size: 8 + merge_batch_size: 8 + +# ── Optimizer ──────────────────────────────────── +optimizer: + learning_rate: 4 # Max edits per step (edit budget) + lr_scheduler: cosine # cosine | linear | constant | autonomous + use_slow_update: true # Epoch-boundary momentum + use_meta_skill: true # Cross-epoch optimizer memory + +# ── Evaluation ─────────────────────────────────── +evaluation: + use_gate: true # Validation gating + eval_test: true # Run test eval after training + +# ── Model ──────────────────────────────────────── +# Override only what differs from the inherited defaults. +model: + optimizer_backend: openai_chat # openai_chat | claude_chat | qwen_chat | minimax_chat + target_backend: openai_chat # … plus codex_exec / claude_code_exec for target only + reasoning_effort: medium diff --git a/skillopt/envs/_template/env_template.py b/skillopt/envs/_template/env_template.py new file mode 100644 index 0000000..330b953 --- /dev/null +++ b/skillopt/envs/_template/env_template.py @@ -0,0 +1,151 @@ +""" +Benchmark Environment Template +=============================== +Copy this file and implement the TODO sections to add a new benchmark. + +The EnvAdapter is responsible for: + 1. Building per-batch environment managers (train and eval splits). + 2. Running rollouts under the current skill document. + 3. Reflecting on those rollouts into raw patch dicts. + 4. Reporting the distinct task types in your data (for stratified + sampling). + +For a fully worked example see ``skillopt/envs/officeqa/``. +""" +from __future__ import annotations + +from skillopt.datasets.base import BatchSpec +from skillopt.envs.base import EnvAdapter +from skillopt.envs._template.loader_template import TemplateBenchmarkLoader + + +class TemplateBenchmarkEnv(EnvAdapter): + """ + Environment adapter for . + + Rename this class. Each abstract method below is required by + :class:`skillopt.envs.base.EnvAdapter`. The template implementations + are minimal so this file is importable and instantiable; replace the + TODOs with real logic. + """ + + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "split_dir", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + workers: int = 4, + analyst_workers: int = 4, + failure_only: bool = False, + minibatch_size: int = 8, + edit_budget: int = 4, + seed: int = 42, + limit: int = 0, + max_completion_tokens: int = 4096, + ) -> None: + self.workers = workers + self.analyst_workers = analyst_workers + self.failure_only = failure_only + self.minibatch_size = minibatch_size + self.edit_budget = edit_budget + self.max_completion_tokens = int(max_completion_tokens) + self.dataloader = TemplateBenchmarkLoader( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + seed=seed, + limit=limit, + ) + + # ── Lifecycle hooks ──────────────────────────────────────────────── + + def setup(self, cfg: dict) -> None: + super().setup(cfg) + self.dataloader.setup(cfg) + + def get_dataloader(self): + return self.dataloader + + # ── Batch → env manager ──────────────────────────────────────────── + + def build_env_from_batch(self, batch: BatchSpec, **kwargs): + # Dataset-backed envs typically just pass items straight through. + return list(batch.payload or []) + + def build_train_env(self, batch_size: int, seed: int, **kwargs): + batch = self.dataloader.build_train_batch( + batch_size=batch_size, seed=seed, **kwargs + ) + return self.build_env_from_batch(batch, **kwargs) + + def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs): + batch = self.dataloader.build_eval_batch( + env_num=env_num, split=split, seed=seed, **kwargs + ) + return self.build_env_from_batch(batch, **kwargs) + + # ── Rollout: run episodes under current skill ────────────────────── + + def rollout( + self, + env_manager, + skill_content: str, + out_dir: str, + **kwargs, + ) -> list[dict]: + """ + Run a batch of episodes under the current skill. + + TODO: replace this loop with your real rollout. For each item: + 1. Build the prompt using `skill_content` as the system message. + 2. Call your target model. + 3. Score the prediction. + 4. Return a dict with at minimum: ``id`` (str), ``hard`` (0|1), + ``soft`` (float in [0, 1]). Add any env-specific extras you + need for reflect() — they will be preserved on + ``RolloutResult.extras``. + """ + items: list[dict] = env_manager + results: list[dict] = [] + for item in items: + # ── REPLACE THIS BLOCK WITH YOUR REAL ROLLOUT ── + results.append( + { + "id": str(item.get("id", "")), + "hard": 0, + "soft": 0.0, + "predicted_answer": "", + "question": item.get("question", ""), + "fail_reason": "template rollout — not implemented", + } + ) + return results + + # ── Reflect (inherited) ───────────────────────────────────────────── + # + # ``reflect`` is inherited from ``EnvAdapter``: the default delegates to + # ``skillopt.gradient.reflect.run_minibatch_reflect`` using your + # ``analyst_error_*`` / ``analyst_success_*`` prompts. You do NOT need to + # implement it — override only if your benchmark needs custom reflection. + + # ── Stratification hint ──────────────────────────────────────────── + + def get_task_types(self) -> list[str]: + """Distinct task-type strings used for stratified sampling.""" + seen: list[str] = [] + all_items = ( + self.dataloader.train_items + + self.dataloader.val_items + + self.dataloader.test_items + ) + for item in all_items: + tt = str(item.get("task_type") or "template") + if tt not in seen: + seen.append(tt) + return seen or ["template"] diff --git a/skillopt/envs/_template/loader_template.py b/skillopt/envs/_template/loader_template.py new file mode 100644 index 0000000..fa8bd44 --- /dev/null +++ b/skillopt/envs/_template/loader_template.py @@ -0,0 +1,87 @@ +""" +Benchmark Data Loader Template +================================ +Copy this file and implement ``load_split_items`` to load your benchmark +data. The loader is a :class:`skillopt.datasets.base.SplitDataLoader` +subclass — the base class handles both ``split_mode="split_dir"`` (read +an existing train/val/test layout) and ``split_mode="ratio"`` (build the +splits from a single raw file deterministically). + +For a fully worked example see +``skillopt/envs/officeqa/dataloader.py``. +""" +from __future__ import annotations + +import json +from pathlib import Path + +from skillopt.datasets.base import SplitDataLoader + + +def _normalize_item(raw: dict) -> dict: + """ + Normalise one raw entry into the dict shape SkillOpt expects. + + The only **hard** requirement is ``"id"`` (str). Add whatever extra + fields your :class:`TemplateBenchmarkEnv.rollout` needs. + """ + return { + "id": str(raw.get("uid") or raw.get("id") or ""), + "question": str(raw.get("question") or raw.get("prompt") or ""), + "ground_truth": str(raw.get("ground_truth") or raw.get("answer") or ""), + "task_type": str(raw.get("category") or raw.get("task_type") or "template"), + # ── add benchmark-specific keys here ── + } + + +class TemplateBenchmarkLoader(SplitDataLoader): + """ + Data loader for . + + Subclass note: you usually only need to implement + :meth:`load_split_items`. The base class drives ``setup(cfg)``, + materialises ratio-mode splits, exposes ``train_items``, + ``val_items``, ``test_items``, and builds ``BatchSpec`` objects on + demand. + + If you want to support ``split_mode="ratio"`` (auto-split a single + file into train/val/test), also implement + :meth:`load_raw_items(data_path)` returning the full list of items. + """ + + def load_split_items(self, split_path: str) -> list[dict]: + """Load all items for one split directory. + + ``split_path`` is e.g. ``data/your_benchmark/train/``. Return a + list of dicts, each shaped like :func:`_normalize_item`'s output. + """ + path = Path(split_path) + + json_files = sorted(path.glob("*.json")) + if json_files: + with json_files[0].open(encoding="utf-8") as f: + payload = json.load(f) + if not isinstance(payload, list): + raise ValueError( + f"Expected JSON array at top level of {json_files[0]}" + ) + return [_normalize_item(row) for row in payload] + + jsonl_files = sorted(path.glob("*.jsonl")) + if jsonl_files: + items: list[dict] = [] + with jsonl_files[0].open(encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + items.append(_normalize_item(json.loads(line))) + return items + + raise FileNotFoundError( + f"No .json or .jsonl file found in {split_path}" + ) + + # Optional — only needed if you intend to use ``split_mode='ratio'``. + # def load_raw_items(self, data_path: str) -> list[dict]: + # ... diff --git a/skillopt/envs/alfworld/__init__.py b/skillopt/envs/alfworld/__init__.py new file mode 100644 index 0000000..e9a28ff --- /dev/null +++ b/skillopt/envs/alfworld/__init__.py @@ -0,0 +1,5 @@ +"""ALFWorld environment adapter for ReflACT.""" + +from skillopt.envs.alfworld.adapter import ALFWorldAdapter + +__all__ = ["ALFWorldAdapter"] diff --git a/skillopt/envs/alfworld/adapter.py b/skillopt/envs/alfworld/adapter.py new file mode 100644 index 0000000..18db01b --- /dev/null +++ b/skillopt/envs/alfworld/adapter.py @@ -0,0 +1,428 @@ +"""ALFWorld environment adapter for ReflACT. + +Connects the ReflACT training loop to ALFWorld by implementing +:class:`~skillopt.envs.base.EnvAdapter`. +""" +from __future__ import annotations + +from dataclasses import dataclass +import json +import os + +from skillopt.datasets.base import BatchSpec +from skillopt.envs.base import EnvAdapter +from skillopt.envs.alfworld.dataloader import ALFWorldDataLoader +from skillopt.envs.alfworld.rollout import ( + build_alfworld_env, + run_alfworld_batch, + TASKS, +) +from skillopt.utils import compute_score + + +@dataclass(frozen=True) +class ALFWorldBatchRun: + """Lazy ALFWorld batch description. + + The adapter materializes this in rollout chunks so a large evaluation set + does not keep every ALFWorld simulator open at once. + """ + + env_num: int + eval_dataset: str + seed: int + is_train: bool + workers: int + specific_gamefiles: list[str] | None = None + result_ids: list[str] | None = None + items: list[dict] | None = None + + def __iter__(self): + return iter(self.items or []) + + def __len__(self) -> int: + return int(self.env_num or 0) + + +class ALFWorldAdapter(EnvAdapter): + """ALFWorld environment adapter. + + Parameters + ---------- + max_steps : int + Maximum steps per ALFWorld episode (default 50). + max_api_workers : int + Maximum concurrent API calls during rollout (default 8). + analyst_workers : int + Parallel workers for analyst stage (default 16). + failure_only : bool + If True, only run error analyst (skip success analyst). + minibatch_size : int + Trajectories per analyst group, M (default 8). + edit_budget : int + Maximum edits per minibatch, L (default 4). + """ + + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "split_dir", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + seed: int = 42, + limit: int = 0, + train_size: int = 0, + max_steps: int = 50, + workers: int = 8, + max_api_workers: int = 8, + analyst_workers: int = 16, + failure_only: bool = False, + minibatch_size: int = 8, + edit_budget: int = 4, + max_completion_tokens: int = 16384, + ) -> None: + self.max_steps = max_steps + self.workers = max(int(workers or 1), 1) + self.max_api_workers = max_api_workers + self.max_completion_tokens = int(max_completion_tokens) + self.analyst_workers = analyst_workers + self.failure_only = failure_only + self.minibatch_size = minibatch_size + self.edit_budget = edit_budget + self.dataloader = ALFWorldDataLoader( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + seed=seed, + limit=limit, + train_size=train_size, + ) + self._traj_cache: dict[str, dict | None] = {} + + def setup(self, cfg: dict) -> None: + super().setup(cfg) + self.dataloader.setup(cfg) + + def _load_traj_data(self, item: dict) -> dict | None: + gamefile = str(item.get("gamefile") or "").strip() + if not gamefile: + return None + if gamefile in self._traj_cache: + return self._traj_cache[gamefile] + + traj_path = os.path.join(os.path.dirname(gamefile), "traj_data.json") + try: + with open(traj_path, encoding="utf-8") as f: + data = json.load(f) + except Exception: + data = None + self._traj_cache[gamefile] = data + return data + + @staticmethod + def _unique_lines(values: list[str], *, limit: int = 0) -> list[str]: + lines: list[str] = [] + seen: set[str] = set() + for raw in values: + line = str(raw or "").strip() + if not line or line in seen: + continue + seen.add(line) + lines.append(line) + if limit > 0 and len(lines) >= limit: + break + return lines + + @staticmethod + def _format_high_pddl(high_pddl: list[dict]) -> list[str]: + steps: list[str] = [] + for idx, step in enumerate(high_pddl or [], start=1): + discrete = step.get("discrete_action") or {} + action = str(discrete.get("action") or "").strip() + args = [str(arg).strip() for arg in (discrete.get("args") or []) if str(arg).strip()] + if action and args: + text = f"{action}({', '.join(args)})" + elif action: + text = action + else: + planner_action = step.get("planner_action") or {} + text = str(planner_action.get("action") or "").strip() + if text: + steps.append(f"{idx}. {text}") + return steps + + def _build_reference_bundle(self, item: dict) -> dict: + data = self._load_traj_data(item) + if not data: + return {} + + anns = ((data.get("turk_annotations") or {}).get("anns") or []) + task_descs = self._unique_lines( + [ann.get("task_desc", "") for ann in anns], + limit=3, + ) + high_descs = self._unique_lines( + [step for ann in anns for step in (ann.get("high_descs") or [])], + limit=12, + ) + pddl_params = { + key: value + for key, value in (data.get("pddl_params") or {}).items() + if value not in ("", None, [], {}) + } + scene = data.get("scene") or {} + scene_summary = { + key: scene.get(key) + for key in ("floor_plan", "scene_num", "dirty_and_empty") + if scene.get(key) not in ("", None, [], {}) + } + high_pddl = self._format_high_pddl((data.get("plan") or {}).get("high_pddl") or []) + task_type = str(data.get("task_type") or item.get("task_type") or "").strip() + return { + "task_type": task_type, + "task_descs": task_descs, + "high_descs": high_descs, + "pddl_params": pddl_params, + "high_pddl": high_pddl, + "scene_summary": scene_summary, + } + + def build_reference_text(self, item: dict) -> str: + bundle = self._build_reference_bundle(item) + if not bundle: + return "" + + parts: list[str] = [] + if bundle["task_type"]: + parts.append(f"## Reference Task Type\n{bundle['task_type']}") + if bundle["task_descs"]: + parts.append( + "## Reference Human Task Descriptions\n" + + "\n".join(f"- {line}" for line in bundle["task_descs"]) + ) + if bundle["high_descs"]: + parts.append( + "## Reference Human High-Level Steps\n" + + "\n".join(f"{idx}. {line}" for idx, line in enumerate(bundle["high_descs"], start=1)) + ) + if bundle["pddl_params"]: + parts.append( + "## Reference PDDL Params\n" + + "\n".join(f"- {key}: {value}" for key, value in bundle["pddl_params"].items()) + ) + if bundle["high_pddl"]: + parts.append( + "## Reference Planner High-Level Plan\n" + "\n".join(bundle["high_pddl"]) + ) + if bundle["scene_summary"]: + parts.append( + "## Reference Scene Summary\n" + + "\n".join(f"- {key}: {value}" for key, value in bundle["scene_summary"].items()) + ) + return "\n\n".join(parts) + + def get_reference_metadata(self, item: dict) -> dict: + bundle = self._build_reference_bundle(item) + if not bundle: + return {"fields": [], "preview": ""} + + fields: list[str] = [] + previews: list[str] = [] + if bundle["task_type"]: + fields.append("task_type") + previews.append(f"[task_type] {bundle['task_type']}") + if bundle["task_descs"]: + fields.append("task_desc") + previews.append("[task_desc]\n" + "\n".join(bundle["task_descs"][:2])) + if bundle["high_descs"]: + fields.append("high_descs") + previews.append("[high_descs]\n" + "\n".join(bundle["high_descs"][:3])) + if bundle["pddl_params"]: + fields.append("pddl_params") + previews.append( + "[pddl_params]\n" + + "\n".join( + f"{key}: {value}" for key, value in list(bundle["pddl_params"].items())[:4] + ) + ) + if bundle["high_pddl"]: + fields.append("plan.high_pddl") + previews.append("[plan.high_pddl]\n" + "\n".join(bundle["high_pddl"][:3])) + if bundle["scene_summary"]: + fields.append("scene") + previews.append( + "[scene]\n" + + "\n".join( + f"{key}: {value}" for key, value in bundle["scene_summary"].items() + ) + ) + return { + "fields": fields, + "preview": "\n\n".join(previews)[:600], + } + + @staticmethod + def _infer_dataset_from_gamefile(gamefile: str) -> tuple[str, bool]: + path = str(gamefile or "") + if "/valid_seen/" in path: + return "eval_in_distribution", False + if "/valid_unseen/" in path: + return "eval_out_of_distribution", False + return "train", True + + def get_dataloader(self): + return self.dataloader + + def _comparison_items(self, items: list[dict]) -> list[dict]: + enriched: list[dict] = [] + for item in items: + row = dict(item) + bundle = self._build_reference_bundle(row) + if bundle.get("task_descs"): + row["task_description"] = bundle["task_descs"][0] + elif bundle.get("task_type"): + row["task_description"] = bundle["task_type"] + enriched.append(row) + return enriched + + def requires_ray(self) -> bool: + return False + + def build_env_from_batch(self, batch: BatchSpec, **kwargs): + gamefiles = list(batch.metadata.get("gamefiles") or []) + result_ids = list(batch.metadata.get("result_ids") or []) + items = self._comparison_items(list(batch.payload or [])) + return ALFWorldBatchRun( + env_num=batch.batch_size, + eval_dataset=batch.metadata.get("eval_dataset", batch.split), + seed=batch.seed, + is_train=batch.metadata.get("is_train", batch.phase == "train"), + specific_gamefiles=gamefiles or None, + result_ids=result_ids or None, + items=items, + workers=self.workers, + ) + + def build_train_env(self, batch_size: int, seed: int, **kwargs): + batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs): + batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def rollout( + self, + env_manager, + skill_content: str, + out_dir: str, + **kwargs, + ) -> list[dict]: + results_path = os.path.join(out_dir, "results.jsonl") + os.makedirs(out_dir, exist_ok=True) + + # Resume support + if os.path.exists(results_path): + existing: list[dict] = [] + with open(results_path) as f: + for line in f: + try: + existing.append(json.loads(line)) + except Exception: + pass + if existing: + return existing + + if isinstance(env_manager, ALFWorldBatchRun): + results = self._run_batch( + env_manager, + skill_content=skill_content, + out_dir=out_dir, + ) + else: + results = run_alfworld_batch( + env_manager=env_manager, + skill_content=skill_content, + max_steps=self.max_steps, + out_root=out_dir, + max_api_workers=self.max_api_workers, + max_completion_tokens=self.max_completion_tokens, + result_ids=getattr(env_manager, "_skillopt_result_ids", None), + ) + + with open(results_path, "w") as f: + for r in results: + f.write(json.dumps(r, ensure_ascii=False) + "\n") + + return results + + @staticmethod + def _close_env(env_manager) -> None: + close = getattr(env_manager, "close", None) + if callable(close): + close() + + def _run_batch( + self, + batch: ALFWorldBatchRun, + skill_content: str, + out_dir: str, + *, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + ) -> list[dict]: + total = int(batch.env_num or 0) + if total <= 0: + return [] + + workers = max(1, min(int(batch.workers or self.workers), total)) + if total > workers: + print( + f" [alfworld rollout] episodes={total} " + f"env_workers={workers} chunks={(total + workers - 1) // workers}" + ) + + all_results: list[dict] = [] + for start in range(0, total, workers): + chunk_size = min(workers, total - start) + chunk_gamefiles = ( + batch.specific_gamefiles[start:start + chunk_size] + if batch.specific_gamefiles + else None + ) + chunk_ids = ( + batch.result_ids[start:start + chunk_size] + if batch.result_ids + else [f"env_{idx:03d}" for idx in range(start, start + chunk_size)] + ) + chunk_env = build_alfworld_env( + env_num=chunk_size, + eval_dataset=batch.eval_dataset, + seed=batch.seed + start, + is_train=batch.is_train, + specific_gamefiles=chunk_gamefiles, + ) + try: + chunk_results = run_alfworld_batch( + env_manager=chunk_env, + skill_content=skill_content, + max_steps=self.max_steps, + out_root=out_dir, + max_api_workers=min(self.max_api_workers, chunk_size), + max_completion_tokens=self.max_completion_tokens, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + result_ids=chunk_ids, + ) + finally: + self._close_env(chunk_env) + all_results.extend(chunk_results) + return all_results + + def get_task_types(self) -> list[str]: + return list(TASKS) diff --git a/skillopt/envs/alfworld/dataloader.py b/skillopt/envs/alfworld/dataloader.py new file mode 100644 index 0000000..80fcd70 --- /dev/null +++ b/skillopt/envs/alfworld/dataloader.py @@ -0,0 +1,123 @@ +"""ALFWorld task dataloader.""" +from __future__ import annotations + +from skillopt.datasets.base import BatchSpec, SplitDataLoader + + +class ALFWorldDataLoader(SplitDataLoader): + """ALFWorld batch planner. + + In split_dir mode, batches are fixed gamefile items so ablations differ + only in how the same training set is batched. + """ + + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "split_dir", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + seed: int = 42, + limit: int = 0, + train_size: int = 0, + **kwargs, + ) -> None: + super().__init__( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + seed=seed, + limit=limit, + ) + self.train_size_override = int(train_size or 0) + + @staticmethod + def _metadata_for_items(items: list[dict], split: str, phase: str) -> dict: + gamefiles = [str(item.get("gamefile") or "") for item in items] + if any(not gamefile for gamefile in gamefiles): + raise ValueError("ALFWorld split items must contain non-empty gamefile paths.") + eval_dataset = "train" + is_train = phase == "train" + first = gamefiles[0] if gamefiles else "" + if "/valid_seen/" in first: + eval_dataset = "eval_in_distribution" + is_train = False + elif "/valid_unseen/" in first: + eval_dataset = "eval_out_of_distribution" + is_train = False + return { + "eval_dataset": eval_dataset, + "is_train": is_train, + "gamefiles": gamefiles, + "result_ids": [str(item.get("id") or idx) for idx, item in enumerate(items)], + } + + def get_train_size(self) -> int: + if self.train_size_override > 0: + return self.train_size_override + return super().get_train_size() + + def build_train_batch(self, batch_size: int, seed: int, **kwargs) -> BatchSpec: + batch = super().build_train_batch(batch_size=batch_size, seed=seed, **kwargs) + items = list(batch.payload or []) + batch.metadata.update(self._metadata_for_items(items, "train", "train")) + return BatchSpec( + phase="train", + split="train", + seed=seed, + batch_size=len(items), + payload=items, + metadata=batch.metadata, + ) + + def plan_train_epoch( + self, + *, + epoch: int, + steps_per_epoch: int, + accumulation: int, + batch_size: int, + seed: int, + **kwargs, + ) -> list[BatchSpec]: + batches = super().plan_train_epoch( + epoch=epoch, + steps_per_epoch=steps_per_epoch, + accumulation=accumulation, + batch_size=batch_size, + seed=seed, + **kwargs, + ) + for batch in batches: + items = list(batch.payload or []) + batch.metadata.update(self._metadata_for_items(items, "train", "train")) + return batches + + def build_eval_batch( + self, + env_num: int, + split: str, + seed: int, + **kwargs, + ) -> BatchSpec: + batch = super().build_eval_batch( + env_num=env_num, + split=split, + seed=seed, + **kwargs, + ) + items = list(batch.payload or []) + batch.metadata.update(self._metadata_for_items(items, split, "eval")) + return BatchSpec( + phase="eval", + split=split, + seed=seed, + batch_size=len(items), + payload=items, + metadata=batch.metadata, + ) diff --git a/skillopt/envs/alfworld/prompts/analyst_error.md b/skillopt/envs/alfworld/prompts/analyst_error.md new file mode 100644 index 0000000..f464716 --- /dev/null +++ b/skillopt/envs/alfworld/prompts/analyst_error.md @@ -0,0 +1,55 @@ +You are an expert failure-analysis agent for ALFWorld embodied household tasks. + +You will be given MULTIPLE failed agent trajectories from a single minibatch +and the current skill document. +Your job is to identify the most important COMMON failure patterns across +the batch and propose a concise set of skill edits. + +## ALFWorld Task Types +- pick_and_place: Put object in/on a receptacle +- pick_two_obj_and_place: Put two instances of an object in/on a receptacle +- look_at_obj_in_light: Examine an object under a desklamp +- pick_heat_then_place_in_recep: Heat an object and put it in/on a receptacle +- pick_cool_then_place_in_recep: Cool an object and put it in/on a receptacle +- pick_clean_then_place_in_recep: Clean an object and put it in/on a receptacle + +## Failure Type Categories +- **navigation_loop**: the agent revisits the same locations repeatedly without progress +- **missed_object**: the agent fails to pick up a visible/reachable goal object +- **wrong_sequence**: the agent performs actions in the wrong order (e.g., placing before transforming) +- **premature_stop**: the agent stops or gets stuck before completing all goal conditions +- **action_loop**: the agent repeats the same action without advancing +- **appliance_error**: the agent misuses or skips an appliance (microwave, fridge, sink) +- **rule_missing**: the skill lacks a relevant rule for this situation +- **rule_wrong**: an existing skill rule is misleading or incorrect +- **rule_ignored**: the skill has the right rule but the agent did not follow it +- **other**: none of the above + +## Analysis Process +1. Read ALL trajectories in the minibatch. +2. Identify the most prevalent, systematic failure patterns across them. +3. For each pattern, classify its failure type. +4. Propose skill edits that address the COMMON patterns — not individual edge cases. +5. Edits must be generalizable; do not hardcode task-specific values. +6. Only patch gaps in the skill — do not duplicate existing content. + +You will be told the maximum number of edits (the budget L). Produce AT MOST L edits, +focusing on the highest-impact patterns. You may produce fewer if warranted. + +Respond ONLY with a valid JSON object (no markdown fences, no extra text): +{ + "batch_size": , + "failure_summary": [ + {"failure_type": "", "count": , "description": ""} + ], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +Only include edits that are needed. "edits" can be an empty list if no patch is warranted. diff --git a/skillopt/envs/alfworld/prompts/analyst_success.md b/skillopt/envs/alfworld/prompts/analyst_success.md new file mode 100644 index 0000000..957d3a8 --- /dev/null +++ b/skillopt/envs/alfworld/prompts/analyst_success.md @@ -0,0 +1,33 @@ +You are an expert success-pattern analyst for AI agents operating in ALFWorld, +a text-based embodied household environment. + +You will be given MULTIPLE successful agent trajectories from a single minibatch +and the current skill document. Your job is to identify generalizable behavior +patterns that are COMMON across the batch and worth encoding in the skill. + +## Rules +- Only propose patches for patterns NOT already covered in the skill. +- Focus on patterns that appear across MULTIPLE trajectories in the batch. +- Be concise. Patterns must generalize beyond specific tasks. +- Prefer reinforcing existing sections over adding new top-level sections. +- If the agents' success involved efficient exploration or smart appliance usage, + consider reinforcing that in the patch. + +You will be told the maximum number of edits (the budget L). Produce AT MOST L edits, +focusing on the most broadly applicable patterns. You may produce fewer if warranted. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "success_patterns": ["", ""], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +"edits" may be empty if the skill already covers all observed patterns. diff --git a/skillopt/envs/alfworld/prompts/rollout_no_history.md b/skillopt/envs/alfworld/prompts/rollout_no_history.md new file mode 100644 index 0000000..d1d605b --- /dev/null +++ b/skillopt/envs/alfworld/prompts/rollout_no_history.md @@ -0,0 +1,8 @@ + +You are an expert agent operating in the ALFRED Embodied Environment. +Your current observation is: {current_observation} +Your admissible actions of the current situation are: [{admissible_actions}]. + +Now it's your turn to take an action. +You should first reason step-by-step about the current situation. This reasoning process MUST be enclosed within tags. +Once you've finished your reasoning, you should choose an admissible action for current step and present it within tags. diff --git a/skillopt/envs/alfworld/prompts/rollout_with_history.md b/skillopt/envs/alfworld/prompts/rollout_with_history.md new file mode 100644 index 0000000..f0a635d --- /dev/null +++ b/skillopt/envs/alfworld/prompts/rollout_with_history.md @@ -0,0 +1,9 @@ + +You are an expert agent operating in the ALFRED Embodied Environment. Your task is to: {task_description} +Prior to this step, you have already taken {step_count} step(s). Below are the most recent {history_length} observations and the corresponding actions you took: {action_history} +You are now at step {current_step} and your current observation is: {current_observation} +Your admissible actions of the current situation are: [{admissible_actions}]. + +Now it's your turn to take an action. +You should first reason step-by-step about the current situation. This reasoning process MUST be enclosed within tags. +Once you've finished your reasoning, you should choose an admissible action for current step and present it within tags. diff --git a/skillopt/envs/alfworld/prompts/rollout_with_memory.md b/skillopt/envs/alfworld/prompts/rollout_with_memory.md new file mode 100644 index 0000000..c90dc7f --- /dev/null +++ b/skillopt/envs/alfworld/prompts/rollout_with_memory.md @@ -0,0 +1,16 @@ + +You are an expert agent operating in the ALFRED Embodied Environment. Your task is to: {task_description} + +## Retrieved Relevant Experience + +{retrieved_memories} + +## Current Progress + +Prior to this step, you have already taken {step_count} step(s). Below are the most recent {history_length} observations and the corresponding actions you took: {action_history} +You are now at step {current_step} and your current observation is: {current_observation} +Your admissible actions of the current situation are: [{admissible_actions}]. + +Now it's your turn to take an action. +You should first reason step-by-step about the current situation. This reasoning process MUST be enclosed within tags. +Once you've finished your reasoning, you should choose an admissible action for current step and present it within tags. diff --git a/skillopt/envs/alfworld/reflect.py b/skillopt/envs/alfworld/reflect.py new file mode 100644 index 0000000..a32d989 --- /dev/null +++ b/skillopt/envs/alfworld/reflect.py @@ -0,0 +1,4 @@ +"""ALFWorld Reflect stage. + +Prompts are now loaded from .md files by the base adapter. +""" diff --git a/skillopt/envs/alfworld/rollout.py b/skillopt/envs/alfworld/rollout.py new file mode 100644 index 0000000..18264c3 --- /dev/null +++ b/skillopt/envs/alfworld/rollout.py @@ -0,0 +1,366 @@ +"""ALFWorld rollout module for ReflACT. + +Provides: + - build_alfworld_env(): build ALFWorld environment (wraps vendored SkillRL env) + - run_alfworld_batch(): run a batch of ALFWorld episodes in parallel + - TASKS: list of ALFWorld task types +""" +from __future__ import annotations + +import concurrent.futures +import json +import os +import re + +from skillopt.model import chat_target + +# ── Constants ───────────────────────────────────────────────────────────────── + +TASKS = [ + "pick_and_place", + "pick_two_obj_and_place", + "look_at_obj_in_light", + "pick_heat_then_place_in_recep", + "pick_cool_then_place_in_recep", + "pick_clean_then_place_in_recep", +] + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _get_task_type(gamefile: str) -> str: + for task in TASKS: + if task in gamefile: + return task + return "other" + + +def _extract_action(model_response: str) -> str | None: + match = re.search(r"(.*?)", model_response, re.DOTALL) + return match.group(1).strip() if match else None + + +def _extract_think(model_response: str) -> str | None: + match = re.search(r"(.*?)", model_response, re.DOTALL) + return match.group(1).strip() if match else None + + +def _build_skill_prompt(skill_content: str) -> str: + """Build the skill section to inject into the agent's system prompt.""" + if not skill_content or not skill_content.strip(): + return "" + return ( + "\n\n## Skill Knowledge\n" + "Below is a skill document with learned strategies. " + "Use these guidelines to inform your decisions:\n\n" + f"{skill_content}\n" + ) + + +def _append_diagnostic_instruction(prompt: str, diagnostic_instruction: str) -> str: + if not diagnostic_instruction or not diagnostic_instruction.strip(): + return prompt + return f"{prompt}\n\n## Training Readout\n{diagnostic_instruction.strip()}\n" + + +def _resolve_alfworld_gamefile(gamefile: str) -> str: + path = os.path.expanduser(os.path.expandvars(str(gamefile))) + if os.path.isabs(path): + return path + + data_root = os.environ.get("ALFWORLD_DATA", "").strip() + if not data_root: + return path + + root = os.path.expanduser(os.path.expandvars(data_root)) + return os.path.abspath(os.path.join(root, path)) + + +def _resolve_alfworld_gamefiles(gamefiles: list[str] | None) -> list[str] | None: + if gamefiles is None: + return None + return [_resolve_alfworld_gamefile(gamefile) for gamefile in gamefiles] + + +# ── Environment builder ────────────────────────────────────────────────────── + + +def build_alfworld_env( + env_num: int, + eval_dataset: str = "eval_out_of_distribution", + seed: int = 42, + is_train: bool = False, + specific_gamefiles: list[str] | None = None, +): + """Build ALFWorld environment manager. + + Args: + env_num: number of parallel environments + eval_dataset: 'eval_in_distribution' or 'eval_out_of_distribution' or train + seed: random seed + is_train: whether to use training set + + Returns: + env_manager: AlfWorldEnvironmentManager instance + """ + from functools import partial + + from omegaconf import OmegaConf + + from skillopt.envs.alfworld.vendor.alfworld_envs import build_alfworld_envs + from skillopt.envs.alfworld.vendor.alfworld_projection import alfworld_projection + from skillopt.envs.alfworld.vendor.env_manager import AlfWorldEnvironmentManager + + HERE = os.path.dirname(os.path.abspath(__file__)) + + alf_config_path = os.path.join(HERE, "vendor", "config_tw.yaml") + env_kwargs = {"eval_dataset": eval_dataset} + resolved_gamefiles = _resolve_alfworld_gamefiles(specific_gamefiles) + + envs = build_alfworld_envs( + alf_config_path, + seed=seed, + env_num=env_num, + group_n=1, + is_train=is_train, + env_kwargs=env_kwargs, + resources_per_worker=None, + gamefiles=resolved_gamefiles, + ) + + config = OmegaConf.create( + { + "env": { + "history_length": 2, + "env_name": "alfworld/AlfredTWEnv", + } + } + ) + + projection_f = partial(alfworld_projection) + env_manager = AlfWorldEnvironmentManager(envs, projection_f, config) + return env_manager + + +# ── Batch rollout ───────────────────────────────────────────────────────────── + + +def run_alfworld_batch( + env_manager, + skill_content: str, + max_steps: int = 50, + out_root: str = "", + max_api_workers: int = 8, + temperature: float = 0.4, + max_completion_tokens: int = 16384, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + result_ids: list[str] | None = None, +) -> list[dict]: + """Run a batch of ALFWorld episodes. + + Returns a list of result dicts compatible with SkillOpt pipeline: + [ + { + "id": "_", + "hard": 0 or 1, + "soft": 0.0 or 1.0, + "n_turns": , + "fail_reason": "", + "agent_ok": True, + "task_type": "", + "gamefile": "", + "task_description": "", + }, + ... + ] + + Also saves conversation.json per environment in out_root/predictions// + """ + skill_prompt = _build_skill_prompt(skill_content) + + obs, infos = env_manager.reset({}) + env_num = len(obs["text"]) + env_dones = [False] * env_num + overall_success = [False] * env_num + + # Build per-env metadata + env_meta: list[dict] = [] + for i in range(env_num): + gamefile = infos[i].get("extra.gamefile", "") if isinstance(infos[i], dict) else "" + task_type = _get_task_type(gamefile) + # Extract task description from initial observation + task_desc = "" + anchor_text = obs["anchor"][i] if "anchor" in obs else "" + task_start = anchor_text.find("Your task is to: ") + if task_start != -1: + task_desc = anchor_text[task_start + len("Your task is to: "):].strip() + + env_meta.append({ + "gamefile": gamefile, + "task_type": task_type, + "task_description": task_desc, + }) + + # Per-env conversation records + conversations: list[list[dict]] = [[] for _ in range(env_num)] + + for step_idx in range(max_steps): + if all(env_dones): + break + + active_indices = [i for i in range(env_num) if not env_dones[i]] + + # Build prompts with skill injection + prompts: dict[int, str] = {} + for i in active_indices: + prompt = obs["text"][i] + if skill_prompt: + # Inject skill before the action instruction + prompt = skill_prompt + "\n" + prompt + if diagnostic_mode and diagnostic_instruction.strip(): + prompt = _append_diagnostic_instruction(prompt, diagnostic_instruction) + prompts[i] = prompt + + # Call API in parallel + actions = ["None"] * env_num + + def call_api(idx): + try: + response, _ = chat_target( + system="You are an expert agent operating in the ALFRED Embodied Environment.", + user=prompts[idx], + max_completion_tokens=max_completion_tokens, + retries=5, + stage="rollout", + timeout=None, + ) + response = (response or "").strip() + if not response: + return idx, "empty model responselook" + if _extract_action(response) is None: + return idx, "missing action taglook" + return idx, response + except Exception: + return idx, "errorlook" + + executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_api_workers) + try: + futures = {executor.submit(call_api, i): i for i in active_indices} + pending_futs = set(futures) + while pending_futs: + done, _ = concurrent.futures.wait( + pending_futs, + timeout=5, + return_when=concurrent.futures.FIRST_COMPLETED, + ) + for future in done: + pending_futs.remove(future) + try: + idx, response = future.result() + except Exception: # noqa: BLE001 + idx = futures[future] + response = "errorlook" + actions[idx] = response + finally: + executor.shutdown(wait=False, cancel_futures=True) + + # Save model responses before stepping + model_responses = {i: actions[i] for i in active_indices} + + # Step environment + obs, rewards, dones, infos = env_manager.step(actions) + + # Record trajectory + for i in active_indices: + step_record = { + "step": step_idx, + "action": _extract_action(model_responses[i]), + "reasoning": _extract_think(model_responses[i]), + "model_response": model_responses[i], + "env_feedback": obs["anchor"][i] if "anchor" in obs else "", + "reward": float(rewards[i]), + "done": bool(dones[i]), + } + conversations[i].append(step_record) + + # Update done status + for i in range(env_num): + if env_dones[i]: + continue + if dones[i]: + env_dones[i] = True + won = bool(infos[i].get("won", False)) + overall_success[i] = won + + # Build results and save conversations + results: list[dict] = [] + pred_dir = os.path.join(out_root, "predictions") if out_root else "" + + for i in range(env_num): + gamefile = env_meta[i]["gamefile"] + task_type = env_meta[i]["task_type"] + task_desc = env_meta[i]["task_description"] + n_turns = len(conversations[i]) + won = overall_success[i] + + # Generate stable task ID from env index and gamefile + task_id = str(result_ids[i]) if result_ids and i < len(result_ids) else f"env_{i:03d}" + + fail_reason = "" + if not won: + if not env_dones[i]: + fail_reason = f"Timeout after {max_steps} steps" + else: + fail_reason = "Episode ended without completing the task" + + result = { + "id": task_id, + "hard": 1 if won else 0, + "soft": 1.0 if won else 0.0, + "n_turns": n_turns, + "fail_reason": fail_reason, + "agent_ok": True, # ALFWorld agent always runs OK (no crash) + "task_type": task_type, + "gamefile": gamefile, + "task_description": task_desc, + "instruction_type": task_type, # for compatibility with v2 pipeline + } + results.append(result) + + # Save conversation + if pred_dir: + conv_dir = os.path.join(pred_dir, task_id) + os.makedirs(conv_dir, exist_ok=True) + with open(os.path.join(conv_dir, "conversation.json"), "w") as f: + json.dump(conversations[i], f, ensure_ascii=False, indent=2) + + return results + + +# ── Item loading (for compatibility with split_three_way) ──────────────────── + + +def load_alfworld_items( + eval_dataset: str, + env_num: int, + seed: int = 42, + is_train: bool = False, +) -> list[dict]: + """Create pseudo-item dicts for ALFWorld environments. + + Since ALFWorld doesn't have a static JSON dataset like SpreadsheetBench, + we create lightweight item dicts that carry enough metadata for the pipeline. + The actual environment is built dynamically. + + Returns: + List of dicts with "id" keys, one per environment slot. + """ + items = [] + for i in range(env_num): + items.append({ + "id": f"env_{i:03d}", + "eval_dataset": eval_dataset, + "env_index": i, + }) + return items diff --git a/skillopt/envs/alfworld/skills/initial.md b/skillopt/envs/alfworld/skills/initial.md new file mode 100644 index 0000000..d19ad02 --- /dev/null +++ b/skillopt/envs/alfworld/skills/initial.md @@ -0,0 +1,45 @@ +# ALFWorld Embodied Agent Skill + +## Overview +This skill guides agents operating in the ALFWorld text-based embodied environment. +The agent must complete household tasks by navigating rooms, interacting with objects, +and using appliances. Actions must be chosen from the admissible action list provided +at each step. + +**Output format**: Always output `...` for reasoning, then `...` for the chosen action. + +--- + +## Task Types + +| Type | Goal | Key Steps | +|------|------|-----------| +| Pick & Place | Put object X in/on receptacle Y | Find X -> take X -> go to Y -> put X in/on Y | +| Pick Two & Place | Put two instances of X in/on Y | Find X1 -> take -> place -> find X2 -> take -> place | +| Examine in Light | Examine object X under desklamp | Find X -> take X -> find desklamp -> use desklamp | +| Clean & Place | Clean object X and put in/on Y | Find X -> take X -> go to sink -> clean X -> go to Y -> put X | +| Heat & Place | Heat object X and put in/on Y | Find X -> take X -> go to microwave -> heat X -> go to Y -> put X | +| Cool & Place | Cool object X and put in/on Y | Find X -> take X -> go to fridge -> cool X -> go to Y -> put X | + +--- + +## General Principles + +1. **Decompose the task**: Parse the goal into ordered sub-goals (locate, acquire, transform, deliver). Complete each before moving to the next. +2. **Systematic exploration**: Search each surface and container exactly once before revisiting. Open closed containers (drawers, cabinets, fridge) before judging them empty. +3. **Grab immediately**: When a required object is visible and reachable, take it right away before moving elsewhere. +4. **Transform before placing**: If the task requires cleaning, heating, or cooling, perform the state change at the appropriate appliance before heading to the final destination. +5. **Direct delivery**: Once holding the transformed (or untransformed) goal object, navigate straight to the target receptacle and place it. +6. **Track progress**: Maintain an internal count of how many objects still need to be found and placed. Only stop searching when the count reaches zero. +7. **Avoid loops**: Never repeat the same action more than twice in a row. If stuck, move to a different unexplored location. +8. **Only choose admissible actions**: Always pick an action from the admissible action list. Do not invent actions. + +--- + +## Common Mistakes to Avoid + +- **Revisiting searched locations**: Keep track of which surfaces/containers have been checked; do not re-examine them. +- **Ignoring visible objects**: If the target object appears in the observation, pick it up immediately. +- **Skipping state changes**: Do not place an object at the destination without first cleaning/heating/cooling it when required. +- **Premature termination**: Do not stop the episode until all goal conditions are verified as met. +- **Action loops**: Repeatedly toggling or examining the same object wastes steps. Move on to new locations instead. diff --git a/skillopt/envs/alfworld/vendor/__init__.py b/skillopt/envs/alfworld/vendor/__init__.py new file mode 100644 index 0000000..93dd8cb --- /dev/null +++ b/skillopt/envs/alfworld/vendor/__init__.py @@ -0,0 +1,9 @@ +"""Vendored ALFWorld environment runtime. + +Minimal subset of SkillRL's agent_system package needed to run +ALFWorld environments with ReflACT. Original source: +https://github.com/NTU-LANTERN/SkillRL (Apache-2.0 License) +""" +from .alfworld_envs import AlfworldEnvs, build_alfworld_envs +from .alfworld_projection import alfworld_projection +from .env_manager import AlfWorldEnvironmentManager diff --git a/skillopt/envs/alfworld/vendor/alfworld_envs.py b/skillopt/envs/alfworld/vendor/alfworld_envs.py new file mode 100644 index 0000000..06b9716 --- /dev/null +++ b/skillopt/envs/alfworld/vendor/alfworld_envs.py @@ -0,0 +1,221 @@ +# Vendored from SkillRL (Apache-2.0 License) +# Original: agent_system/environments/env_package/alfworld/envs.py +# Modified: imports use pip-installed alfworld package instead of vendored copy. + +import os +import multiprocessing as mp +import traceback +import yaml +import gymnasium as gym +import numpy as np + +from alfworld.agents.environment import get_environment + + +def load_config_file(path): + assert os.path.exists(path), f"Invalid config file: {path}" + with open(path) as reader: + config = yaml.safe_load(reader) + return config + + +def compute_reward(info, multi_modal=False): + if multi_modal: + reward = 10.0 * float(info['won']) + float(info['goal_condition_success_rate']) + else: + reward = 10.0 * float(info['won']) + return reward + + +class AlfworldWorker: + """Stateful worker that holds one ALFWorld sub-environment.""" + + def __init__(self, config, seed, base_env, gamefile=None): + if gamefile: + base_env.game_files = [gamefile] + if hasattr(base_env, "num_games"): + base_env.num_games = 1 + self.env = base_env.init_env(batch_size=1) + self.env.seed(seed) + + def step(self, action): + actions = [action] + obs, scores, dones, infos = self.env.step(actions) + infos['observation_text'] = obs + return obs, scores, dones, infos + + def reset(self): + obs, infos = self.env.reset() + infos['observation_text'] = obs + return obs, infos + + +def _worker_loop(cmd_q, result_q, config, seed, is_train, eval_dataset, gamefile): + """Run one ALFWorld environment in a child process.""" + try: + env_type = config['env']['type'] + base_env = get_environment(env_type)( + config, + train_eval='train' if is_train else eval_dataset, + ) + worker = AlfworldWorker(config, seed, base_env, gamefile) + result_q.put((True, "ready")) + except BaseException: + result_q.put((False, traceback.format_exc())) + return + + while True: + cmd, payload = cmd_q.get() + if cmd == "close": + result_q.put((True, None)) + return + try: + if cmd == "reset": + result = worker.reset() + elif cmd == "step": + result = worker.step(payload) + else: + raise ValueError(f"Unknown ALFWorld worker command: {cmd}") + result_q.put((True, result)) + except BaseException: + result_q.put((False, traceback.format_exc())) + + +class _ProcessWorker: + """Small stdlib actor wrapper for one environment process.""" + + def __init__(self, ctx, config, seed, is_train, eval_dataset, gamefile=None): + self.cmd_q = ctx.Queue(maxsize=1) + self.result_q = ctx.Queue(maxsize=1) + self.process = ctx.Process( + target=_worker_loop, + args=(self.cmd_q, self.result_q, config, seed, is_train, eval_dataset, gamefile), + ) + self.process.start() + ok, payload = self.result_q.get() + if not ok: + self.close(kill=True) + raise RuntimeError(f"Failed to start ALFWorld worker:\n{payload}") + + def send(self, cmd, payload=None): + self.cmd_q.put((cmd, payload)) + + def recv(self): + ok, payload = self.result_q.get() + if not ok: + raise RuntimeError(f"ALFWorld worker failed:\n{payload}") + return payload + + def close(self, kill=False): + if self.process.is_alive() and not kill: + try: + self.send("close") + self.recv() + except Exception: + kill = True + if kill and self.process.is_alive(): + self.process.terminate() + self.process.join(timeout=5) + if self.process.is_alive(): + self.process.kill() + self.process.join(timeout=1) + self.cmd_q.close() + self.result_q.close() + + +class AlfworldEnvs(gym.Env): + """Vectorized ALFWorld environment using local process workers.""" + + def __init__(self, alf_config_path, seed, env_num, group_n, + resources_per_worker, is_train=True, env_kwargs=None, gamefiles=None): + super().__init__() + if env_kwargs is None: + env_kwargs = {} + + eval_dataset = env_kwargs.get('eval_dataset', 'eval_in_distribution') + config = load_config_file(alf_config_path) + env_type = config['env']['type'] + self.multi_modal = (env_type == 'AlfredThorEnv') + self.num_processes = env_num * group_n + self.group_n = group_n + self.gamefiles = list(gamefiles or []) + if self.gamefiles and len(self.gamefiles) != self.num_processes: + raise ValueError( + f"Expected {self.num_processes} gamefiles, got {len(self.gamefiles)}" + ) + + start_method = os.environ.get("ALFWORLD_WORKER_START_METHOD") or None + ctx = mp.get_context(start_method) if start_method else mp.get_context() + self.workers = [] + for i in range(self.num_processes): + worker_gamefile = self.gamefiles[i] if self.gamefiles else None + worker = _ProcessWorker( + ctx, + config, + seed + (i // self.group_n), + is_train, + eval_dataset, + worker_gamefile, + ) + self.workers.append(worker) + + self.prev_admissible_commands = [None for _ in range(self.num_processes)] + + def step(self, actions): + assert len(actions) == self.num_processes + + for i, worker in enumerate(self.workers): + worker.send("step", actions[i]) + results = [worker.recv() for worker in self.workers] + + text_obs_list = [] + rewards_list = [] + dones_list = [] + info_list = [] + + for i, (obs, scores, dones, info) in enumerate(results): + for k in info.keys(): + info[k] = info[k][0] + text_obs_list.append(obs[0]) + dones_list.append(dones[0]) + info_list.append(info) + self.prev_admissible_commands[i] = info['admissible_commands'] + rewards_list.append(compute_reward(info, self.multi_modal)) + + image_obs_list = None + return text_obs_list, image_obs_list, rewards_list, dones_list, info_list + + def reset(self): + for worker in self.workers: + worker.send("reset") + results = [worker.recv() for worker in self.workers] + + text_obs_list = [] + info_list = [] + + for i, (obs, info) in enumerate(results): + for k in info.keys(): + info[k] = info[k][0] + text_obs_list.append(obs[0]) + self.prev_admissible_commands[i] = info['admissible_commands'] + info_list.append(info) + + image_obs_list = None + return text_obs_list, image_obs_list, info_list + + @property + def get_admissible_commands(self): + return self.prev_admissible_commands + + def close(self): + for worker in self.workers: + worker.close() + + +def build_alfworld_envs(alf_config_path, seed, env_num, group_n, + resources_per_worker, is_train=True, env_kwargs=None, gamefiles=None): + """Build vectorized ALFWorld environments.""" + return AlfworldEnvs( + alf_config_path, seed, env_num, group_n, + resources_per_worker, is_train, env_kwargs, gamefiles, + ) diff --git a/skillopt/envs/alfworld/vendor/alfworld_projection.py b/skillopt/envs/alfworld/vendor/alfworld_projection.py new file mode 100644 index 0000000..8c499ff --- /dev/null +++ b/skillopt/envs/alfworld/vendor/alfworld_projection.py @@ -0,0 +1,60 @@ +# Vendored from SkillRL (Apache-2.0 License) +# Original: agent_system/environments/env_package/alfworld/projection.py + +from typing import List +import re + + +def alfworld_projection(actions: List[str], action_pools: List[List[str]]): + """Process raw model outputs into valid ALFWorld actions. + + Extracts text from ``...`` tags and validates that + the response also contains ``...`` tags. + + Parameters + ---------- + actions : list[str] + Raw model outputs, one per environment. + action_pools : list[list[str]] + Admissible action lists per environment (unused but kept for API compat). + + Returns + ------- + actions : list[str] + Cleaned action strings. + valids : list[int] + 1 if the action was successfully parsed, 0 otherwise. + """ + valids = [0] * len(actions) + + for i in range(len(actions)): + original_str = actions[i] + actions[i] = actions[i].lower() + + start_tag = "" + end_tag = "" + start_idx = actions[i].find(start_tag) + end_idx = actions[i].find(end_tag) + try: + if start_idx == -1 or end_idx == -1: + actions[i] = actions[i][-30:] + continue + + extracted_action = actions[i][start_idx + len(start_tag):end_idx].strip().lower() + actions[i] = extracted_action + valids[i] = 1 + + except Exception: + actions[i] = actions[i][-30:] + + # Require ... + think_start_idx = original_str.find("") + think_end_idx = original_str.find("") + if think_start_idx == -1 or think_end_idx == -1: + valids[i] = 0 + + # Reject responses containing Chinese characters + if re.search(r'[\u4e00-\u9fff]', original_str): + valids[i] = 0 + + return actions, valids diff --git a/skillopt/envs/alfworld/vendor/alfworld_prompts.py b/skillopt/envs/alfworld/vendor/alfworld_prompts.py new file mode 100644 index 0000000..bb7ec49 --- /dev/null +++ b/skillopt/envs/alfworld/vendor/alfworld_prompts.py @@ -0,0 +1,8 @@ +# Vendored from SkillRL (Apache-2.0 License) +# Original: agent_system/environments/prompts/alfworld.py + +from skillopt.prompts import load_prompt + +ALFWORLD_TEMPLATE_NO_HIS = load_prompt("rollout_no_history", env="alfworld") +ALFWORLD_TEMPLATE = load_prompt("rollout_with_history", env="alfworld") +ALFWORLD_TEMPLATE_WITH_MEMORY = load_prompt("rollout_with_memory", env="alfworld") diff --git a/skillopt/envs/alfworld/vendor/config_tw.yaml b/skillopt/envs/alfworld/vendor/config_tw.yaml new file mode 100644 index 0000000..e9bf169 --- /dev/null +++ b/skillopt/envs/alfworld/vendor/config_tw.yaml @@ -0,0 +1,145 @@ +dataset: + data_path: '$ALFWORLD_DATA/json_2.1.1/train' + eval_id_data_path: '$ALFWORLD_DATA/json_2.1.1/valid_seen' # null/None to disable + eval_ood_data_path: '$ALFWORLD_DATA/json_2.1.1/valid_unseen' # null/None to disable + num_train_games: -1 # max training games (<=0 indicates full dataset) + num_eval_games: -1 # max evaluation games (<=0 indicates full dataset) + +logic: + domain: '$ALFWORLD_DATA/logic/alfred.pddl' # PDDL domain file that defines the world dynamics + grammar: '$ALFWORLD_DATA/logic/alfred.twl2' # Grammar file that defines the text feedbacks + +env: + type: 'AlfredTWEnv' # 'AlfredTWEnv' or 'AlfredThorEnv' or 'AlfredHybrid' + # regen_game_files: False # check if game is solvable by expert and save to game.tw-pddl file + domain_randomization: False # shuffle Textworld print order and object id nums + task_types: [1, 2, 3, 4, 5, 6] # task-type ids: 1 - Pick & Place, 2 - Examine in Light, 3 - Clean & Place, 4 - Heat & Place, 5 - Cool & Place, 6 - Pick Two & Place + expert_timeout_steps: 150 # max steps before timeout for expert to solve the task + expert_type: "handcoded" # 'handcoded' or 'planner'. Note: the planner is very slow for real-time use + goal_desc_human_anns_prob: 0.0 # prob of using human-annotated goal language instead of templated goals (1.0 indicates all human annotations from ALFRED) + + hybrid: + start_eps: 100000 # starting episode of hybrid training, tw-only training upto this point + thor_prob: 0.5 # prob of AlfredThorEnv during hybrid training + eval_mode: "tw" # 'tw' or 'thor' - env used for evaluation during hybrid training + + thor: + screen_width: 300 # width of THOR window + screen_height: 300 # height of THOR window + smooth_nav: False # smooth rotations, looks, and translations during navigation (very slow) + save_frames_to_disk: False # save frame PNGs to disk (useful for making videos) + save_frames_path: './videos/' # path to save frame PNGs + +controller: + type: 'oracle' # 'oracle' or 'oracle_astar' or 'mrcnn' or 'mrcnn_astar' (aka BUTLER) + debug: False + load_receps: True # load receptacle locations from precomputed dict (if available) + +mask_rcnn: + pretrained_model_path: '$ALFWORLD_DATA/detectors/mrcnn.pth' + +general: + random_seed: 42 + use_cuda: True # disable this when running on machine without cuda + visdom: False # plot training/eval curves, run with visdom server + task: 'alfred' + training_method: 'dagger' # 'dqn' or 'dagger' + save_path: './training/' # path to save pytorch models + observation_pool_capacity: 3 # k-size queue, 0 indicates no observation + hide_init_receptacles: False # remove initial observation containing navigable receptacles + + training: + batch_size: 10 + max_episode: 50000 + smoothing_eps: 0.1 + optimizer: + learning_rate: 0.001 + clip_grad_norm: 5 + + evaluate: + run_eval: True + batch_size: 10 + env: + type: "AlfredTWEnv" + + checkpoint: + report_frequency: 1000 # report every N episode + experiment_tag: 'test' # name of experiment + load_pretrained: False # during test, enable this so that the agent load your pretrained model + load_from_tag: 'not loading anything' # name of pre-trained model to load in save_path + + model: + encoder_layers: 1 + decoder_layers: 1 + encoder_conv_num: 5 + block_hidden_dim: 64 + n_heads: 1 + dropout: 0.1 + block_dropout: 0.1 + recurrent: True + +rl: + action_space: "admissible" # 'admissible' (candidates from text engine) or 'generation' (seq2seq-style generation) or 'beam_search_choice' or 'exhaustive' (not working) + max_target_length: 20 # max token length for seq2seq generation + beam_width: 10 # 1 means greedy + generate_top_k: 3 + + training: + max_nb_steps_per_episode: 50 # terminate after this many steps + learn_start_from_this_episode: 0 # delay updates until this epsiode + target_net_update_frequency: 500 # sync target net with online net per this many epochs + + replay: + accumulate_reward_from_final: True + count_reward_lambda: 0.0 # 0 to disable + novel_object_reward_lambda: 0.0 # 0 to disable + discount_gamma_game_reward: 0.9 + discount_gamma_count_reward: 0.5 + discount_gamma_novel_object_reward: 0.5 + replay_memory_capacity: 500000 # adjust this depending on your RAM size + replay_memory_priority_fraction: 0.5 + update_per_k_game_steps: 5 + replay_batch_size: 64 + multi_step: 3 + replay_sample_history_length: 4 + replay_sample_update_from: 2 + + epsilon_greedy: + noisy_net: False # if this is true, then epsilon greedy is disabled + epsilon_anneal_episodes: 1000 # -1 if not annealing + epsilon_anneal_from: 0.3 + epsilon_anneal_to: 0.1 + +dagger: + action_space: "generation" # 'admissible' (candidates from text engine) or 'generation' (seq2seq-style generation) or 'exhaustive' (not working) + max_target_length: 20 # max token length for seq2seq generation + beam_width: 10 # 1 means greedy + generate_top_k: 5 + unstick_by_beam_search: False # use beam-search for failed actions, set True during evaluation + + training: + max_nb_steps_per_episode: 50 # terminate after this many steps + + fraction_assist: + fraction_assist_anneal_episodes: 50000 + fraction_assist_anneal_from: 1.0 + fraction_assist_anneal_to: 0.01 + + fraction_random: + fraction_random_anneal_episodes: 0 + fraction_random_anneal_from: 0.0 + fraction_random_anneal_to: 0.0 + + replay: + replay_memory_capacity: 500000 + update_per_k_game_steps: 5 + replay_batch_size: 64 + replay_sample_history_length: 4 + replay_sample_update_from: 2 + +vision_dagger: + model_type: "resnet" # 'resnet' (whole image features) or 'maskrcnn_whole' (whole image MaskRCNN feats) or 'maskrcnn' (top k MaskRCNN detection feats) or 'no_vision' (zero vision input) + resnet_fc_dim: 64 + maskrcnn_top_k_boxes: 10 # top k box features + use_exploration_frame_feats: False # append feats from initial exploration (memory intensive!) + sequence_aggregation_method: "average" # 'sum' or 'average' or 'rnn' diff --git a/skillopt/envs/alfworld/vendor/env_base.py b/skillopt/envs/alfworld/vendor/env_base.py new file mode 100644 index 0000000..00affa7 --- /dev/null +++ b/skillopt/envs/alfworld/vendor/env_base.py @@ -0,0 +1,84 @@ +# Vendored from SkillRL (Apache-2.0 License) +# Original: agent_system/environments/base.py +# Trimmed to only include what ALFWorld needs. + +from typing import List, Tuple, Dict, Any +import numpy as np +from collections import defaultdict + + +def to_numpy(data): + """Convert data to numpy array.""" + # Lazy-check for torch.Tensor to avoid hard dependency on torch + _torch_tensor = None + try: + import torch + _torch_tensor = torch.Tensor + except ImportError: + pass + + if _torch_tensor is not None and isinstance(data, _torch_tensor): + data = data.detach().cpu().numpy() + elif isinstance(data, np.ndarray): + pass + elif isinstance(data, (int, float, bool, Tuple, List)): + data = np.array(data) + else: + raise ValueError(f"Unsupported type: {type(data)})") + return data + + +class EnvironmentManagerBase: + """Base class for vectorized environment managers. + + Manages a set of parallel environments, handles action projection, + observation post-processing, and history tracking. + """ + + def __init__(self, envs, projection_f, config): + self.envs = envs + self.projection_f = projection_f + self.config = config + + def reset(self, kwargs) -> Dict[str, Any]: + obs, infos = self.envs.reset() + return {'text': None, 'image': obs, 'anchor': None}, infos + + def step(self, text_actions: List[str]): + actions, valids = self.projection_f(text_actions) + next_obs, rewards, dones, infos = self.envs.step(actions) + + next_observations = { + 'text': None, + 'image': next_obs, + 'anchor': None, + } + for i, info in enumerate(infos): + info['is_action_valid'] = to_numpy(valids[i]) + + rewards = to_numpy(rewards) + dones = to_numpy(dones) + return next_observations, rewards, dones, infos + + def close(self) -> None: + self.envs.close() + + def success_evaluator(self, *args, **kwargs) -> Dict[str, np.ndarray]: + total_infos = kwargs['total_infos'] + total_batch_list = kwargs['total_batch_list'] + batch_size = len(total_batch_list) + + success = defaultdict(list) + for bs in range(batch_size): + self._process_batch(bs, total_batch_list, total_infos, success) + assert len(success['success_rate']) == batch_size + return {key: np.array(value) for key, value in success.items()} + + def _process_batch(self, batch_idx, total_batch_list, total_infos, success): + for i in reversed(range(len(total_batch_list[batch_idx]))): + batch_item = total_batch_list[batch_idx][i] + if batch_item['active_masks']: + info = total_infos[batch_idx][i] + won_value = float(info['won']) + success['success_rate'].append(won_value) + return diff --git a/skillopt/envs/alfworld/vendor/env_manager.py b/skillopt/envs/alfworld/vendor/env_manager.py new file mode 100644 index 0000000..d937e4d --- /dev/null +++ b/skillopt/envs/alfworld/vendor/env_manager.py @@ -0,0 +1,139 @@ +# Vendored from SkillRL (Apache-2.0 License) +# Original: agent_system/environments/env_manager.py +# Trimmed to only include AlfWorldEnvironmentManager and its helpers. + +from typing import List, Dict, Any +from collections import defaultdict +import numpy as np + +from skillopt.envs.alfworld.vendor.env_base import EnvironmentManagerBase, to_numpy +from skillopt.envs.alfworld.vendor.alfworld_prompts import ( + ALFWORLD_TEMPLATE, + ALFWORLD_TEMPLATE_NO_HIS, + ALFWORLD_TEMPLATE_WITH_MEMORY, +) +from skillopt.envs.alfworld.vendor.memory import SimpleMemory + + +def parse_gamefile(infos): + gamefile = [] + for info in infos: + if 'extra.gamefile' in info: + gamefile.append(info['extra.gamefile']) + else: + gamefile.append(None) + return gamefile + + +def set_gamefile(infos, gamefile): + for i in range(len(infos)): + if 'extra.gamefile' in infos[i]: + infos[i]['extra.gamefile'] = gamefile[i] + else: + infos[i]['extra.gamefile'] = None + return infos + + +class AlfWorldEnvironmentManager(EnvironmentManagerBase): + """Manages parallel ALFWorld environments with observation templating.""" + + def __init__(self, envs, projection_f, config): + self.memory = SimpleMemory() + self.retrieval_memory = None + super().__init__(envs, projection_f, config) + + def reset(self, kwargs): + text_obs, image_obs, infos = self.envs.reset() + self.gamefile = parse_gamefile(infos) + self.memory.reset(batch_size=len(text_obs)) + self.tasks = [] + self.pre_text_obs = text_obs + self.extract_task(text_obs) + + full_text_obs = self.build_text_obs(text_obs, self.envs.get_admissible_commands, init=True) + return {'text': full_text_obs, 'image': image_obs, 'anchor': text_obs}, infos + + def step(self, text_actions: List[str]): + actions, valids = self.projection_f(text_actions, self.envs.get_admissible_commands) + text_obs, image_obs, rewards, dones, infos = self.envs.step(actions) + self.memory.store({'text_obs': self.pre_text_obs, 'action': actions}) + self.pre_text_obs = text_obs + + full_text_obs = self.build_text_obs(text_obs, self.envs.get_admissible_commands) + if infos[0].get("extra.gamefile") is None: + infos = set_gamefile(infos, self.gamefile) + + for i, info in enumerate(infos): + info['is_action_valid'] = to_numpy(valids[i]) + + next_observations = {'text': full_text_obs, 'image': image_obs, 'anchor': text_obs} + rewards = to_numpy(rewards) + dones = to_numpy(dones) + return next_observations, rewards, dones, infos + + def extract_task(self, text_obs: List[str]): + for obs in text_obs: + task_start = obs.find('Your task is to: ') + if task_start != -1: + self.tasks.append(obs[task_start + len('Your task is to: '):].strip()) + else: + raise ValueError("Task description not found in text observation.") + + def build_text_obs(self, text_obs: List[str], admissible_actions: List[List[str]], init: bool = False) -> List[str]: + postprocess_text_obs = [] + if not init and self.config.env.history_length > 0: + memory_contexts, valid_lens = self.memory.fetch( + self.config.env.history_length, + obs_key="text_obs", + action_key="action", + ) + + for i in range(len(text_obs)): + reformatted_admissible_actions = "\n ".join( + f"'{s}'" for s in admissible_actions[i] if s != 'help' + ) + + if init or self.config.env.history_length <= 0: + obs = ALFWORLD_TEMPLATE_NO_HIS.format( + current_observation=text_obs[i], + admissible_actions=reformatted_admissible_actions, + ) + else: + obs = ALFWORLD_TEMPLATE.format( + task_description=self.tasks[i], + step_count=len(self.memory[i]), + history_length=valid_lens[i], + action_history=memory_contexts[i], + current_step=len(self.memory[i]) + 1, + current_observation=text_obs[i], + admissible_actions=reformatted_admissible_actions, + ) + postprocess_text_obs.append(obs) + return postprocess_text_obs + + def _process_batch(self, batch_idx, total_batch_list, total_infos, success): + for i in reversed(range(len(total_batch_list[batch_idx]))): + batch_item = total_batch_list[batch_idx][i] + if batch_item['active_masks']: + info = total_infos[batch_idx][i] + won_value = float(info['won']) + success['success_rate'].append(won_value) + + gamefile = info.get("extra.gamefile") + if gamefile: + self._process_gamefile(gamefile, won_value, success) + return + + def _process_gamefile(self, gamefile, won_value, success): + tasks = [ + "pick_and_place", + "pick_two_obj_and_place", + "look_at_obj_in_light", + "pick_heat_then_place_in_recep", + "pick_cool_then_place_in_recep", + "pick_clean_then_place_in_recep", + ] + for task in tasks: + if task in gamefile: + success[f"{task}_success_rate"].append(won_value) + break diff --git a/skillopt/envs/alfworld/vendor/memory.py b/skillopt/envs/alfworld/vendor/memory.py new file mode 100644 index 0000000..045f306 --- /dev/null +++ b/skillopt/envs/alfworld/vendor/memory.py @@ -0,0 +1,87 @@ +# Vendored from SkillRL (Apache-2.0 License) +# Original: agent_system/memory/base.py + agent_system/memory/memory.py +# Merged into a single file for simplicity. + +from abc import ABC, abstractmethod +from typing import List, Dict, Any, Tuple + + +class BaseMemory(ABC): + """Base class for memory management.""" + + @abstractmethod + def __len__(self): + pass + + @abstractmethod + def __getitem__(self, idx: int): + pass + + @abstractmethod + def reset(self, batch_size: int): + pass + + @abstractmethod + def store(self, record: Dict[str, List[Any]]): + pass + + @abstractmethod + def fetch(self, step: int): + pass + + +class SimpleMemory(BaseMemory): + """Per-environment history buffer for storing observations and actions.""" + + def __init__(self): + self._data = None + self.keys = None + self.batch_size = 0 + + def __len__(self): + return len(self._data) + + def __getitem__(self, idx): + return self._data[idx] + + def reset(self, batch_size: int): + if self._data is not None: + self._data.clear() + self._data = [[] for _ in range(batch_size)] + self.batch_size = batch_size + self.keys = None + + def store(self, record: Dict[str, List[Any]]): + if self.keys is None: + self.keys = list(record.keys()) + assert self.keys == list(record.keys()) + + for env_idx in range(self.batch_size): + self._data[env_idx].append({k: record[k][env_idx] for k in self.keys}) + + def fetch( + self, + history_length: int, + obs_key: str = "text_obs", + action_key: str = "action", + ) -> Tuple[List[str], List[int]]: + memory_contexts, valid_lengths = [], [] + + for env_idx in range(self.batch_size): + recent = self._data[env_idx][-history_length:] + valid_len = len(recent) + start_idx = len(self._data[env_idx]) - valid_len + + lines = [] + for j, rec in enumerate(recent): + step_num = start_idx + j + 1 + act = rec[action_key] + obs = rec[obs_key] + lines.append( + f"[Observation {step_num}: '{obs}', Action {step_num}: '{act}']" + ) + + memory_contexts.append("\n".join(lines)) + valid_lengths.append(valid_len) + + return memory_contexts, valid_lengths diff --git a/skillopt/envs/base.py b/skillopt/envs/base.py new file mode 100644 index 0000000..243c2b7 --- /dev/null +++ b/skillopt/envs/base.py @@ -0,0 +1,329 @@ +"""ReflACT environment adapter — abstract interface. + +To connect ReflACT to a new environment (benchmark, simulator, etc.), +implement a subclass of :class:`EnvAdapter` with environment-specific +rollout and reflection logic. + +Example:: + + class MyBenchAdapter(EnvAdapter): + def build_train_env(self, batch_size, seed, **kw): + return MyEnvManager(split="train", n=batch_size, seed=seed) + + def build_eval_env(self, env_num, split, seed, **kw): + return MyEnvManager(split=split, n=env_num, seed=seed) + + def rollout(self, env_manager, skill_content, out_dir, **kw): + # Run episodes, return [{"id": ..., "hard": 0/1, "soft": 0.0-1.0, ...}] + ... + + def reflect(self, results, skill_content, out_dir, **kw): + # Analyze trajectories, return list of patch dicts + ... + + def get_task_types(self): + return ["task_a", "task_b"] +""" +from __future__ import annotations + +from abc import ABC, abstractmethod +import os +import random + +from skillopt.datasets.base import BaseDataLoader, BatchSpec +from skillopt.prompts import load_prompt + + +class EnvAdapter(ABC): + """Abstract adapter for connecting ReflACT to any environment. + + Subclasses must implement all abstract methods. The ReflACT trainer + calls these methods at the appropriate pipeline stages. + """ + + # ── Lifecycle hooks ──────────────────────────────────────────────────── + + def setup(self, cfg: dict) -> None: + """Called once by the trainer before the training loop begins. + + Override to perform one-time initialization that requires the full + config (e.g., data loading, split creation). Default is a no-op. + """ + self._cfg = dict(cfg) + + def get_dataloader(self) -> BaseDataLoader | None: + """Return the task dataloader used by this adapter, if any.""" + return None + + def requires_ray(self) -> bool: + """Return whether this adapter requires Ray runtime initialization.""" + return False + + def build_reference_text(self, item: dict) -> str: + """Return hidden reference material for reflection, if any.""" + return str(item.get("reference_text") or "").strip() + + def get_reference_metadata(self, item: dict) -> dict: + """Return structured metadata about hidden reference material.""" + reference_text = self.build_reference_text(item) + if not reference_text: + return {"fields": [], "preview": ""} + return { + "fields": ["reference_text"], + "preview": reference_text[:400], + } + + def attach_reference_context( + self, + results: list[dict], + items: list[dict] | None, + ) -> list[dict]: + """Attach environment-specific hidden reference text to result dicts.""" + if not results or not items: + return list(results) + + item_by_id = { + str(item.get("id")): item + for item in items + if isinstance(item, dict) and item.get("id") is not None + } + enriched: list[dict] = [] + for row in results: + merged = dict(row) + item = item_by_id.get(str(row.get("id"))) + if item: + reference_text = self.build_reference_text(item) + if reference_text: + merged["reference_text"] = reference_text + enriched.append(merged) + return enriched + + def select_representative_items( + self, + results: list[dict], + items: list[dict] | None, + *, + n_failures: int, + n_successes: int, + seed: int | None = None, + ) -> list[dict]: + """Select a small diverse subset of current-batch items by outcome.""" + if not items: + return [] + + item_by_id = { + str(item.get("id")): item + for item in items + if isinstance(item, dict) and item.get("id") is not None + } + failures = [ + (result, item_by_id[str(result.get("id"))]) + for result in results + if not result.get("hard") and str(result.get("id")) in item_by_id + ] + successes = [ + (result, item_by_id[str(result.get("id"))]) + for result in results + if result.get("hard") and str(result.get("id")) in item_by_id + ] + + rng = random.Random(seed) + + def _pick(pool: list[tuple[dict, dict]], quota: int) -> list[dict]: + if quota <= 0 or not pool: + return [] + shuffled = list(pool) + rng.shuffle(shuffled) + + picked_ids: set[str] = set() + picked: list[dict] = [] + seen_types: set[str] = set() + + for result, item in shuffled: + task_type = str(result.get("task_type") or item.get("task_type") or item.get("subtype") or "unknown") + item_id = str(item["id"]) + if task_type in seen_types or item_id in picked_ids: + continue + picked.append(item) + picked_ids.add(item_id) + seen_types.add(task_type) + if len(picked) >= quota: + return picked + + for _, item in shuffled: + item_id = str(item["id"]) + if item_id in picked_ids: + continue + picked.append(item) + picked_ids.add(item_id) + if len(picked) >= quota: + break + return picked + + selected = _pick(failures, n_failures) + selected_ids = {str(item["id"]) for item in selected} + selected.extend( + item for item in _pick(successes, n_successes) + if str(item["id"]) not in selected_ids + ) + return selected + + def build_env_from_batch(self, batch: BatchSpec, **kwargs): + """Build an environment manager or item list from a :class:`BatchSpec`. + + Default behavior preserves the legacy adapter API by routing training + batches through :meth:`build_train_env` and evaluation batches through + :meth:`build_eval_env`. + """ + if batch.phase == "train": + return self.build_train_env(batch_size=batch.batch_size, seed=batch.seed, **kwargs) + return self.build_eval_env( + env_num=batch.batch_size, + split=batch.split, + seed=batch.seed, + **kwargs, + ) + + @abstractmethod + def build_train_env(self, batch_size: int, seed: int, **kwargs): + """Build a training environment manager. + + Returns + ------- + object + An environment manager that can be passed to :meth:`rollout`. + """ + + @abstractmethod + def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs): + """Build an evaluation environment manager. + + Parameters + ---------- + env_num : int + Number of evaluation environments. + split : str + Dataset split (e.g. ``"valid_seen"``, ``"valid_unseen"``). + seed : int + Random seed for reproducibility. + + Returns + ------- + object + An environment manager that can be passed to :meth:`rollout`. + """ + + @abstractmethod + def rollout( + self, + env_manager, + skill_content: str, + out_dir: str, + **kwargs, + ) -> list[dict]: + """Run a batch of episodes using the current skill. + + Returns + ------- + list[dict] + Each dict conforms to :class:`~skillopt.types.RolloutResult`: + must have ``"id"`` (str), ``"hard"`` (0/1), ``"soft"`` + (float 0-1). May include env-specific fields. + """ + + def reflect( + self, + results: list[dict], + skill_content: str, + out_dir: str, + **kwargs, + ) -> list[dict | None]: + """Analyze rollout results and produce patches. + + Default implementation: delegate to the shared minibatch reflect + stage. Every built-in benchmark uses this unchanged — override only + if your environment needs custom reflection logic. + + Each returned dict conforms to :class:`~skillopt.types.RawPatch`: + ``"patch"`` (with ``"edits"`` list) + ``"source_type"`` + (``"failure"`` or ``"success"``); ``None`` entries are filtered out. + """ + from skillopt.gradient.reflect import run_minibatch_reflect + + return run_minibatch_reflect( + results=results, + skill_content=skill_content, + prediction_dir=kwargs.get( + "prediction_dir", os.path.join(out_dir, "predictions") + ), + patches_dir=kwargs.get( + "patches_dir", os.path.join(out_dir, "patches") + ), + workers=self.analyst_workers, + failure_only=self.failure_only, + minibatch_size=self.minibatch_size, + edit_budget=self.edit_budget, + random_seed=kwargs.get("random_seed"), + error_system=self.get_error_minibatch_prompt(), + success_system=self.get_success_minibatch_prompt(), + step_buffer_context=kwargs.get("step_buffer_context", ""), + meta_skill_context=kwargs.get("meta_skill_context", ""), + update_mode=getattr(self, "_cfg", {}).get("skill_update_mode", "patch"), + ) + + @abstractmethod + def get_task_types(self) -> list[str]: + """Return the list of task type names for this environment.""" + + # ── Prompt configuration (two-level priority) ──────────────────────── + # + # Priority: env-specific prompt file > generic default prompt file. + # + # Prompts are loaded from ``.md`` files via ``load_prompt(name, env)``: + # 1. ``skillopt/envs//prompts/.md`` (env-specific) + # 2. ``skillopt/prompts/.md`` (generic fallback) + # + # Subclasses can still override ``get_*_prompt()`` for full control. + + @property + def _env_name(self) -> str: + """Derive the env directory name from this adapter's module path.""" + # e.g. "skillopt.envs.searchqa.adapter" → "searchqa" + module = type(self).__module__ + parts = module.split(".") + if len(parts) >= 3 and parts[-3] == "envs": + return parts[-2] + return "" + + def _load_env_prompt(self, name: str) -> str | None: + """Load a prompt with env-specific override. Returns None if not found.""" + try: + return load_prompt(name, env=self._env_name) + except FileNotFoundError: + return None + + def get_error_minibatch_prompt(self) -> str | None: + update_mode = getattr(self, "_cfg", {}).get("skill_update_mode", "patch") + raw_mode = str(update_mode).strip().lower() + if raw_mode in {"full_rewrite", "full_rewrite_minibatch", "minibatch_full_rewrite", "skill_rewrite_minibatch"}: + prompt = self._load_env_prompt("analyst_error_full_rewrite") + if prompt is not None: + return prompt + if raw_mode in {"rewrite", "rewrite_from_suggestions", "suggestions", "rewrite_suggestions"}: + prompt = self._load_env_prompt("analyst_error_rewrite") + if prompt is not None: + return prompt + return self._load_env_prompt("analyst_error") + + def get_success_minibatch_prompt(self) -> str | None: + update_mode = getattr(self, "_cfg", {}).get("skill_update_mode", "patch") + raw_mode = str(update_mode).strip().lower() + if raw_mode in {"full_rewrite", "full_rewrite_minibatch", "minibatch_full_rewrite", "skill_rewrite_minibatch"}: + prompt = self._load_env_prompt("analyst_success_full_rewrite") + if prompt is not None: + return prompt + if raw_mode in {"rewrite", "rewrite_from_suggestions", "suggestions", "rewrite_suggestions"}: + prompt = self._load_env_prompt("analyst_success_rewrite") + if prompt is not None: + return prompt + return self._load_env_prompt("analyst_success") diff --git a/skillopt/envs/docvqa/__init__.py b/skillopt/envs/docvqa/__init__.py new file mode 100644 index 0000000..38c999d --- /dev/null +++ b/skillopt/envs/docvqa/__init__.py @@ -0,0 +1 @@ +"""DocVQA environment package for ReflACT.""" diff --git a/skillopt/envs/docvqa/adapter.py b/skillopt/envs/docvqa/adapter.py new file mode 100644 index 0000000..ddf1dbf --- /dev/null +++ b/skillopt/envs/docvqa/adapter.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +from skillopt.datasets.base import BatchSpec +from skillopt.envs.base import EnvAdapter +from skillopt.envs.docvqa.dataloader import DocVQADataLoader +from skillopt.envs.docvqa.rollout import run_batch + + +class DocVQAAdapter(EnvAdapter): + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "split_dir", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + max_turns: int = 1, + exec_timeout: int = 120, + workers: int = 16, + analyst_workers: int = 16, + failure_only: bool = False, + minibatch_size: int = 8, + edit_budget: int = 4, + seed: int = 42, + limit: int = 0, + image_detail: str = "auto", + max_completion_tokens: int = 16384, + ) -> None: + self.max_turns = max_turns + self.exec_timeout = exec_timeout + self.workers = workers + self.max_completion_tokens = int(max_completion_tokens) + self.analyst_workers = analyst_workers + self.failure_only = failure_only + self.minibatch_size = minibatch_size + self.edit_budget = edit_budget + self.image_detail = image_detail + self.dataloader = DocVQADataLoader( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + seed=seed, + limit=limit, + ) + + def setup(self, cfg: dict) -> None: + super().setup(cfg) + self.dataloader.setup(cfg) + + def get_dataloader(self): + return self.dataloader + + def build_env_from_batch(self, batch: BatchSpec, **kwargs): + return list(batch.payload or []) + + def build_train_env(self, batch_size: int, seed: int, **kwargs): + batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs): + batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def rollout(self, env_manager, skill_content: str, out_dir: str, **kwargs) -> list[dict]: + items: list[dict] = env_manager + return run_batch( + items=items, + out_root=out_dir, + skill_content=skill_content, + max_turns=self.max_turns, + exec_timeout=self.exec_timeout, + workers=self.workers, + image_detail=self.image_detail, + max_completion_tokens=self.max_completion_tokens, + diagnostic_mode=kwargs.get("diagnostic_mode", False), + diagnostic_instruction=kwargs.get("diagnostic_instruction", ""), + task_timeout=self.exec_timeout, + ) + + def get_task_types(self) -> list[str]: + seen: list[str] = [] + for item in self.dataloader.train_items + self.dataloader.val_items + self.dataloader.test_items: + task_type = str(item.get("task_type") or "docvqa") + if task_type not in seen: + seen.append(task_type) + return seen or ["docvqa"] diff --git a/skillopt/envs/docvqa/dataloader.py b/skillopt/envs/docvqa/dataloader.py new file mode 100644 index 0000000..212f0ef --- /dev/null +++ b/skillopt/envs/docvqa/dataloader.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import ast +import csv +from pathlib import Path + +from skillopt.datasets.base import SplitDataLoader + + +def _parse_answers(raw: str) -> list[str]: + text = str(raw or "").strip() + if not text: + return [] + try: + parsed = ast.literal_eval(text) + except Exception: + return [text] + if isinstance(parsed, list): + return [str(item).strip() for item in parsed if str(item).strip()] + return [str(parsed).strip()] + + +def _extract_document_path(question: str) -> tuple[str, str]: + marker = "document_path:" + if marker not in question: + return question.strip(), "" + main, tail = question.split(marker, 1) + return main.strip(), tail.strip() + + +def _normalize_row(row: dict[str, str]) -> dict: + question_text, document_path = _extract_document_path(str(row.get("question") or "")) + answers = _parse_answers(row.get("answer") or row.get("ground_truth") or "") + image_path = str(row.get("image_path") or document_path or "").strip() + task_type = str(row.get("topic") or row.get("category") or "docvqa").strip() or "docvqa" + return { + "id": str(row.get("questionId") or row.get("id") or "").strip(), + "question": question_text, + "answer": answers[0] if answers else "", + "answers": answers, + "task_type": task_type, + "subtask": task_type, + "image_paths": [image_path] if image_path else [], + "image_path": image_path, + "questionId": str(row.get("questionId") or "").strip(), + "docId": str(row.get("docId") or "").strip(), + "ucsf_document_id": str(row.get("ucsf_document_id") or "").strip(), + "ucsf_document_page_no": str(row.get("ucsf_document_page_no") or "").strip(), + "source_split": str(row.get("source_split") or "").strip(), + } + + +class DocVQADataLoader(SplitDataLoader): + def load_split_items(self, split_path: str) -> list[dict]: + path = Path(split_path) + csv_files = sorted(path.glob("*.csv")) + if not csv_files: + raise FileNotFoundError(f"No .csv file found in {split_path}") + with csv_files[0].open(encoding="utf-8", newline="") as f: + reader = csv.DictReader(f) + return [_normalize_row(row) for row in reader] diff --git a/skillopt/envs/docvqa/evaluator.py b/skillopt/envs/docvqa/evaluator.py new file mode 100644 index 0000000..85c09ef --- /dev/null +++ b/skillopt/envs/docvqa/evaluator.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import ast +import json +from collections.abc import Iterable +from typing import Any + +DEFAULT_ANLS_THRESHOLD = 0.5 + + +def _normalize_text(value: Any) -> str: + if value is None: + return "" + text = str(value).strip().lower() + return " ".join(text.split()) + + +def _levenshtein_distance(a: str, b: str) -> int: + if a == b: + return 0 + if not a: + return len(b) + if not b: + return len(a) + if len(a) > len(b): + a, b = b, a + previous = list(range(len(b) + 1)) + for i, char_a in enumerate(a, start=1): + current = [i] + for j, char_b in enumerate(b, start=1): + insert_cost = current[j - 1] + 1 + delete_cost = previous[j] + 1 + replace_cost = previous[j - 1] + (char_a != char_b) + current.append(min(insert_cost, delete_cost, replace_cost)) + previous = current + return previous[-1] + + +def _score_single_answer(predicted: Any, target: Any, threshold: float) -> float: + predicted_norm = _normalize_text(predicted) + target_norm = _normalize_text(target) + if not predicted_norm and not target_norm: + return 1.0 + if not predicted_norm or not target_norm: + return 0.0 + distance = _levenshtein_distance(predicted_norm, target_norm) + normalized_distance = distance / max(len(predicted_norm), len(target_norm)) + if normalized_distance >= threshold: + return 0.0 + return 1.0 - normalized_distance + + +def _extract_answer_strings(raw: Any) -> list[str]: + if raw is None: + return [""] + if isinstance(raw, str): + text = raw.strip() + if not text: + return [""] + parsed = None + if text[0] in "[{": + try: + parsed = json.loads(text) + except json.JSONDecodeError: + try: + parsed = ast.literal_eval(text) + except (ValueError, SyntaxError): + parsed = None + if parsed is None: + return [text] + return _extract_answer_strings(parsed) + if isinstance(raw, dict): + for key in ("answers", "ground_truth", "answer"): + if key in raw: + return _extract_answer_strings(raw[key]) + return [str(raw)] + if isinstance(raw, Iterable) and not isinstance(raw, (bytes, bytearray)): + answers: list[str] = [] + for item in raw: + if isinstance(item, dict): + for key in ("text", "answer", "value"): + if key in item: + answers.extend(_extract_answer_strings(item[key])) + break + else: + answers.append(str(item)) + continue + answers.append(str(item)) + return answers or [""] + return [str(raw)] + + +def extract_answer(text: str) -> str: + lower = text.lower() + start = lower.rfind("") + end = lower.rfind("") + if start != -1 and end != -1 and end > start: + return text[start + len(""):end].strip() + lines = [line.strip() for line in text.splitlines() if line.strip()] + return lines[-1] if lines else text.strip() + + +def evaluate(prediction_text: str, gold_answers: Any) -> dict: + answer = extract_answer(prediction_text) + answers = _extract_answer_strings(gold_answers) + score = 0.0 + for target in answers: + score = max(score, _score_single_answer(answer, target, DEFAULT_ANLS_THRESHOLD)) + return { + "anls": score, + "predicted_answer": answer, + "gold_answers": answers, + } diff --git a/skillopt/envs/docvqa/prompts/analyst_error.md b/skillopt/envs/docvqa/prompts/analyst_error.md new file mode 100644 index 0000000..9f6c367 --- /dev/null +++ b/skillopt/envs/docvqa/prompts/analyst_error.md @@ -0,0 +1,35 @@ +You are an expert failure-analysis agent for visual document question answering tasks. + +You will be given MULTIPLE failed DocVQA trajectories from a single minibatch and the current skill document. Each trajectory includes the model response and an evaluation result scored with ANLS against one or more acceptable answers. + +Your job is to identify the most important COMMON failure patterns across the batch and propose concise skill edits. + +## Failure Type Categories +- evidence_miss: the model overlooked the relevant visible region or line +- near_match_confusion: the model selected a nearby but incorrect text span +- normalization_error: the answer differed mainly in formatting, spacing, punctuation, or minor text normalization +- reading_error: the model misread the document content +- other: none of the above + +## Rules +- Focus on common, reusable reading and extraction behaviors. +- Do not hardcode image-specific answers. +- Prefer concise edits that improve evidence selection and exact span extraction. + +Respond ONLY with a valid JSON object (no markdown fences, no extra text): +{ + "batch_size": , + "failure_summary": [ + {"failure_type": "", "count": , "description": ""} + ], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +Only include edits that are needed. "edits" can be an empty list if no patch is warranted. diff --git a/skillopt/envs/docvqa/prompts/analyst_success.md b/skillopt/envs/docvqa/prompts/analyst_success.md new file mode 100644 index 0000000..2ce71d8 --- /dev/null +++ b/skillopt/envs/docvqa/prompts/analyst_success.md @@ -0,0 +1,24 @@ +You are an expert success-pattern analyst for visual document question answering tasks. + +You will be given MULTIPLE successful DocVQA trajectories from a single minibatch and the current skill document. Your job is to identify common visual reading and exact-answer extraction behaviors worth encoding in the skill. + +## Rules +- Focus on patterns shared across multiple successful trajectories. +- Reinforce reusable behaviors like locating the right region, copying exact spans, and preferring the shortest exact answer over paraphrase. +- Only propose patches for patterns not already captured by the current skill. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "success_patterns": ["", ""], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +"edits" may be empty if the skill already covers all observed patterns. diff --git a/skillopt/envs/docvqa/prompts/rollout_system.md b/skillopt/envs/docvqa/prompts/rollout_system.md new file mode 100644 index 0000000..e859c02 --- /dev/null +++ b/skillopt/envs/docvqa/prompts/rollout_system.md @@ -0,0 +1,12 @@ +You are an expert visual document question answering agent. + +{skill_section}You will receive a document image and a question about the document. +Read the visual evidence carefully and answer concisely. + +Rules: +- Ground the answer in the visible document content. +- Prefer exact spans, numbers, dates, and names from the document. +- Do not invent content that is not visible. +- If multiple near-matches exist, choose the one best supported by the document. + +Return the final answer inside .... diff --git a/skillopt/envs/docvqa/rollout.py b/skillopt/envs/docvqa/rollout.py new file mode 100644 index 0000000..6396163 --- /dev/null +++ b/skillopt/envs/docvqa/rollout.py @@ -0,0 +1,391 @@ +from __future__ import annotations + +import json +import os +import time +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait + +from skillopt.envs.docvqa.evaluator import evaluate +from skillopt.model import chat_target_messages, get_target_backend, is_target_exec_backend +from skillopt.model.codex_harness import prepare_workspace, render_skill_md, run_target_exec +from skillopt.prompts import load_prompt + + +def _build_system(skill_content: str) -> str: + if skill_content.strip(): + skill_section = f"## Skill\n{skill_content.strip()}\n\n" + else: + skill_section = "" + return load_prompt("rollout_system", env="docvqa").format(skill_section=skill_section) + + +def _image_to_data_uri(path: str) -> str: + import base64 + import mimetypes + + mime = mimetypes.guess_type(path)[0] or "image/png" + with open(path, "rb") as f: + encoded = base64.b64encode(f.read()).decode("ascii") + return f"data:{mime};base64,{encoded}" + + +def _build_messages( + item: dict, + skill_content: str, + image_detail: str, + *, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", +) -> tuple[list[dict], str, str]: + system = _build_system(skill_content) + user_text = item["question"] + "\n\nReturn the final answer inside ...." + if diagnostic_mode and diagnostic_instruction.strip(): + user_text += f"\n\n## Training Readout\n{diagnostic_instruction.strip()}" + image_url = {"url": _image_to_data_uri(item["image_path"])} + if image_detail and image_detail != "auto": + image_url["detail"] = image_detail + messages = [ + {"role": "system", "content": system}, + { + "role": "user", + "content": [ + {"type": "text", "text": user_text}, + {"type": "image_url", "image_url": image_url}, + ], + }, + ] + return messages, system, user_text + + +def _build_codex_skill(skill_content: str) -> str: + return render_skill_md( + skill_content, + description="Dynamic ReflACT skill for solving the current DocVQA document-image question.", + preamble=( + "Use this skill when answering the current DocVQA question.\n" + "Inspect the attached document image carefully and return the final answer inside ...." + ), + ) + + +def _run_codex_once( + *, + pred_dir: str, + item: dict, + skill_content: str, + model: str, + timeout: int, + image_detail: str, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + previous_response: str = "", +) -> tuple[str, str, str, str]: + _ = image_detail + _messages, _system, user_text = _build_messages( + item, + skill_content, + image_detail, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + ) + task_parts = [user_text] + image_abs = os.path.abspath(item["image_path"]) + task_parts.append( + "## Document Image\n" + "The document image is available in this workspace via `ATTACHMENTS.md`.\n" + f"Original image path: `{image_abs}`\n" + "Open or inspect that image before answering; do not answer from memory." + ) + if previous_response: + task_parts.append( + "## Previous Attempt\n" + f"{previous_response}\n\n" + "Review the same document image carefully and correct the answer if needed." + ) + task_text = "\n\n".join(task_parts) + skill_md = _build_codex_skill(skill_content) + work_dir = os.path.join(pred_dir, "codex_exec") + prepare_workspace( + work_dir=work_dir, + skill_md=skill_md, + task_text=task_text, + images=[item["image_path"]], + ) + prompt = ( + "Use the `skillopt-target` skill available in this workspace.\n" + "Read `task.md`, inspect the attached document image, and answer the DocVQA question.\n" + "Return the final answer inside ...." + ) + final_message, raw = run_target_exec( + work_dir=work_dir, + prompt=prompt, + model=model, + timeout=timeout, + images=[item["image_path"]], + ) + return final_message or raw, raw, skill_md, task_text + + +def process_one( + item: dict, + out_root: str, + skill_content: str, + *, + max_turns: int = 1, + exec_timeout: int = 120, + image_detail: str = "auto", + max_completion_tokens: int = 16384, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", +) -> dict: + item_id = str(item["id"]) + result = { + "id": item_id, + "question": item["question"], + "task_type": item.get("subtask") or item.get("task_type") or "docvqa", + "task_description": item["question"], + "hard": 0, + "soft": 0.0, + "predicted_answer": "", + "response": "", + "fail_reason": "", + "agent_ok": False, + "n_turns": 0, + "image_paths": item.get("image_paths", []), + "gold_answer": item.get("answers", []), + } + try: + response = "" + system_prompt = "" + user_text = "" + conversation: list[dict] = [] + if is_target_exec_backend(): + from skillopt.model import azure_openai as _llm + + conversation = [ + { + "role": "user", + "content": item["question"] + "\n\n" + f"[image] {os.path.basename(item['image_path'])}", + } + ] + for turn in range(max_turns): + response, _raw, system_prompt, user_text = _run_codex_once( + pred_dir=os.path.join(out_root, "predictions", item_id), + item=item, + skill_content=skill_content, + model=_llm.TARGET_DEPLOYMENT, + timeout=exec_timeout, + image_detail=image_detail, + diagnostic_mode=diagnostic_mode if turn == 0 else False, + diagnostic_instruction=diagnostic_instruction if turn == 0 else "", + previous_response=response if turn > 0 else "", + ) + conversation.append({"type": "message", "turn": turn + 1, "content": response}) + if "" in response.lower(): + break + else: + messages, system_prompt, user_text = _build_messages( + item, + skill_content, + image_detail, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + ) + conversation = [ + { + "role": "user", + "content": user_text + "\n\n" + f"[image] {os.path.basename(item['image_path'])}", + } + ] + for turn in range(max_turns): + if turn == 0: + resp_text, _ = chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=5, + stage="rollout", + timeout=exec_timeout, + ) + else: + refinement_messages = [ + messages[0], + messages[1], + {"role": "assistant", "content": response}, + {"role": "user", "content": "Review the same image carefully and answer again. Keep the final answer inside ...."}, + ] + resp_text, _ = chat_target_messages( + messages=refinement_messages, + max_completion_tokens=max_completion_tokens, + retries=5, + stage="rollout", + timeout=exec_timeout, + ) + response = resp_text + conversation.append({"type": "message", "turn": turn + 1, "content": resp_text}) + if "" in resp_text.lower(): + break + + result["response"] = response + result["agent_ok"] = True + result["n_turns"] = len(conversation) - 1 + + pred_dir = os.path.join(out_root, "predictions", item_id) + os.makedirs(pred_dir, exist_ok=True) + with open(os.path.join(pred_dir, "target_system_prompt.txt"), "w", encoding="utf-8") as f: + f.write(system_prompt) + with open(os.path.join(pred_dir, "target_user_prompt.txt"), "w", encoding="utf-8") as f: + f.write(user_text) + + eval_result = evaluate(response, item.get("answers", [])) + result["predicted_answer"] = eval_result["predicted_answer"] + result["hard"] = int(eval_result["anls"] >= 0.999) + result["soft"] = eval_result["anls"] + if result["soft"] <= 0.0: + result["fail_reason"] = f"predicted '{eval_result['predicted_answer']}' but expected one of {item.get('answers', [])}" + + eval_detail = ( + "[EVALUATION RESULT]\n" + f"Question: {item['question']}\n" + f"Predicted answer: {eval_result['predicted_answer']!r}\n" + f"Gold answers: {item.get('answers', [])!r}\n" + f"ANLS: {eval_result['anls']:.4f}" + ) + conversation.append({"role": "system", "content": eval_detail}) + with open(os.path.join(pred_dir, "conversation.json"), "w", encoding="utf-8") as f: + json.dump(conversation, f, ensure_ascii=False, indent=2) + except Exception as e: # noqa: BLE001 + result["fail_reason"] = f"error: {e}" + return result + + +def run_batch( + items: list[dict], + out_root: str, + skill_content: str, + *, + max_turns: int = 1, + exec_timeout: int = 120, + workers: int = 16, + image_detail: str = "auto", + max_completion_tokens: int = 16384, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + task_timeout: int = 600, +) -> list[dict]: + task_timeout = max(int(task_timeout), int(exec_timeout) + 60) + results_path = os.path.join(out_root, "results.jsonl") + os.makedirs(out_root, exist_ok=True) + + done_ids: set[str] = set() + existing: list[dict] = [] + if os.path.exists(results_path): + with open(results_path, encoding="utf-8") as f: + for line in f: + try: + row = json.loads(line) + except Exception: + continue + done_ids.add(str(row["id"])) + existing.append(row) + + pending = [item for item in items if str(item["id"]) not in done_ids] + if not pending: + return existing + + def _timeout_result(item: dict) -> dict: + return { + "id": str(item["id"]), + "question": item.get("question", ""), + "task_type": item.get("subtask") or item.get("task_type") or "docvqa", + "task_description": item.get("question", ""), + "hard": 0, + "soft": 0.0, + "predicted_answer": "", + "response": "", + "fail_reason": f"task-timeout-{task_timeout}s", + "agent_ok": False, + "n_turns": 0, + "image_paths": item.get("image_paths", []), + "gold_answer": item.get("answers", []), + "phase": "timeout", + } + + def _error_result(item: dict, exc: Exception) -> dict: + row = _timeout_result(item) + row["phase"] = "error" + row["fail_reason"] = f"unexpected: {type(exc).__name__}: {exc}" + return row + + started_at: dict[str, float] = {} + + def _run_one(item: dict) -> dict: + started_at[str(item["id"])] = time.time() + return process_one( + item, + out_root, + skill_content, + max_turns=max_turns, + exec_timeout=exec_timeout, + image_detail=image_detail, + max_completion_tokens=max_completion_tokens, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + ) + + total = len(existing) + len(pending) + completed = len(existing) + correct = sum(1 for r in existing if r.get("hard", 0)) + if existing: + print(f" [rollout] resuming: {completed}/{total} already done", flush=True) + + results = list(existing) + with open(results_path, "a", encoding="utf-8") as outf: + ex = ThreadPoolExecutor(max_workers=workers) + try: + futs = {ex.submit(_run_one, item): item for item in pending} + pending_futs = set(futs) + while pending_futs: + done, _ = wait(pending_futs, timeout=5, return_when=FIRST_COMPLETED) + now = time.time() + timed_out = [ + fut for fut in pending_futs - done + if str(futs[fut]["id"]) in started_at + and now - started_at[str(futs[fut]["id"])] >= task_timeout + ] + for fut in done: + pending_futs.remove(fut) + item = futs[fut] + try: + res = fut.result() + except Exception as exc: # noqa: BLE001 + res = _error_result(item, exc) + results.append(res) + completed += 1 + if res.get("hard", 0): + correct += 1 + acc = correct / completed if completed else 0 + print( + f" [rollout] {completed}/{total} " + f"(acc={acc:.3f}) id={res['id']} " + f"hard={res.get('hard', '?')}", + flush=True, + ) + outf.write(json.dumps(res, ensure_ascii=False) + "\n") + outf.flush() + for fut in timed_out: + pending_futs.remove(fut) + fut.cancel() + res = _timeout_result(futs[fut]) + results.append(res) + completed += 1 + acc = correct / completed if completed else 0 + print( + f" [rollout] {completed}/{total} " + f"(acc={acc:.3f}) id={res['id']} TIMEOUT", + flush=True, + ) + outf.write(json.dumps(res, ensure_ascii=False) + "\n") + outf.flush() + finally: + ex.shutdown(wait=False, cancel_futures=True) + return results diff --git a/skillopt/envs/docvqa/skills/initial.md b/skillopt/envs/docvqa/skills/initial.md new file mode 100644 index 0000000..806fbe6 --- /dev/null +++ b/skillopt/envs/docvqa/skills/initial.md @@ -0,0 +1,11 @@ +# DocVQA Skill + +## Visual Evidence Discipline +- Read the document carefully before answering. +- Prefer the smallest exact text span that answers the question. +- When several nearby strings look similar, choose the one whose surrounding labels or layout best match the question. + +## Exact Answer Discipline +- Copy names, numbers, and dates exactly from the document whenever possible. +- Prefer direct extraction over paraphrase. +- Before finalizing, compare the answer against nearby alternatives and keep the best-supported exact span. diff --git a/skillopt/envs/livemathematicianbench/__init__.py b/skillopt/envs/livemathematicianbench/__init__.py new file mode 100644 index 0000000..bcc2138 --- /dev/null +++ b/skillopt/envs/livemathematicianbench/__init__.py @@ -0,0 +1 @@ +"""LiveMathematicianBench environment package for ReflACT.""" diff --git a/skillopt/envs/livemathematicianbench/adapter.py b/skillopt/envs/livemathematicianbench/adapter.py new file mode 100644 index 0000000..ef96c86 --- /dev/null +++ b/skillopt/envs/livemathematicianbench/adapter.py @@ -0,0 +1,129 @@ +"""LiveMathematicianBench environment adapter for ReflACT.""" +from __future__ import annotations + +import json + +from skillopt.datasets.base import BatchSpec +from skillopt.envs.base import EnvAdapter +from skillopt.envs.livemathematicianbench.dataloader import LiveMathematicianBenchDataLoader +from skillopt.envs.livemathematicianbench.rollout import run_batch +from skillopt.model import get_target_backend + + +class LiveMathematicianBenchAdapter(EnvAdapter): + """LiveMathematicianBench adapter.""" + + def build_reference_text(self, item: dict) -> str: + parts: list[str] = [] + theorem = str(item.get("theorem") or "").strip() + sketch = str(item.get("sketch") or "").strip() + if theorem: + parts.append(f"## Reference Theorem\n{theorem}") + if sketch: + parts.append(f"## Reference Sketch\n{sketch}") + return "\n\n".join(parts) + + def get_reference_metadata(self, item: dict) -> dict: + fields: list[str] = [] + previews: list[str] = [] + theorem = str(item.get("theorem") or "").strip() + sketch = str(item.get("sketch") or "").strip() + if theorem: + fields.append("theorem") + previews.append(f"[theorem]\n{theorem[:220]}") + if sketch: + fields.append("sketch") + previews.append(f"[sketch]\n{sketch[:220]}") + return { + "fields": fields, + "preview": "\n\n".join(previews)[:500], + } + + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "ratio", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + max_turns: int = 1, + exec_timeout: int = 600, + workers: int = 64, + analyst_workers: int = 16, + failure_only: bool = False, + minibatch_size: int = 8, + edit_budget: int = 4, + seed: int = 42, + limit: int = 0, + shuffle_choices: bool = True, + use_theorem: bool = False, + use_sketch: bool = False, + max_completion_tokens: int = 16384, + ) -> None: + self.max_turns = max_turns + self.exec_timeout = exec_timeout + self.workers = workers + self.max_completion_tokens = int(max_completion_tokens) + self.analyst_workers = analyst_workers + self.failure_only = failure_only + self.minibatch_size = minibatch_size + self.edit_budget = edit_budget + self.use_theorem = use_theorem + self.use_sketch = use_sketch + self.dataloader = LiveMathematicianBenchDataLoader( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + seed=seed, + limit=limit, + shuffle_choices=shuffle_choices, + ) + + def setup(self, cfg: dict) -> None: + super().setup(cfg) + self.dataloader.setup(cfg) + + def get_dataloader(self): + return self.dataloader + + def build_env_from_batch(self, batch: BatchSpec, **kwargs): + return list(batch.payload or []) + + def build_train_env(self, batch_size: int, seed: int, **kwargs): + batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs): + batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def rollout( + self, + env_manager, + skill_content: str, + out_dir: str, + **kwargs, + ) -> list[dict]: + items: list[dict] = env_manager + return run_batch( + items=items, + out_root=out_dir, + skill_content=skill_content, + max_turns=self.max_turns, + exec_timeout=self.exec_timeout, + workers=self.workers, + max_completion_tokens=self.max_completion_tokens, + use_theorem=self.use_theorem, + use_sketch=self.use_sketch, + diagnostic_mode=kwargs.get("diagnostic_mode", False), + diagnostic_instruction=kwargs.get("diagnostic_instruction", ""), + diagnostic_trace_context_by_id=kwargs.get("diagnostic_trace_context_by_id"), + task_timeout=self.exec_timeout, + ) + + def get_task_types(self) -> list[str]: + return self.dataloader.get_task_types() diff --git a/skillopt/envs/livemathematicianbench/dataloader.py b/skillopt/envs/livemathematicianbench/dataloader.py new file mode 100644 index 0000000..3ab53f5 --- /dev/null +++ b/skillopt/envs/livemathematicianbench/dataloader.py @@ -0,0 +1,308 @@ +"""LiveMathematicianBench task dataloader.""" +from __future__ import annotations + +import glob +import hashlib +import json +import os +import random +from typing import Any + +from skillopt.datasets.base import BatchSpec, SplitDataLoader + + +# ── Raw data loading utilities (for preprocessing / standalone eval) ───── + +_CHOICE_LABELS = ["A", "B", "C", "D", "E", "F", "G"] + + +def _load_json(path: str) -> Any: + with open(path) as f: + return json.load(f) + + +def _iter_monthly_files(data_path: str) -> list[str]: + if not data_path: + return [] + if os.path.isfile(data_path): + return [data_path] + if os.path.isdir(data_path): + nested = glob.glob( + os.path.join(data_path, "**", "qa_*_final.json"), + recursive=True, + ) + flat = glob.glob(os.path.join(data_path, "qa_*_final.json")) + return sorted(set(nested + flat)) + return [] + + +def _coerce_choices(raw_choices: Any) -> list[dict]: + if isinstance(raw_choices, list): + choices: list[dict] = [] + for idx, item in enumerate(raw_choices): + if isinstance(item, dict): + label = str(item.get("label") or _CHOICE_LABELS[idx]).strip() + text = str(item.get("text") or item.get("content") or "").strip() + else: + label = _CHOICE_LABELS[idx] + text = str(item).strip() + if text: + choices.append({"label": label, "text": text}) + return choices + + if isinstance(raw_choices, dict): + labels = sorted(raw_choices.keys()) + return [ + {"label": str(label).strip(), "text": str(raw_choices[label]).strip()} + for label in labels + if str(raw_choices[label]).strip() + ] + + return [] + + +def _coerce_theorem_types(raw: Any) -> list[str]: + if isinstance(raw, list): + return [str(x).strip() for x in raw if str(x).strip()] + if raw is None: + return [] + text = str(raw).strip() + return [text] if text else [] + + +def _normalize_label(text: str) -> str: + return str(text).strip().upper().rstrip(".):") + + +def _normalize_item(item: dict, row_idx: int, source_path: str) -> dict: + mcq = item.get("mcq", {}) if isinstance(item.get("mcq"), dict) else {} + question = str(mcq.get("question") or item.get("question") or "").strip() + choices = _coerce_choices(mcq.get("choices") or item.get("choices") or []) + correct = mcq.get("correct_choice") or item.get("correct_choice") or {} + + if isinstance(correct, dict): + correct_label = _normalize_label(correct.get("label", "")) + correct_text = str(correct.get("text") or "").strip() + else: + correct_label = _normalize_label(correct) + correct_text = "" + + choice_by_label = { + _normalize_label(choice["label"]): choice["text"] + for choice in choices + } + if correct_label and not correct_text: + correct_text = choice_by_label.get(correct_label, "") + if correct_label and correct_text and correct_label not in choice_by_label: + choices.append({"label": correct_label, "text": correct_text}) + choices.sort(key=lambda choice: _CHOICE_LABELS.index(choice["label"]) if choice["label"] in _CHOICE_LABELS else len(_CHOICE_LABELS)) + choice_by_label[correct_label] = correct_text + + month = str(item.get("month") or "").strip() + item_no = item.get("no", row_idx + 1) + item_id = f"{month}:{item_no}" if month else str(item_no) + + return { + "id": item_id, + "month": month, + "no": item_no, + "paper_link": str(item.get("paper_link") or "").strip(), + "theorem": str(item.get("theorem") or "").strip(), + "sketch": str(item.get("sketch") or "").strip(), + "theorem_type": _coerce_theorem_types(item.get("theorem_type")), + "question": question, + "choices": choices, + "correct_choice": { + "label": correct_label, + "text": correct_text, + }, + "source_path": source_path, + } + + +def load_items(data_path: str) -> list[dict]: + """Load and normalise LiveMathematicianBench items from JSON files.""" + files = _iter_monthly_files(data_path) + if not files: + raise ValueError( + "LiveMathematicianBench requires data_path to be a qa_*_final.json file " + "or a directory containing monthly qa_*_final.json files." + ) + + items: list[dict] = [] + for path in files: + raw = _load_json(path) + if not isinstance(raw, list): + raise ValueError(f"Expected JSON array in {path}, got {type(raw).__name__}") + for row_idx, item in enumerate(raw): + norm = _normalize_item(item, row_idx=row_idx, source_path=path) + if norm["question"] and norm["choices"] and norm["correct_choice"]["label"]: + items.append(norm) + if not items: + raise ValueError(f"No valid LiveMathematicianBench items loaded from {data_path}") + return items + + +# ── Dataloader ─────────────────────────────────────────────────────────── + +class LiveMathematicianBenchDataLoader(SplitDataLoader): + """LiveMathematicianBench dataloader with per-seed choice shuffling.""" + + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "ratio", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + seed: int = 42, + limit: int = 0, + shuffle_choices: bool = True, + **kwargs, + ) -> None: + super().__init__( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + seed=seed, + limit=limit, + ) + self.shuffle_choices = shuffle_choices + self._task_types: list[str] = [] + + def load_raw_items(self, data_path: str) -> list[dict]: + return load_items(data_path) + + def setup(self, cfg: dict) -> None: + super().setup(cfg) + all_items = self.train_items + self.val_items + self.test_items + task_types: set[str] = set() + for item in all_items: + for name in item.get("theorem_type", []): + if name: + task_types.add(name) + self._task_types = sorted(task_types) + + def get_task_types(self) -> list[str]: + return list(self._task_types) + + # ── Choice shuffling ───────────────────────────────────────────────── + + @staticmethod + def _item_shuffle_seed(item_id: str, seed: int) -> int: + digest = hashlib.sha256(f"{seed}:{item_id}".encode("utf-8")).hexdigest() + return int(digest[:16], 16) + + def _shuffle_item_choices(self, item: dict, seed: int) -> dict: + if not self.shuffle_choices: + return { + **item, + "choices": [dict(c) for c in item["choices"]], + "correct_choice": dict(item["correct_choice"]), + } + + shuffled_choices = [dict(c) for c in item["choices"]] + rng = random.Random(self._item_shuffle_seed(str(item["id"]), seed)) + rng.shuffle(shuffled_choices) + + original_correct = _normalize_label(item["correct_choice"]["label"]) + remapped_choices: list[dict] = [] + new_correct_choice = dict(item["correct_choice"]) + + for idx, choice in enumerate(shuffled_choices): + new_label = _CHOICE_LABELS[idx] + old_label = _normalize_label(choice["label"]) + remapped_choices.append({"label": new_label, "text": choice["text"]}) + if old_label == original_correct: + new_correct_choice = {"label": new_label, "text": choice["text"]} + + transformed = dict(item) + transformed["choices"] = remapped_choices + transformed["correct_choice"] = new_correct_choice + return transformed + + def _materialize_batch(self, items: list[dict], seed: int) -> list[dict]: + return [self._shuffle_item_choices(item, seed) for item in items] + + # ── Batch construction (override for choice shuffling) ─────────────── + + def plan_train_epoch( + self, + *, + epoch: int, + steps_per_epoch: int, + accumulation: int, + batch_size: int, + seed: int, + **kwargs, + ) -> list[BatchSpec]: + """Build a shuffled epoch while preserving per-batch choice shuffling.""" + epoch_rng = random.Random(seed + epoch * 1000) + items = list(self.train_items) + epoch_rng.shuffle(items) + + total_batches = steps_per_epoch * accumulation + if total_batches <= 0: + return [] + + batches: list[BatchSpec] = [] + cursor = 0 + for batch_idx in range(total_batches): + batch_seed = seed + epoch * 1000 + batch_idx + 1 + batch_items = items[cursor: cursor + batch_size] + cursor += len(batch_items) + + if not batch_items and items: + refill_rng = random.Random(batch_seed) + batch_items = list(items) + refill_rng.shuffle(batch_items) + batch_items = batch_items[:batch_size] + + batch_items = self._materialize_batch(batch_items, batch_seed) + batches.append( + BatchSpec( + phase="train", + split="train", + seed=batch_seed, + batch_size=len(batch_items), + payload=batch_items, + ) + ) + + return batches + + def build_train_batch(self, batch_size: int, seed: int, **kwargs) -> BatchSpec: + rng = random.Random(seed) + items = list(self.train_items) + rng.shuffle(items) + items = self._materialize_batch(items[:batch_size], seed) + return BatchSpec( + phase="train", + split="train", + seed=seed, + batch_size=len(items), + payload=items, + ) + + def build_eval_batch( + self, + env_num: int, + split: str, + seed: int, + **kwargs, + ) -> BatchSpec: + items = self.get_split_items(split) + if env_num and env_num < len(items): + items = items[:env_num] + items = self._materialize_batch(items, seed) + return BatchSpec( + phase="eval", + split=split, + seed=seed, + batch_size=len(items), + payload=items, + ) diff --git a/skillopt/envs/livemathematicianbench/evaluator.py b/skillopt/envs/livemathematicianbench/evaluator.py new file mode 100644 index 0000000..d15db3e --- /dev/null +++ b/skillopt/envs/livemathematicianbench/evaluator.py @@ -0,0 +1,62 @@ +"""LiveMathematicianBench evaluation helpers.""" +from __future__ import annotations + +import re + + +def extract_answer(text: str) -> str: + matches = re.findall(r"(.*?)", text, re.DOTALL | re.IGNORECASE) + if matches: + return matches[-1].strip() + lines = [ln.strip() for ln in text.strip().splitlines() if ln.strip()] + if lines: + return lines[-1] + return text.strip() + + +def normalize_label(text: str) -> str: + return str(text).strip().upper().rstrip(".):") + + +def parse_choice_label(prediction_text: str, choices: list[dict]) -> str: + answer = extract_answer(prediction_text) + label = normalize_label(answer) + valid_labels = {normalize_label(choice.get("label", "")) for choice in choices} + if label in valid_labels: + return label + + answer_lower = answer.lower() + for choice in choices: + choice_label = normalize_label(choice.get("label", "")) + choice_text = str(choice.get("text", "")).strip() + if choice_text and choice_text.lower() == answer_lower: + return choice_label + + first_token = normalize_label(answer.split()[0]) if answer.split() else "" + if first_token in valid_labels: + return first_token + return label + + +def evaluate(prediction_text: str, correct_choice: dict, choices: list[dict]) -> dict: + predicted_label = parse_choice_label(prediction_text, choices) + correct_label = normalize_label(correct_choice.get("label", "")) + predicted_text = "" + correct_text = str(correct_choice.get("text", "")).strip() + + for choice in choices: + if normalize_label(choice.get("label", "")) == predicted_label: + predicted_text = str(choice.get("text", "")).strip() + break + + is_correct = float(predicted_label == correct_label) + return { + "em": is_correct, + "f1": is_correct, + "sub_em": is_correct, + "predicted_answer": predicted_label or extract_answer(prediction_text), + "predicted_label": predicted_label, + "predicted_text": predicted_text, + "correct_label": correct_label, + "correct_text": correct_text, + } diff --git a/skillopt/envs/livemathematicianbench/prompts/analyst_error.md b/skillopt/envs/livemathematicianbench/prompts/analyst_error.md new file mode 100644 index 0000000..dac1d04 --- /dev/null +++ b/skillopt/envs/livemathematicianbench/prompts/analyst_error.md @@ -0,0 +1,37 @@ +You are an expert failure-analysis agent for theorem-grounded mathematical multiple-choice questions. + +You will be given MULTIPLE failed trajectories from a single minibatch and the current skill document. +Each trajectory includes the target's response and an evaluation result showing the predicted option +versus the correct option. + +Your job is to identify COMMON reasoning failures across the batch and propose concise skill edits. + +## Failure Type Categories +- **quantifier_miss**: the agent missed exact quantifiers, scope, or existence/uniqueness conditions +- **strength_mismatch**: the agent preferred a weaker or stronger statement than what was proved +- **condition_miss**: the agent ignored hypotheses, equality cases, or domain restrictions +- **option_confusion**: the agent confused similar answer choices or failed to compare them exactly +- **other**: none of the above + +## Rules +1. Focus on patterns that recur across the minibatch. +2. Prefer edits that improve exact choice discrimination, not theorem-specific memorization. +3. Do not hardcode paper-specific content. +4. Only patch gaps not already covered by the skill. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "failure_summary": [ + {"failure_type": "", "count": , "description": ""} + ], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} diff --git a/skillopt/envs/livemathematicianbench/prompts/analyst_success.md b/skillopt/envs/livemathematicianbench/prompts/analyst_success.md new file mode 100644 index 0000000..7ff47d1 --- /dev/null +++ b/skillopt/envs/livemathematicianbench/prompts/analyst_success.md @@ -0,0 +1,25 @@ +You are an expert success-pattern analyst for theorem-grounded mathematical multiple-choice questions. + +You will be given MULTIPLE successful trajectories from a minibatch and the current skill document. +Identify generalizable behavior patterns that are genuinely helping the agent choose the exact correct option. + +## Rules +- Focus on broadly useful reasoning behaviors. +- Prefer patterns about exact comparison of options, quantifiers, and equality conditions. +- Do not add theorem-specific facts. +- "edits" may be empty if the skill already captures the useful patterns. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "success_patterns": ["", ""], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} diff --git a/skillopt/envs/livemathematicianbench/prompts/rollout_system.md b/skillopt/envs/livemathematicianbench/prompts/rollout_system.md new file mode 100644 index 0000000..607153d --- /dev/null +++ b/skillopt/envs/livemathematicianbench/prompts/rollout_system.md @@ -0,0 +1,12 @@ +You are an expert mathematical reasoning agent solving multiple-choice questions. + +{skill_section}## Task Format +You will receive one mathematics multiple-choice question and its answer choices. +Reason carefully about quantifiers, hypotheses, extremal wording, and exact equality conditions. + +## Answer Format +Think step by step, then provide your final answer inside ... tags. +Inside the tags, output only the single choice label, such as A or C. + +Example: +B diff --git a/skillopt/envs/livemathematicianbench/reflect.py b/skillopt/envs/livemathematicianbench/reflect.py new file mode 100644 index 0000000..b738481 --- /dev/null +++ b/skillopt/envs/livemathematicianbench/reflect.py @@ -0,0 +1,4 @@ +"""LiveMathematicianBench Reflect stage. + +Prompts are now loaded from .md files by the base adapter. +""" diff --git a/skillopt/envs/livemathematicianbench/rollout.py b/skillopt/envs/livemathematicianbench/rollout.py new file mode 100644 index 0000000..01de404 --- /dev/null +++ b/skillopt/envs/livemathematicianbench/rollout.py @@ -0,0 +1,434 @@ +"""LiveMathematicianBench rollout — theorem-grounded math MCQ agent.""" +from __future__ import annotations + +import json +import os +import time +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait + +from skillopt.envs.livemathematicianbench.evaluator import evaluate +from skillopt.model import chat_target, get_target_backend, is_target_exec_backend +from skillopt.model.codex_harness import prepare_workspace, render_skill_md, run_target_exec +from skillopt.prompts import load_prompt + +def _build_system(skill_content: str) -> str: + if skill_content.strip(): + skill_section = f"## Skill\n{skill_content.strip()}\n\n" + else: + skill_section = "" + return load_prompt("rollout_system", env="livemathematicianbench").format(skill_section=skill_section) + + +def _format_choices(choices: list[dict]) -> str: + return "\n".join( + f"{choice['label']}. {choice['text']}" + for choice in choices + ) + + +def _build_user( + item: dict, + *, + use_theorem: bool = False, + use_sketch: bool = False, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", +) -> str: + parts = [f"## Question\n{item['question']}", f"## Choices\n{_format_choices(item['choices'])}"] + if use_theorem and item.get("theorem"): + parts.append(f"## Theorem\n{item['theorem']}") + if use_sketch and item.get("sketch"): + parts.append(f"## Proof Sketch\n{item['sketch']}") + if diagnostic_trace_context.strip(): + parts.append( + "## Previous Codex Trace Snapshot\n" + "This is a partial transcript from an earlier attempt. Use it as your current reasoning context.\n\n" + f"{diagnostic_trace_context.strip()}" + ) + if diagnostic_mode and diagnostic_instruction.strip(): + parts.append(f"## Training Readout\n{diagnostic_instruction.strip()}") + return "\n\n".join(parts) + + +def _build_codex_skill(skill_content: str) -> str: + return render_skill_md( + skill_content, + description="Dynamic ReflACT skill for solving the current LiveMathematicianBench multiple-choice question.", + preamble=( + "Use this skill when solving the current math multiple-choice question.\n" + "Inspect the option wording carefully and output only the final choice label inside ...." + ), + ) + +def _run_codex_once( + *, + pred_dir: str, + skill_content: str, + item: dict, + model: str, + timeout: int, + use_theorem: bool = False, + use_sketch: bool = False, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", + previous_response: str = "", +) -> tuple[str, str, str, str]: + user = _build_user( + item, + use_theorem=use_theorem, + use_sketch=use_sketch, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + task_parts = [user] + if previous_response: + task_parts.append( + "## Previous Attempt\n" + f"{previous_response}\n\n" + "Re-evaluate the exact option wording. If needed, correct it." + ) + task_text = "\n\n".join(task_parts) + skill_md = _build_codex_skill(skill_content) + work_dir = os.path.join(pred_dir, "codex_exec") + prepare_workspace(work_dir=work_dir, skill_md=skill_md, task_text=task_text) + prompt = ( + "Use the `skillopt-target` skill available in this workspace.\n" + "Read `task.md` and solve the multiple-choice problem.\n" + "Output only the final choice label inside ...." + ) + final_message, raw = run_target_exec( + work_dir=work_dir, + prompt=prompt, + model=model, + timeout=timeout, + ) + return final_message or raw, raw, skill_md, task_text + + +def process_one( + item: dict, + out_root: str, + skill_content: str, + *, + max_turns: int = 1, + use_theorem: bool = False, + use_sketch: bool = False, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", + exec_timeout: int | None = 300, + max_completion_tokens: int = 16384, +) -> dict: + item_id = str(item["id"]) + result = { + "id": item_id, + "question": item["question"], + "task_type": item.get("theorem_type", ["math_mcq"])[0] if item.get("theorem_type") else "math_mcq", + "hard": 0, + "soft": 0.0, + "predicted_answer": "", + "predicted_label": "", + "predicted_text": "", + "correct_label": item["correct_choice"]["label"], + "correct_text": item["correct_choice"]["text"], + "response": "", + "fail_reason": "", + "agent_ok": False, + "n_turns": 0, + } + + try: + pred_dir = os.path.join(out_root, "predictions", item_id) + os.makedirs(pred_dir, exist_ok=True) + llm_timeout = int(exec_timeout) if exec_timeout and int(exec_timeout) > 0 else None + + if is_target_exec_backend(): + from skillopt.model import azure_openai as _llm + + conversation: list[dict] = [] + response = "" + system = "" + user = "" + for turn in range(max_turns): + response, raw, system, user = _run_codex_once( + pred_dir=pred_dir, + skill_content=skill_content, + item=item, + model=_llm.TARGET_DEPLOYMENT, + timeout=llm_timeout, + use_theorem=use_theorem, + use_sketch=use_sketch, + diagnostic_mode=diagnostic_mode if turn == 0 else False, + diagnostic_instruction=diagnostic_instruction if turn == 0 else "", + diagnostic_trace_context=diagnostic_trace_context if turn == 0 else "", + previous_response=response if turn > 0 else "", + ) + conversation.append({"type": "message", "turn": turn + 1, "content": response}) + if "" in response.lower(): + break + + result["response"] = response + result["agent_ok"] = True + result["n_turns"] = len(conversation) + + with open(os.path.join(pred_dir, "target_system_prompt.txt"), "w", encoding="utf-8") as f: + f.write(system) + with open(os.path.join(pred_dir, "target_user_prompt.txt"), "w", encoding="utf-8") as f: + f.write(user) + + eval_result = evaluate(response, item["correct_choice"], item["choices"]) + result["hard"] = int(eval_result["em"]) + result["soft"] = eval_result["f1"] + result["predicted_answer"] = eval_result["predicted_answer"] + result["predicted_label"] = eval_result["predicted_label"] + result["predicted_text"] = eval_result["predicted_text"] + if not result["hard"]: + result["fail_reason"] = ( + f"MCQ=0: predicted '{eval_result['predicted_label'] or eval_result['predicted_answer']}' " + f"but expected '{eval_result['correct_label']}'" + ) + eval_detail = ( + f"[EVALUATION RESULT]\n" + f"Question: {item['question']}\n" + f"Predicted label: {eval_result['predicted_label']!r}\n" + f"Predicted text: {eval_result['predicted_text']!r}\n" + f"Correct label: {eval_result['correct_label']!r}\n" + f"Correct text: {eval_result['correct_text']!r}\n" + f"Exact Match: {eval_result['em']}" + ) + conversation.append({"role": "system", "content": eval_detail}) + with open(os.path.join(pred_dir, "conversation.json"), "w") as f: + json.dump(conversation, f, ensure_ascii=False, indent=2) + return result + + system = _build_system(skill_content) + user = _build_user( + item, + use_theorem=use_theorem, + use_sketch=use_sketch, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + conversation: list[dict] = [] + response = "" + + for turn in range(max_turns): + if turn == 0: + resp_text, _ = chat_target( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=5, + stage="rollout", + timeout=llm_timeout, + ) + else: + refinement = ( + f"Your previous answer was:\n{response}\n\n" + "Re-evaluate the exact option wording. If needed, correct it. " + "Output only the final choice label inside ...." + ) + resp_text, _ = chat_target( + system=system, + user=refinement, + max_completion_tokens=max_completion_tokens, + retries=5, + stage="rollout", + timeout=llm_timeout, + ) + response = resp_text + conversation.append({"type": "message", "turn": turn + 1, "content": resp_text}) + if "" in resp_text.lower(): + break + + result["response"] = response + result["agent_ok"] = True + result["n_turns"] = len(conversation) + + with open(os.path.join(pred_dir, "target_system_prompt.txt"), "w", encoding="utf-8") as f: + f.write(system) + with open(os.path.join(pred_dir, "target_user_prompt.txt"), "w", encoding="utf-8") as f: + f.write(user) + + eval_result = evaluate(response, item["correct_choice"], item["choices"]) + result["hard"] = int(eval_result["em"]) + result["soft"] = eval_result["f1"] + result["predicted_answer"] = eval_result["predicted_answer"] + result["predicted_label"] = eval_result["predicted_label"] + result["predicted_text"] = eval_result["predicted_text"] + + if not result["hard"]: + result["fail_reason"] = ( + f"MCQ=0: predicted '{eval_result['predicted_label'] or eval_result['predicted_answer']}' " + f"but expected '{eval_result['correct_label']}'" + ) + + eval_detail = ( + f"[EVALUATION RESULT]\n" + f"Question: {item['question']}\n" + f"Predicted label: {eval_result['predicted_label']!r}\n" + f"Predicted text: {eval_result['predicted_text']!r}\n" + f"Correct label: {eval_result['correct_label']!r}\n" + f"Correct text: {eval_result['correct_text']!r}\n" + f"Exact Match: {eval_result['em']}" + ) + conversation.append({"role": "system", "content": eval_detail}) + + with open(os.path.join(pred_dir, "conversation.json"), "w") as f: + json.dump(conversation, f, ensure_ascii=False, indent=2) + + except Exception as e: # noqa: BLE001 + result["fail_reason"] = f"error: {e}" + + return result + + +def run_batch( + items: list[dict], + out_root: str, + skill_content: str, + *, + max_turns: int = 1, + exec_timeout: int | None = 300, + workers: int = 64, + max_completion_tokens: int = 16384, + use_theorem: bool = False, + use_sketch: bool = False, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context_by_id: dict[str, str] | None = None, + task_timeout: int | None = 600, +) -> list[dict]: + exec_timeout_value = int(exec_timeout) if exec_timeout and int(exec_timeout) > 0 else 0 + task_timeout_value = int(task_timeout) if task_timeout and int(task_timeout) > 0 else 0 + if exec_timeout_value <= 0 or task_timeout_value <= 0: + task_timeout = None + else: + task_timeout = max(task_timeout_value, exec_timeout_value + 60) + results_path = os.path.join(out_root, "results.jsonl") + os.makedirs(out_root, exist_ok=True) + + done_ids: set[str] = set() + existing: list[dict] = [] + if os.path.exists(results_path): + with open(results_path) as f: + for line in f: + try: + r = json.loads(line) + done_ids.add(str(r["id"])) + existing.append(r) + except Exception: + pass + + pending = [it for it in items if str(it["id"]) not in done_ids] + if not pending: + return existing + + total = len(existing) + len(pending) + completed = len(existing) + correct_count = sum(1 for r in existing if r.get("hard", 0)) + if existing: + print(f" [rollout] resuming: {completed}/{total} already done", flush=True) + + results = list(existing) + + started_at: dict[str, float] = {} + + def _run_one(it: dict) -> dict: + started_at[str(it["id"])] = time.time() + return process_one( + it, + out_root, + skill_content, + max_turns=max_turns, + exec_timeout=exec_timeout, + max_completion_tokens=max_completion_tokens, + use_theorem=use_theorem, + use_sketch=use_sketch, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=(diagnostic_trace_context_by_id or {}).get(str(it["id"]), ""), + ) + + def _timeout_result(it: dict) -> dict: + correct = it.get("correct_choice") or {} + return { + "id": str(it["id"]), + "question": it.get("question", ""), + "task_type": it.get("theorem_type", ["math_mcq"])[0] if it.get("theorem_type") else "math_mcq", + "hard": 0, + "soft": 0.0, + "predicted_answer": "", + "predicted_label": "", + "predicted_text": "", + "correct_label": correct.get("label", ""), + "correct_text": correct.get("text", ""), + "response": "", + "fail_reason": f"task-timeout-{task_timeout}s", + "agent_ok": False, + "n_turns": 0, + } + + def _error_result(it: dict, exc: Exception) -> dict: + res = _timeout_result(it) + res["fail_reason"] = f"error: {type(exc).__name__}: {exc}" + return res + + with open(results_path, "a") as outf: + ex = ThreadPoolExecutor(max_workers=workers) + try: + futs = { + ex.submit(_run_one, it): it + for it in pending + } + pending_futs = set(futs) + while pending_futs: + done, _ = wait(pending_futs, timeout=5, return_when=FIRST_COMPLETED) + now = time.time() + timed_out = [ + fut for fut in pending_futs - done + if task_timeout is not None + if str(futs[fut]["id"]) in started_at + and now - started_at[str(futs[fut]["id"])] >= task_timeout + ] + for fut in done: + pending_futs.remove(fut) + item = futs[fut] + try: + res = fut.result() + except Exception as e: # noqa: BLE001 + res = _error_result(item, e) + results.append(res) + completed += 1 + if res.get("hard", 0): + correct_count += 1 + acc = correct_count / completed if completed else 0 + print( + f" [rollout] {completed}/{total} " + f"(acc={acc:.3f}) id={res['id']} " + f"hard={res.get('hard', '?')}", + flush=True, + ) + outf.write(json.dumps(res, ensure_ascii=False) + "\n") + outf.flush() + for fut in timed_out: + pending_futs.remove(fut) + res = _timeout_result(futs[fut]) + results.append(res) + completed += 1 + acc = correct_count / completed if completed else 0 + print( + f" [rollout] {completed}/{total} " + f"(acc={acc:.3f}) id={res['id']} TIMEOUT", + flush=True, + ) + outf.write(json.dumps(res, ensure_ascii=False) + "\n") + outf.flush() + finally: + ex.shutdown(wait=False, cancel_futures=True) + + return results diff --git a/skillopt/envs/livemathematicianbench/skills/initial.md b/skillopt/envs/livemathematicianbench/skills/initial.md new file mode 100644 index 0000000..d34f603 --- /dev/null +++ b/skillopt/envs/livemathematicianbench/skills/initial.md @@ -0,0 +1,16 @@ +# Live Mathematical MCQ Heuristics + +## Option Comparison +- Compare all options before committing. The correct choice is often the strongest statement justified by the question, while nearby distractors are weaker, overstrong, or miss an equality case. +- Track exact quantifiers such as "there exists", "for every", "if and only if", and "exactly when". + +## Theorem-Level Precision +- Check whether an option weakens the conclusion by dropping a characterization, equality clause, or full equivalence. +- Check whether an option overstates the theorem by upgrading regularity, removing scale restrictions, or changing an existential statement into a universal one. + +## Hypotheses +- Verify the hypotheses and domain carefully. Distractors often keep the theorem shape but alter the required assumptions. +- Pay close attention to equality cases, extremal conditions, and whether a result applies to the full family or only a restricted subfamily. + +## Final Answer +- Output the final answer as the single option label only. diff --git a/skillopt/envs/officeqa/__init__.py b/skillopt/envs/officeqa/__init__.py new file mode 100644 index 0000000..5316aaf --- /dev/null +++ b/skillopt/envs/officeqa/__init__.py @@ -0,0 +1 @@ +"""OfficeQA environment package for ReflACT.""" diff --git a/skillopt/envs/officeqa/adapter.py b/skillopt/envs/officeqa/adapter.py new file mode 100644 index 0000000..63419d4 --- /dev/null +++ b/skillopt/envs/officeqa/adapter.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import os + +from skillopt.datasets.base import BatchSpec +from skillopt.envs.base import EnvAdapter +from skillopt.envs.officeqa.dataloader import OfficeQADataLoader +from skillopt.envs.officeqa.rollout import run_batch + + +class OfficeQAAdapter(EnvAdapter): + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "split_dir", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + workers: int = 8, + analyst_workers: int = 8, + failure_only: bool = False, + minibatch_size: int = 8, + edit_budget: int = 4, + seed: int = 42, + limit: int = 0, + max_tool_turns: int = 12, + max_completion_tokens: int = 16384, + search_mode: str = "offline", + max_queries_per_turn: int = 4, + search_api_url: str = os.environ.get("OFFICEQA_SEARCH_API_URL", "http://localhost:8080/search_tool/search"), + search_auth_env: str = "OFFICEQA_CUSTOM_SEARCH_AUTH", + search_provider: str = "duckduckgo", + search_max_num_results: int = 4, + search_timeout_seconds: int = 20, + use_local_tools: bool = True, + data_dirs: list[str] | str | None = None, + docs_dirs: list[str] | str | None = None, ) -> None: + self.workers = workers + self.analyst_workers = analyst_workers + self.failure_only = failure_only + self.minibatch_size = minibatch_size + self.edit_budget = edit_budget + self.max_tool_turns = max_tool_turns + self.max_completion_tokens = int(max_completion_tokens) + self.search_mode = str(search_mode or "offline") + self.max_queries_per_turn = int(max_queries_per_turn) + self.search_api_url = str(search_api_url or "").strip() + self.search_auth_env = str(search_auth_env or "OFFICEQA_CUSTOM_SEARCH_AUTH").strip() + self.search_provider = str(search_provider or "duckduckgo").strip() + self.search_max_num_results = int(search_max_num_results) + self.search_timeout_seconds = int(search_timeout_seconds) + self.use_local_tools = bool(use_local_tools) + self.data_dirs = data_dirs if data_dirs is not None else docs_dirs + self.dataloader = OfficeQADataLoader( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + seed=seed, + limit=limit, + ) + + def setup(self, cfg: dict) -> None: + super().setup(cfg) + self.dataloader.setup(cfg) + + def get_dataloader(self): + return self.dataloader + + def build_env_from_batch(self, batch: BatchSpec, **kwargs): + return list(batch.payload or []) + + def build_train_env(self, batch_size: int, seed: int, **kwargs): + batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs): + batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def rollout(self, env_manager, skill_content: str, out_dir: str, **kwargs) -> list[dict]: + items: list[dict] = env_manager + return run_batch( + items=items, + out_root=out_dir, + skill_content=skill_content, + workers=self.workers, + max_tool_turns=self.max_tool_turns, + max_completion_tokens=self.max_completion_tokens, + search_mode=self.search_mode, + max_queries_per_turn=self.max_queries_per_turn, + search_api_url=self.search_api_url, + search_auth_env=self.search_auth_env, + search_provider=self.search_provider, + search_max_num_results=self.search_max_num_results, + search_timeout_seconds=self.search_timeout_seconds, + use_local_tools=self.use_local_tools, + data_dirs=self.data_dirs, + diagnostic_mode=kwargs.get("diagnostic_mode", False), + diagnostic_instruction=kwargs.get("diagnostic_instruction", ""), + ) + + def get_task_types(self) -> list[str]: + seen: list[str] = [] + for item in self.dataloader.train_items + self.dataloader.val_items + self.dataloader.test_items: + task_type = str(item.get("task_type") or "officeqa") + if task_type not in seen: + seen.append(task_type) + return seen or ["officeqa"] diff --git a/skillopt/envs/officeqa/dataloader.py b/skillopt/envs/officeqa/dataloader.py new file mode 100644 index 0000000..a9c22b4 --- /dev/null +++ b/skillopt/envs/officeqa/dataloader.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import csv +import json +import os +from pathlib import Path + +from skillopt.datasets.base import SplitDataLoader + + +def _parse_list_field(value: str | list[str] | None) -> list[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(item).strip() for item in value if str(item).strip()] + text = str(value).strip() + if not text: + return [] + try: + loaded = json.loads(text) + except json.JSONDecodeError: + loaded = None + if isinstance(loaded, list): + return [str(item).strip() for item in loaded if str(item).strip()] + if "\n" in text: + return [part.strip() for part in text.splitlines() if part.strip()] + if "," in text and not text.lower().endswith(".txt"): + return [part.strip() for part in text.split(",") if part.strip()] + return [text] + + +def _normalize_row(row: dict[str, str]) -> dict: + item_id = str(row.get("uid") or row.get("id") or "").strip() + question = str(row.get("question") or "").strip() + ground_truth = str(row.get("ground_truth") or row.get("answer") or "").strip() + task_type = str(row.get("category") or row.get("difficulty") or "officeqa").strip() or "officeqa" + source_files = _parse_list_field(row.get("source_files")) + source_docs = _parse_list_field(row.get("source_docs")) + split = str(row.get("split") or "").strip() + return { + "id": item_id, + "uid": item_id, + "question": question, + "ground_truth": ground_truth, + "answers": [ground_truth] if ground_truth else [], + "task_type": task_type, + "category": task_type, + "source_files": source_files, + "source_docs": source_docs, + "split": split, + } + + +class OfficeQADataLoader(SplitDataLoader): + def load_split_items(self, split_path: str) -> list[dict]: + path = Path(split_path) + csv_files = sorted(path.glob("*.csv")) + if csv_files: + with csv_files[0].open(encoding="utf-8", newline="") as f: + reader = csv.DictReader(f) + return [_normalize_row(row) for row in reader] + + json_files = sorted(path.glob("*.json")) + if json_files: + with json_files[0].open(encoding="utf-8") as f: + data = json.load(f) + if not isinstance(data, list): + raise ValueError(f"Expected JSON array in {json_files[0]}") + return [_normalize_row(item) for item in data] + + raise FileNotFoundError(f"No .csv or .json file found in {split_path}") diff --git a/skillopt/envs/officeqa/evaluator.py b/skillopt/envs/officeqa/evaluator.py new file mode 100644 index 0000000..124d25d --- /dev/null +++ b/skillopt/envs/officeqa/evaluator.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import re +import string +from collections import Counter + + +_NUMERIC_CHARS = set("0123456789.-") + + +def normalize_answer(text: str) -> str: + text = text.lower().strip() + text = text.replace(",", "") + text = "".join(ch for ch in text if ch not in string.punctuation or ch in _NUMERIC_CHARS or ch == "%") + text = re.sub(r"\b(million|millions|billion|billions|dollars|dollar|nominal)\b", " ", text) + text = " ".join(text.split()) + return text + + +def exact_match(prediction: str, gold: str) -> float: + return 1.0 if normalize_answer(prediction) == normalize_answer(gold) else 0.0 + + +def token_f1(prediction: str, gold: str) -> float: + pred_tokens = normalize_answer(prediction).split() + gold_tokens = normalize_answer(gold).split() + if not pred_tokens or not gold_tokens: + return 1.0 if pred_tokens == gold_tokens else 0.0 + common = Counter(pred_tokens) & Counter(gold_tokens) + n_common = sum(common.values()) + if n_common == 0: + return 0.0 + precision = n_common / len(pred_tokens) + recall = n_common / len(gold_tokens) + return 2 * precision * recall / (precision + recall) + + +def evaluate(prediction: str, gold: str) -> dict: + em = exact_match(prediction, gold) + f1 = token_f1(prediction, gold) + return { + "em": em, + "f1": f1, + "predicted_answer": prediction.strip(), + "gold_answer": gold, + } diff --git a/skillopt/envs/officeqa/prompts/analyst_error.md b/skillopt/envs/officeqa/prompts/analyst_error.md new file mode 100644 index 0000000..ec9a87e --- /dev/null +++ b/skillopt/envs/officeqa/prompts/analyst_error.md @@ -0,0 +1,37 @@ +You are an expert failure-analysis agent for OfficeQA document-retrieval question answering tasks. + +You will be given MULTIPLE failed OfficeQA trajectories from a single minibatch and the current skill document. The trajectories may include local document tool calls such as file search, grep, and partial reads. + +Your job is to identify COMMON failure patterns across the batch and propose concise skill edits. + +## Failure Type Categories +- retrieval_miss: the agent searched the wrong file or failed to narrow to the right file +- evidence_miss: the agent read documents but missed the decisive evidence span +- operand_error: the agent extracted the wrong value or the wrong operands +- calculation_error: the agent identified the right evidence but computed the result incorrectly +- answer_format: the agent reached the right result but formatted it wrong +- other: none of the above + +## Rules +- Focus on patterns common across multiple trajectories. +- Prefer general retrieval and evidence-grounding rules over task-specific hacks. +- Only patch gaps in the skill; do not duplicate rules already present. +- Do not hardcode file names, years, or question-specific constants unless the pattern truly requires a reusable retrieval heuristic. + +Respond ONLY with a valid JSON object (no markdown fences, no extra text): +{ + "batch_size": , + "failure_summary": [ + {"failure_type": "", "count": , "description": ""} + ], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +Only include edits that are needed. "edits" can be an empty list if no patch is warranted. diff --git a/skillopt/envs/officeqa/prompts/analyst_success.md b/skillopt/envs/officeqa/prompts/analyst_success.md new file mode 100644 index 0000000..4ce3da5 --- /dev/null +++ b/skillopt/envs/officeqa/prompts/analyst_success.md @@ -0,0 +1,25 @@ +You are an expert success-pattern analyst for OfficeQA document-retrieval question answering tasks. + +You will be given MULTIPLE successful OfficeQA trajectories from a single minibatch and the current skill document. Your job is to identify common retrieval, evidence-selection, and numeric-grounding behaviors worth encoding in the skill. + +## Rules +- Focus on patterns shared across multiple successful trajectories. +- Prefer reusable retrieval and extraction discipline over question-specific tips. +- Reinforce compact, high-value behaviors such as narrowing files early, reading only the relevant span, building a clean operand ledger, and copying the final answer from checked evidence. +- Only propose patches for patterns not already captured in the current skill. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "success_patterns": ["", ""], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +"edits" may be empty if the skill already covers all observed patterns. diff --git a/skillopt/envs/officeqa/prompts/rollout_system.md b/skillopt/envs/officeqa/prompts/rollout_system.md new file mode 100644 index 0000000..db22931 --- /dev/null +++ b/skillopt/envs/officeqa/prompts/rollout_system.md @@ -0,0 +1,15 @@ +You are an expert OfficeQA agent working over local Treasury bulletin text files. + +{skill_section}## Rules +1. Use only the provided local document tools to inspect candidate files. +2. Narrow to the most relevant file before reading long passages. +3. Prefer short targeted searches, then small reads around matching evidence. +4. Do not invent values that are not grounded in the retrieved text. +5. When the question requires arithmetic, compute only after extracting the exact operands. +6. If you have enough evidence, return the final answer inside .... + +## Tool Use +Use the provided function tools directly when you need them. Prefer searching and small reads before answering. Do not ask the user for permission to use tools; just call the tools. + +## Final Answer Format +When you are ready to answer, emit the final answer inside ... and do not request another tool. diff --git a/skillopt/envs/officeqa/rollout.py b/skillopt/envs/officeqa/rollout.py new file mode 100644 index 0000000..01afe8b --- /dev/null +++ b/skillopt/envs/officeqa/rollout.py @@ -0,0 +1,799 @@ +from __future__ import annotations +import json +import os +import re +from concurrent.futures import ThreadPoolExecutor, as_completed +from skillopt.envs.officeqa.evaluator import evaluate +from skillopt.envs.officeqa.tool_runtime import ( + build_oracle_parsed_pages_context, + custom_search, + resolve_candidate_files, + resolve_docs_roots, + run_tool, +) +from skillopt.model import chat_target_messages, get_target_backend, is_target_exec_backend +from skillopt.model.codex_harness import prepare_workspace, render_skill_md, run_target_exec +from skillopt.prompts import load_prompt +_TOOL_SCHEMAS = [ + { + "type": "function", + "function": { + "name": "glob", + "description": "Find candidate local document files by filename or relative-path glob pattern.", + "parameters": { + "type": "object", + "properties": {"pattern": {"type": "string"}}, + "required": ["pattern"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "read", + "description": "Read a local text document excerpt by path and line window.", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "start": {"type": "integer"}, + "limit": {"type": "integer"}, + }, + "required": ["path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "grep", + "description": "Search a local text document for a literal pattern and return matching lines.", + "parameters": { + "type": "object", + "properties": { + "pattern": {"type": "string"}, + "path": {"type": "string"}, + }, + "required": ["pattern", "path"], + }, + }, + }, +] +_FINAL_RE = re.compile(r"(.*?)", re.IGNORECASE | re.DOTALL) +_SEARCH_RE = re.compile(r"(.*?)", re.IGNORECASE | re.DOTALL) +_DEFAULT_SEARCH_MODE = "offline" +_CUSTOM_SEARCH_MODE = "custom_search" +_AZURE_SEARCH_MODE = "azure_search" +def _normalize_search_mode(search_mode: str | None) -> str: + normalized = str(search_mode or _DEFAULT_SEARCH_MODE).strip().lower() + if normalized in {"custom", _CUSTOM_SEARCH_MODE}: + return _CUSTOM_SEARCH_MODE + if normalized in {"azure", _AZURE_SEARCH_MODE}: + return _AZURE_SEARCH_MODE + return _DEFAULT_SEARCH_MODE +def _build_system( + skill_content: str, + *, + search_mode: str = _DEFAULT_SEARCH_MODE, + use_local_tools: bool = True, + max_tool_turns: int = 12, + max_queries_per_turn: int = 4, +) -> str: + if skill_content.strip(): + skill_section = f"## Skill\n{skill_content.strip()}\n\n" + else: + skill_section = "" + normalized_search_mode = _normalize_search_mode(search_mode) + if normalized_search_mode == _AZURE_SEARCH_MODE: + return ( + "You are an expert OfficeQA research assistant. Solve the question using the model's built-in web " + "search tool when needed, keep the answer grounded in authoritative evidence, and return the final " + "answer inside ....\n\n" + + skill_section + ).rstrip() + if normalized_search_mode == _CUSTOM_SEARCH_MODE: + protocol = ( + "You are an expert OfficeQA research assistant. Solve the question using the provided oracle parsed " + "OfficeQA page(s) and evidence returned by the controller-managed custom search loop.\n\n" + "Search protocol:\n" + f"- You have at most {max_tool_turns} model rounds total.\n" + f"- On any non-final round, you may either return `[\"query 1\", \"query 2\"]` " + f"with up to {max_queries_per_turn} queries, or return `...` if you are ready.\n" + "- If you request search, do not include an answer in the same response.\n" + "- On the final round, you must return `...` and must not request more search.\n" + "- Base your answer on the returned evidence, reconcile conflicting snippets carefully, and stay concise.\n\n" + ) + return protocol + skill_section + "Return the final answer inside ... when you are ready." + if not use_local_tools: + return ( + "You are an expert OfficeQA research assistant. Solve the question using the provided oracle parsed " + "OfficeQA page(s) and source hints. Do not request or assume access to any external search or local " + "function tools. Return the final answer inside ....\n\n" + + skill_section + ).rstrip() + return load_prompt("rollout_system", env="officeqa").format(skill_section=skill_section) +def _build_round_instruction( + *, + turn: int, + max_tool_turns: int, + max_queries_per_turn: int, +) -> str: + if turn >= max_tool_turns: + return ( + "## Round Policy\n" + f"This is the final round ({turn}/{max_tool_turns}). You must return `...` now. " + "Do not output ``." + ) + remaining_rounds = max_tool_turns - turn + return ( + "## Round Policy\n" + f"This is round {turn}/{max_tool_turns}. " + f"You may either return `...` now, or request up to {max_queries_per_turn} search queries " + f"inside `...`. " + f"After this response, at most {remaining_rounds} model rounds remain." + ) +def _message_debug_metadata(message: object) -> dict: + metadata = getattr(message, "metadata", None) + if isinstance(metadata, dict): + return metadata + return {} +def _build_user( + item: dict, + candidate_files: list[str] | None = None, + *, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + corpus_note: str = "", + search_mode: str = _DEFAULT_SEARCH_MODE, + turn: int = 1, + max_tool_turns: int = 12, + max_queries_per_turn: int = 4, + oracle_context: str = "", +) -> str: + normalized_search_mode = _normalize_search_mode(search_mode) + parts = [f"## Question\n{item['question']}"] + if oracle_context.strip(): + parts.append(f"## Oracle Parsed Pages\n{oracle_context.strip()}") + if normalized_search_mode == _DEFAULT_SEARCH_MODE: + file_block = "\n".join(f"- {path}" for path in (candidate_files or [])[:20]) or "- none resolved" + if corpus_note.strip(): + parts.append(f"## Document Corpus\n{corpus_note.strip()}") + parts.append(f"## Candidate Files\n{file_block}") + if item.get("source_docs"): + parts.append("## Source Hints\n" + "\n".join(f"- {hint}" for hint in item["source_docs"])) + if normalized_search_mode != _DEFAULT_SEARCH_MODE and item.get("source_files"): + parts.append("## File Hints\n" + "\n".join(f"- {hint}" for hint in item["source_files"])) + if diagnostic_mode and diagnostic_instruction.strip(): + parts.append(f"## Training Readout\n{diagnostic_instruction.strip()}") + if normalized_search_mode == _CUSTOM_SEARCH_MODE: + parts.append( + _build_round_instruction( + turn=turn, + max_tool_turns=max_tool_turns, + max_queries_per_turn=max_queries_per_turn, + ) + ) + parts.append( + "## Output Format\n" + "If you need more evidence, return only `[...]`.\n" + "If you are ready to answer, return only `...`." + ) + parts.append( + "Use only the provided oracle parsed pages and controller-provided custom search evidence. " + "Do not rely on any built-in web search capability." + ) + elif normalized_search_mode == _AZURE_SEARCH_MODE: + parts.append("Use the model's built-in web search tool when needed. Return the final answer inside ....") + return "\n\n".join(parts) +def _extract_answer(text: str) -> str: + match = _FINAL_RE.search(text) + if match: + return match.group(1).strip() + lines = [line.strip() for line in text.splitlines() if line.strip()] + return lines[-1] if lines else text.strip() +def _extract_search_queries(text: str) -> list[str]: + match = _SEARCH_RE.search(text or "") + if not match: + return [] + raw = match.group(1).strip() + if not raw: + return [] + parsed_queries: list[str] = [] + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + parsed = None + if isinstance(parsed, dict): + for key in ("queries", "search_queries", "query"): + value = parsed.get(key) + if isinstance(value, str) and value.strip(): + parsed_queries = [value.strip()] + break + if isinstance(value, list): + parsed_queries = [str(item).strip() for item in value if str(item).strip()] + break + elif isinstance(parsed, list): + parsed_queries = [str(item).strip() for item in parsed if str(item).strip()] + elif isinstance(parsed, str) and parsed.strip(): + parsed_queries = [parsed.strip()] + if not parsed_queries: + raw_lines = [line.strip(" -*\t\r\n\"'") for line in raw.splitlines()] + parsed_queries = [line for line in raw_lines if line] + if len(parsed_queries) <= 1 and parsed_queries: + multi = [part.strip(" \"'") for part in re.split(r"[;,]", parsed_queries[0]) if part.strip(" \"'")] + if len(multi) > 1: + parsed_queries = multi + deduped: list[str] = [] + seen: set[str] = set() + for query in parsed_queries: + normalized = query.strip() + if not normalized or normalized in seen: + continue + seen.add(normalized) + deduped.append(normalized) + return deduped +def _docs_link_targets(docs_roots: list[str]) -> list[tuple[str, str]]: + return [(root, os.path.join("docs", f"root_{idx}")) for idx, root in enumerate(docs_roots, start=1)] +def _workspace_doc_path(path: str, docs_roots: list[str]) -> str: + resolved_path = os.path.realpath(path) + for idx, root in enumerate(docs_roots, start=1): + resolved_root = os.path.realpath(root) + if resolved_path == resolved_root or resolved_path.startswith(resolved_root + os.sep): + rel_path = os.path.relpath(resolved_path, resolved_root) + return os.path.join("docs", f"root_{idx}", rel_path) + return path +def _build_codex_skill(skill_content: str) -> str: + return render_skill_md( + skill_content, + description="Dynamic ReflACT skill for solving the current OfficeQA local-document question.", + preamble=( + "Use this skill when answering the current OfficeQA question.\n" + "Inspect the provided local document excerpts or files, ground the answer in the evidence,\n" + "and return the final answer inside ...." + ), + ) +def _run_codex_once( + *, + pred_dir: str, + item: dict, + skill_content: str, + candidate_files: list[str], + docs_roots: list[str], + model: str, + timeout: int, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + previous_response: str = "", + oracle_context: str = "", +) -> tuple[str, str, str, str]: + rel_files = [_workspace_doc_path(path, docs_roots) for path in candidate_files[:20]] + corpus_note = ( + "The full OfficeQA document corpus is available under `docs/`. " + "The candidate files below are source hints or likely starting points; search the full corpus if needed." + ) + user = _build_user( + item, + rel_files, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + corpus_note=corpus_note, + oracle_context=oracle_context, + ) + task_parts = [user] + if previous_response: + task_parts.append( + "## Previous Attempt\n" + f"{previous_response}\n\n" + "Review the local documents again and correct the answer if needed." + ) + task_text = "\n\n".join(task_parts) + skill_md = _build_codex_skill(skill_content) + work_dir = os.path.join(pred_dir, "codex_exec") + prepare_workspace( + work_dir=work_dir, + skill_md=skill_md, + task_text=task_text, + link_dirs=_docs_link_targets(docs_roots), + ) + prompt = ( + "Use the `skillopt-target` skill available in this workspace.\n" + "Read `task.md`, inspect or search the full OfficeQA corpus under `docs/`, and answer the question.\n" + "Treat candidate files in `task.md` as hints, not an access limit.\n" + "Return the final answer inside ...." + ) + final_message, raw = run_target_exec( + work_dir=work_dir, + prompt=prompt, + model=model, + timeout=timeout, + data_dirs=docs_roots, + ) + return final_message or raw, raw, skill_md, task_text +def _execute_custom_search_round( + queries: list[str], + *, + api_url: str, + auth_env: str, + provider: str, + max_num_results: int, + timeout: int, +) -> str: + blocks = [] + for index, query in enumerate(queries, start=1): + try: + result = custom_search( + query, + api_url=api_url, + auth_env=auth_env, + provider=provider, + max_num_results=max_num_results, + timeout=timeout, + ) + except Exception as search_error: # noqa: BLE001 + result = f"Query: {query}\n\n[search error: {search_error}]" + blocks.append(f"## Query {index}\n{result}") + return "\n\n".join(blocks) +def _run_custom_search_process( + item: dict, + skill_content: str, + *, + max_tool_turns: int, + max_completion_tokens: int, + max_queries_per_turn: int, + diagnostic_mode: bool, + diagnostic_instruction: str, + search_api_url: str, + search_auth_env: str, + search_provider: str, + search_max_num_results: int, + search_timeout_seconds: int, + oracle_context: str = "", +) -> tuple[str, str, str, str, list[dict], str, dict]: + if not str(search_api_url or "").strip(): + raise ValueError("custom_search mode requires a non-empty search_api_url") + if not os.environ.get(search_auth_env, "").strip(): + raise ValueError(f"custom_search mode requires auth token env var {search_auth_env}") + if get_target_backend() not in {"openai_chat", "qwen_chat"}: + raise ValueError("custom_search mode is only supported with target_backend='openai_chat' or 'qwen_chat'") + system = _build_system( + skill_content, + search_mode=_CUSTOM_SEARCH_MODE, + max_tool_turns=max_tool_turns, + max_queries_per_turn=max_queries_per_turn, + ) + initial_user = _build_user( + item, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + search_mode=_CUSTOM_SEARCH_MODE, + turn=1, + max_tool_turns=max_tool_turns, + max_queries_per_turn=max_queries_per_turn, + oracle_context=oracle_context, + ) + latest_user = initial_user + messages: list[dict] = [ + {"role": "system", "content": system}, + {"role": "user", "content": initial_user}, + ] + conversation: list[dict] = [{"role": "user", "content": initial_user}] + final_response = "" + final_answer = "" + fail_reason = "" + last_response_metadata: dict = {} + for turn in range(1, max_tool_turns + 1): + message, _ = chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=5, + stage="rollout", + return_message=True, + ) + response = message.content or "" + final_response = response + last_response_metadata = _message_debug_metadata(message) + messages.append({"role": "assistant", "content": response}) + message_event = {"type": "message", "turn": turn, "content": response} + if last_response_metadata: + message_event["response_metadata"] = last_response_metadata + conversation.append(message_event) + if "" in response.lower(): + final_answer = _extract_answer(response) + return system, latest_user, final_response, final_answer, conversation, "", last_response_metadata + if turn == max_tool_turns: + fail_reason = f"Final round ({max_tool_turns}) ended without ..." + break + queries = _extract_search_queries(response)[:max_queries_per_turn] + if not queries: + fail_reason = "Model neither produced search queries nor a final answer" + break + results_text = _execute_custom_search_round( + queries, + api_url=search_api_url, + auth_env=search_auth_env, + provider=search_provider, + max_num_results=search_max_num_results, + timeout=search_timeout_seconds, + ) + conversation.append({"type": "tool_call", "turn": turn, "cmd": f"custom_search({queries!r})", "obs": results_text}) + latest_user = ( + f"## Search Results Round {turn}\n{results_text}\n\n" + + _build_round_instruction( + turn=turn + 1, + max_tool_turns=max_tool_turns, + max_queries_per_turn=max_queries_per_turn, + ) + + "\n\nFollow the round policy above exactly." + ) + messages.append({"role": "user", "content": latest_user}) + conversation.append({"role": "user", "turn": turn + 1, "content": latest_user}) + return system, latest_user, final_response, final_answer, conversation, fail_reason, last_response_metadata +def _run_azure_search_process( + item: dict, + skill_content: str, + *, + max_completion_tokens: int, + diagnostic_mode: bool, + diagnostic_instruction: str, +) -> tuple[str, str, str, str, list[dict], str, dict]: + if get_target_backend() != "openai_chat": + raise ValueError("azure_search mode is only supported with target_backend='openai_chat'") + system = _build_system(skill_content, search_mode=_AZURE_SEARCH_MODE) + user = _build_user( + item, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + search_mode=_AZURE_SEARCH_MODE, + ) + messages = [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + conversation: list[dict] = [{"role": "user", "content": user}] + message, _ = chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=5, + stage="rollout", + return_message=True, + tools=[{"type": "web_search"}], + ) + response = message.content or "" + last_response_metadata = _message_debug_metadata(message) + message_event = {"type": "message", "content": response} + if last_response_metadata: + message_event["response_metadata"] = last_response_metadata + conversation.append(message_event) + if "" in response.lower(): + return system, user, response, _extract_answer(response), conversation, "", last_response_metadata + return system, user, response, "", conversation, "Model did not produce a final answer", last_response_metadata +def _run_offline_no_tools_process( + item: dict, + skill_content: str, + *, + max_completion_tokens: int, + diagnostic_mode: bool, + diagnostic_instruction: str, + candidate_files: list[str], + oracle_context: str = "", +) -> tuple[str, str, str, str, list[dict], str, dict]: + system = _build_system(skill_content, search_mode=_DEFAULT_SEARCH_MODE, use_local_tools=False) + user = _build_user( + item, + candidate_files, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + search_mode=_DEFAULT_SEARCH_MODE, + oracle_context=oracle_context, + ) + messages = [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + conversation: list[dict] = [{"role": "user", "content": user}] + message, _ = chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=5, + stage="rollout", + return_message=True, + ) + response = message.content or "" + last_response_metadata = _message_debug_metadata(message) + message_event = {"type": "message", "content": response} + if last_response_metadata: + message_event["response_metadata"] = last_response_metadata + conversation.append(message_event) + if "" in response.lower(): + return system, user, response, _extract_answer(response), conversation, "", last_response_metadata + return system, user, response, "", conversation, "Model did not produce a final answer", last_response_metadata +def process_one( + item: dict, + out_root: str, + skill_content: str, + *, + max_tool_turns: int = 12, + max_completion_tokens: int = 16384, + search_mode: str = _DEFAULT_SEARCH_MODE, + max_queries_per_turn: int = 4, + search_api_url: str = "", + search_auth_env: str = "OFFICEQA_CUSTOM_SEARCH_AUTH", + search_provider: str = "duckduckgo", + search_max_num_results: int = 4, + search_timeout_seconds: int = 20, + use_local_tools: bool = True, + data_dirs: list[str] | str | None = None, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", +) -> dict: + item_id = str(item["id"]) + pred_dir = os.path.join(out_root, "predictions", item_id) + os.makedirs(pred_dir, exist_ok=True) + normalized_search_mode = _normalize_search_mode(search_mode) + docs_roots: list[str] = [] + candidate_files: list[str] = [] + oracle_context = "" + if normalized_search_mode == _DEFAULT_SEARCH_MODE: + docs_roots = resolve_docs_roots(data_dirs) + candidate_files = resolve_candidate_files(item.get("source_files", []), docs_roots) + oracle_context = build_oracle_parsed_pages_context( + item.get("source_files", []), + item.get("source_docs", []), + docs_roots, + evidence_note=( + "Treat it as primary document evidence and combine it with local document tool evidence when useful." + if use_local_tools + else "Treat it as primary document evidence for answering the question." + ), + ) + elif normalized_search_mode == _CUSTOM_SEARCH_MODE: + docs_roots = resolve_docs_roots(data_dirs) + if item.get("source_files"): + candidate_files = resolve_candidate_files(item.get("source_files", []), docs_roots) + oracle_context = build_oracle_parsed_pages_context( + item.get("source_files", []), + item.get("source_docs", []), + docs_roots, + ) + system = _build_system( + skill_content, + search_mode=normalized_search_mode, + use_local_tools=use_local_tools, + max_tool_turns=max_tool_turns, + max_queries_per_turn=max_queries_per_turn, + ) + user = _build_user( + item, + candidate_files if normalized_search_mode == _DEFAULT_SEARCH_MODE else None, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + search_mode=normalized_search_mode, + max_tool_turns=max_tool_turns, + max_queries_per_turn=max_queries_per_turn, + oracle_context=oracle_context, + ) + conversation: list[dict] = [{"role": "user", "content": user}] + final_response = "" + final_answer = "" + fail_reason = "" + last_response_metadata: dict = {} + allowed_files = [os.path.basename(path) for path in candidate_files] + try: + if normalized_search_mode == _CUSTOM_SEARCH_MODE: + system, user, final_response, final_answer, conversation, fail_reason, last_response_metadata = _run_custom_search_process( + item, + skill_content, + max_tool_turns=max_tool_turns, + max_completion_tokens=max_completion_tokens, + max_queries_per_turn=max_queries_per_turn, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + search_api_url=search_api_url, + search_auth_env=search_auth_env, + search_provider=search_provider, + search_max_num_results=search_max_num_results, + search_timeout_seconds=search_timeout_seconds, + oracle_context=oracle_context, + ) + elif normalized_search_mode == _AZURE_SEARCH_MODE: + system, user, final_response, final_answer, conversation, fail_reason, last_response_metadata = _run_azure_search_process( + item, + skill_content, + max_completion_tokens=max_completion_tokens, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + ) + elif not use_local_tools: + system, user, final_response, final_answer, conversation, fail_reason, last_response_metadata = _run_offline_no_tools_process( + item, + skill_content, + max_completion_tokens=max_completion_tokens, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + candidate_files=candidate_files, + oracle_context=oracle_context, + ) + elif is_target_exec_backend(): + from skillopt.model import azure_openai as _llm + response = "" + system = "" + user = "" + for turn in range(1, max_tool_turns + 1): + response, _raw, system, user = _run_codex_once( + pred_dir=pred_dir, + item=item, + skill_content=skill_content, + candidate_files=candidate_files, + docs_roots=docs_roots, + model=_llm.TARGET_DEPLOYMENT, + timeout=180, + diagnostic_mode=diagnostic_mode if turn == 1 else False, + diagnostic_instruction=diagnostic_instruction if turn == 1 else "", + previous_response=response if turn > 1 else "", + oracle_context=oracle_context, + ) + final_response = response + conversation.append({"type": "message", "turn": turn, "content": response}) + if "" in response.lower(): + final_answer = _extract_answer(response) + break + if not final_answer: + fail_reason = f"Exceeded codex turn budget ({max_tool_turns})" + system = system or _build_codex_skill(skill_content) + user = user or _build_user(item, [_workspace_doc_path(path, docs_roots) for path in candidate_files]) + else: + messages: list[dict] = [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + for turn in range(1, max_tool_turns + 1): + message, _ = chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=5, + stage="rollout", + tools=_TOOL_SCHEMAS, + tool_choice="auto", + return_message=True, + ) + response = message.content or "" + final_response = response + assistant_message = {"role": "assistant", "content": response} + if getattr(message, "tool_calls", None): + assistant_message["tool_calls"] = [tool_call.model_dump(mode="json") for tool_call in message.tool_calls] + messages.append(assistant_message) + conversation.append({"type": "message", "content": response}) + if getattr(message, "tool_calls", None): + for tool_call in message.tool_calls: + tool_name = tool_call.function.name + arguments = json.loads(tool_call.function.arguments) if tool_call.function.arguments else {} + cmd, obs = run_tool(tool_name, arguments, allowed_roots=docs_roots, allowed_files=allowed_files) + conversation.append({"type": "tool_call", "cmd": cmd, "obs": obs}) + messages.append({ + "role": "tool", + "tool_call_id": tool_call.id, + "content": obs, + }) + continue + if "" in response.lower(): + final_answer = _extract_answer(response) + break + if turn == max_tool_turns: + fail_reason = f"Exceeded tool-turn budget ({max_tool_turns})" + else: + fail_reason = "Model neither produced a tool request nor a final answer" + break + except Exception as e: # noqa: BLE001 + fail_reason = f"error: {e}" + with open(os.path.join(pred_dir, "target_system_prompt.txt"), "w", encoding="utf-8") as f: + f.write(system) + with open(os.path.join(pred_dir, "target_user_prompt.txt"), "w", encoding="utf-8") as f: + f.write(user) + with open(os.path.join(pred_dir, "conversation.json"), "w", encoding="utf-8") as f: + json.dump(conversation, f, ensure_ascii=False, indent=2) + eval_result = evaluate(final_answer, item.get("ground_truth", "")) if final_answer else {"em": 0.0, "f1": 0.0, "predicted_answer": "", "gold_answer": item.get("ground_truth", "")} + result = { + "id": item_id, + "question": item.get("question", ""), + "task_type": item.get("task_type", "officeqa"), + "task_description": item.get("question", ""), + "predicted_answer": eval_result["predicted_answer"], + "response": final_response, + "ground_truth": item.get("ground_truth", ""), + "source_files": item.get("source_files", []), + "resolved_source_paths": candidate_files, + "oracle_parsed_pages_included": bool(oracle_context), + "oracle_parsed_pages_chars": len(oracle_context), + "use_local_tools": bool(use_local_tools), + "hard": int(eval_result["em"]), + "soft": eval_result["f1"], + "fail_reason": fail_reason or ("" if eval_result["em"] else f"predicted '{eval_result['predicted_answer']}' but expected '{item.get('ground_truth', '')}'"), + "agent_ok": not fail_reason, + "n_turns": len(conversation), + "last_finish_reason": last_response_metadata.get("finish_reason", ""), + "target_system_prompt": system, + "target_user_prompt": user, + } + return result +def run_batch( + items: list[dict], + out_root: str, + skill_content: str, + *, + workers: int = 8, + max_tool_turns: int = 12, + max_completion_tokens: int = 16384, + search_mode: str = _DEFAULT_SEARCH_MODE, + max_queries_per_turn: int = 4, + search_api_url: str = "", + search_auth_env: str = "OFFICEQA_CUSTOM_SEARCH_AUTH", + search_provider: str = "duckduckgo", + search_max_num_results: int = 4, + search_timeout_seconds: int = 20, + use_local_tools: bool = True, + data_dirs: list[str] | str | None = None, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", +) -> list[dict]: + results_path = os.path.join(out_root, "results.jsonl") + os.makedirs(out_root, exist_ok=True) + done_ids: set[str] = set() + existing: list[dict] = [] + if os.path.exists(results_path): + with open(results_path, encoding="utf-8") as f: + for line in f: + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + done_ids.add(str(row.get("id"))) + existing.append(row) + pending = [item for item in items if str(item["id"]) not in done_ids] + if not pending: + return existing + total = len(existing) + len(pending) + completed = len(existing) + correct_count = sum(1 for r in existing if r.get("hard", 0)) + if existing: + print(f" [rollout] resuming: {completed}/{total} already done", flush=True) + + results = list(existing) + with open(results_path, "a", encoding="utf-8") as outf, ThreadPoolExecutor(max_workers=workers) as ex: + futs = { + ex.submit( + process_one, + item, + out_root, + skill_content, + max_tool_turns=max_tool_turns, + max_completion_tokens=max_completion_tokens, + search_mode=search_mode, + max_queries_per_turn=max_queries_per_turn, + search_api_url=search_api_url, + search_auth_env=search_auth_env, + search_provider=search_provider, + search_max_num_results=search_max_num_results, + search_timeout_seconds=search_timeout_seconds, + use_local_tools=use_local_tools, + data_dirs=data_dirs, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + ): item + for item in pending + } + for fut in as_completed(futs): + res = fut.result() + results.append(res) + completed += 1 + if res.get("hard", 0): + correct_count += 1 + acc = correct_count / completed if completed else 0 + print( + f" [rollout] {completed}/{total} " + f"(acc={acc:.3f}) id={res.get('id', '?')} " + f"hard={res.get('hard', '?')}", + flush=True, + ) + outf.write(json.dumps(res, ensure_ascii=False) + "\n") + outf.flush() + return results diff --git a/skillopt/envs/officeqa/skills/initial.md b/skillopt/envs/officeqa/skills/initial.md new file mode 100644 index 0000000..530b753 --- /dev/null +++ b/skillopt/envs/officeqa/skills/initial.md @@ -0,0 +1,15 @@ +# OfficeQA Skill + +## Retrieval Discipline +- Start by narrowing to the most likely candidate file before reading long passages. +- Prefer targeted search terms that name the exact entity, period, measure, or table concept from the question. +- After a promising match, read only a small surrounding span and verify it matches the requested year, basis, and unit. + +## Evidence Discipline +- Extract the exact value from the retrieved text before doing any arithmetic. +- Keep track of each operand's period, unit, and semantic role so nearby proxy values are not mixed in. +- If the question asks for a transformed or derived quantity, compute only after confirming every operand. + +## Final Answer Discipline +- Return the final answer only after one last consistency check against the retrieved evidence. +- Copy the final answer from a checked value, not from an unverified intermediate guess. diff --git a/skillopt/envs/officeqa/tool_runtime.py b/skillopt/envs/officeqa/tool_runtime.py new file mode 100644 index 0000000..89be327 --- /dev/null +++ b/skillopt/envs/officeqa/tool_runtime.py @@ -0,0 +1,552 @@ +from __future__ import annotations + +import fnmatch +import html +import json +import os +import re +import socket +import time +from functools import lru_cache +from html.parser import HTMLParser +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import parse_qs, urlparse +from urllib.request import Request, urlopen + +_MAX_READ_CHARS = 4000 +_MAX_GREP_MATCHES = 20 +_MAX_GLOB_MATCHES = 50 +_MAX_ORACLE_PAGE_CHARS = 24000 +_MAX_ORACLE_CONTEXT_CHARS = 80000 +DEFAULT_USER_AGENT = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/135.0 Safari/537.36" +) +DEFAULT_CUSTOM_SEARCH_URL = "http://apisix.westus2.cloudapp.azure.com/search_tool/search" +DEFAULT_CUSTOM_SEARCH_AUTH_ENV = "OFFICEQA_CUSTOM_SEARCH_AUTH" +DEFAULT_CUSTOM_SEARCH_PROVIDER = "duckduckgo" +DEFAULT_CUSTOM_SEARCH_MAX_RESULTS = 4 +DEFAULT_CUSTOM_SEARCH_TIMEOUT = 20 +DEFAULT_CUSTOM_SEARCH_MAX_RETRIES = 4 +DEFAULT_CUSTOM_SEARCH_INITIAL_BACKOFF_SECONDS = 1.0 + + +def _normalize_data_dirs(data_dirs: list[str] | tuple[str, ...] | str | None, project_root: Path) -> list[str]: + if data_dirs is None: + return [] + if isinstance(data_dirs, str): + items = [part.strip() for chunk in data_dirs.split(os.pathsep) for part in chunk.split(",")] + else: + items = [str(item).strip() for item in data_dirs] + resolved: list[str] = [] + for item in items: + if not item: + continue + path = Path(item).expanduser() + if not path.is_absolute(): + path = project_root / path + resolved.append(str(path)) + return resolved + + +def resolve_docs_roots(data_dirs: list[str] | tuple[str, ...] | str | None = None) -> list[str]: + project_root = Path(__file__).resolve().parents[3] + env_value = os.environ.get("OFFICEQA_DOCS_DIR", "").strip() + candidates = _normalize_data_dirs(data_dirs, project_root) + candidates.extend(_normalize_data_dirs(env_value, project_root)) + candidates.extend([ + str(project_root / "data" / "officeqa_docs_official"), + str(project_root / "data" / "officeqa_smoke_docs"), + os.path.expanduser("~/officeqa-sparse/treasury_bulletins_parsed"), + os.path.expanduser("~/officeqa/treasury_bulletins_parsed"), + ]) + roots: list[str] = [] + seen: set[str] = set() + for candidate in candidates: + path = Path(candidate).expanduser() + if not path.is_dir(): + continue + transformed = path / "transformed" + resolved = str((transformed if transformed.is_dir() else path).resolve()) + if resolved in seen: + continue + seen.add(resolved) + roots.append(resolved) + if not roots: + raise FileNotFoundError("OfficeQA docs directory not found. Set OFFICEQA_DOCS_DIR or env.data_dirs.") + return roots + + +def _is_allowed(path: str, allowed_roots: list[str], allowed_files: list[str]) -> bool: + try: + resolved = str(Path(path).resolve()) + except FileNotFoundError: + return False + if not any(resolved.startswith(root + os.sep) or resolved == root for root in allowed_roots): + return False + if not allowed_files: + return True + base = os.path.basename(resolved) + return base in allowed_files + + +def resolve_candidate_files(source_files: list[str], allowed_roots: list[str]) -> list[str]: + resolved: list[str] = [] + seen: set[str] = set() + for root in allowed_roots: + for dirpath, _, filenames in os.walk(root): + for filename in filenames: + if source_files and filename not in source_files: + continue + full = str(Path(dirpath, filename).resolve()) + if full in seen: + continue + seen.add(full) + resolved.append(full) + return resolved + + +def _as_list(value: object) -> list[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(item).strip() for item in value if str(item).strip()] + text = str(value).strip() + if not text: + return [] + try: + loaded = json.loads(text) + except json.JSONDecodeError: + loaded = None + if isinstance(loaded, list): + return [str(item).strip() for item in loaded if str(item).strip()] + if "\n" in text: + return [part.strip() for part in text.splitlines() if part.strip()] + return [text] + + +def _extract_page_number(source_doc: str) -> int | None: + text = str(source_doc or "").strip() + if not text: + return None + parsed = urlparse(text) + query = parse_qs(parsed.query) + for key in ("page", "pagenum", "page_id"): + for raw_value in query.get(key, []): + try: + return int(str(raw_value).strip()) + except ValueError: + continue + match = re.search(r"(?:[?&]|^)page=(\d+)", text) + if match: + return int(match.group(1)) + return None + + +def _iter_oracle_refs(source_files: object, source_docs: object) -> list[tuple[str, int, str]]: + files = _as_list(source_files) + docs = _as_list(source_docs) + refs: list[tuple[str, int, str]] = [] + seen: set[tuple[str, int, str]] = set() + if not files or not docs: + return refs + for index, source_doc in enumerate(docs): + page_number = _extract_page_number(source_doc) + if page_number is None: + continue + if index < len(files): + source_file = files[index] + elif len(files) == 1: + source_file = files[0] + else: + continue + key = (source_file, page_number, source_doc) + if key in seen: + continue + seen.add(key) + refs.append(key) + return refs + + +def _parsed_root_candidates(docs_roots: list[str]) -> list[Path]: + candidates: list[Path] = [] + seen: set[str] = set() + for root in docs_roots: + path = Path(root).expanduser() + for candidate in ( + path, + path.parent, + path / "treasury_bulletins_parsed", + path.parent / "treasury_bulletins_parsed", + ): + resolved = str(candidate.resolve()) if candidate.exists() else str(candidate) + if resolved in seen: + continue + seen.add(resolved) + candidates.append(candidate) + return candidates + + +def _locate_parsed_json(source_file: str, docs_roots: list[str]) -> Path | None: + source_path = Path(str(source_file).strip()) + stem = source_path.stem if source_path.suffix else source_path.name + if not stem: + return None + candidate_names = [stem + ".json"] + if source_path.suffix == ".json": + candidate_names.insert(0, source_path.name) + for root in _parsed_root_candidates(docs_roots): + for name in candidate_names: + path = root / "jsons" / name + if path.is_file(): + return path + return None + + +class _TableMarkdownParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.rows: list[list[str]] = [] + self._row: list[str] | None = None + self._cell: list[str] | None = None + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag.lower() == "tr": + self._row = [] + elif tag.lower() in {"td", "th"} and self._row is not None: + self._cell = [] + + def handle_data(self, data: str) -> None: + if self._cell is not None: + self._cell.append(data) + + def handle_endtag(self, tag: str) -> None: + normalized_tag = tag.lower() + if normalized_tag in {"td", "th"} and self._cell is not None and self._row is not None: + cell = re.sub(r"\s+", " ", "".join(self._cell)).strip() + self._row.append(cell) + self._cell = None + elif normalized_tag == "tr" and self._row is not None: + if any(cell for cell in self._row): + self.rows.append(self._row) + self._row = None + self._cell = None + + +def _escape_markdown_cell(value: str) -> str: + return str(value).replace("\n", " ").replace("|", "\\|").strip() + + +def _html_table_to_markdown(raw_html: str) -> str: + parser = _TableMarkdownParser() + try: + parser.feed(raw_html) + except Exception: # noqa: BLE001 + parser.rows = [] + rows = parser.rows + if not rows: + text = re.sub(r"(?is)<[^>]+>", " ", raw_html) + return re.sub(r"\s+", " ", html.unescape(text)).strip() + width = max(len(row) for row in rows) + normalized_rows = [row + [""] * (width - len(row)) for row in rows] + header = normalized_rows[0] + body = normalized_rows[1:] + lines = [ + "| " + " | ".join(_escape_markdown_cell(cell) for cell in header) + " |", + "| " + " | ".join(["---"] * width) + " |", + ] + lines.extend("| " + " | ".join(_escape_markdown_cell(cell) for cell in row) + " |" for row in body) + return "\n".join(lines) + + +def _render_parsed_content(content: str) -> str: + text = content.strip() + if not text: + return "" + if " set[int]: + page_ids: set[int] = set() + bbox = element.get("bbox") + if not isinstance(bbox, list): + return page_ids + for box in bbox: + if not isinstance(box, dict): + continue + raw_page_id = box.get("page_id") + try: + page_ids.add(int(raw_page_id)) + except (TypeError, ValueError): + continue + return page_ids + + +@lru_cache(maxsize=256) +def _load_parsed_elements(json_path: str) -> tuple[dict, ...]: + with open(json_path, encoding="utf-8") as f: + payload = json.load(f) + document = payload.get("document") if isinstance(payload, dict) else {} + elements = document.get("elements") if isinstance(document, dict) else [] + if not isinstance(elements, list): + return () + return tuple(element for element in elements if isinstance(element, dict)) + + +@lru_cache(maxsize=2048) +def _render_parsed_page(json_path: str, page_number: int) -> str: + rendered: list[str] = [] + for element in _load_parsed_elements(json_path): + if page_number not in _element_page_ids(element): + continue + content = element.get("content") + if not isinstance(content, str) or not content.strip(): + continue + section = _render_parsed_content(content) + if section: + rendered.append(section) + return "\n\n".join(rendered).strip() + + +def build_oracle_parsed_pages_context( + source_files: object, + source_docs: object, + docs_roots: list[str], + *, + max_page_chars: int = _MAX_ORACLE_PAGE_CHARS, + max_total_chars: int = _MAX_ORACLE_CONTEXT_CHARS, + evidence_note: str = "Treat it as primary document evidence and combine it with custom web search results when useful.", +) -> str: + """Render oracle parsed OfficeQA pages referenced by source_docs/source_files.""" + refs = _iter_oracle_refs(source_files, source_docs) + if not refs: + return "" + + blocks: list[str] = [] + total_chars = 0 + seen_pages: set[tuple[str, int]] = set() + for source_file, page_number, source_doc in refs: + json_path = _locate_parsed_json(source_file, docs_roots) + if json_path is None: + continue + page_key = (str(json_path), page_number) + if page_key in seen_pages: + continue + seen_pages.add(page_key) + page_text = _render_parsed_page(str(json_path), page_number) + if not page_text: + continue + if len(page_text) > max_page_chars: + omitted = len(page_text) - max_page_chars + page_text = page_text[:max_page_chars].rstrip() + f"\n\n[... {omitted} characters omitted from this parsed page ...]" + block = ( + f"### {source_file} page {page_number}\n" + f"Source URL: {source_doc}\n\n" + f"{page_text}" + ) + if total_chars + len(block) > max_total_chars: + remaining = max_total_chars - total_chars + if remaining <= 0: + break + block = block[:remaining].rstrip() + "\n\n[... oracle parsed page context truncated ...]" + blocks.append(block) + break + blocks.append(block) + total_chars += len(block) + if not blocks: + return "" + return ( + "The following content is pre-parsed from the oracle OfficeQA source page(s). " + f"{evidence_note.strip()}\n\n" + + "\n\n".join(blocks) + ) + + +def _extract_search_items(payload: object) -> list[dict]: + if isinstance(payload, list): + return [item for item in payload if isinstance(item, dict)] + if not isinstance(payload, dict): + return [] + candidate_keys = ( + "results", + "items", + "data", + "organic", + "organic_results", + "search_results", + "webPages", + "value", + ) + for key in candidate_keys: + value = payload.get(key) + if isinstance(value, list): + return [item for item in value if isinstance(item, dict)] + if isinstance(value, dict): + nested = _extract_search_items(value) + if nested: + return nested + return [] + + +def _normalize_search_item(item: dict, index: int) -> str: + title = str( + item.get("title") + or item.get("name") + or item.get("headline") + or item.get("source") + or f"Result {index}" + ).strip() + url = str( + item.get("url") + or item.get("link") + or item.get("href") + or item.get("display_url") + or "" + ).strip() + snippet = str( + item.get("snippet") + or item.get("description") + or item.get("body") + or item.get("text") + or item.get("content") + or "" + ).strip() + lines = [f"[{index}] {title}"] + if url: + lines.append(f"URL: {url}") + if snippet: + lines.append(f"Snippet: {snippet}") + return "\n".join(lines) + + +def _format_search_payload(query: str, payload: object) -> str: + items = _extract_search_items(payload) + header = f"Query: {query}" + if not items: + body = json.dumps(payload, ensure_ascii=False) if payload else "[no results]" + return f"{header}\n{body}" + rendered = [_normalize_search_item(item, index) for index, item in enumerate(items, start=1)] + return f"{header}\n\n" + "\n\n".join(rendered) + + +def _is_retryable_search_http_error(status_code: int) -> bool: + return status_code in {408, 429} or status_code >= 500 + + +def custom_search( + query: str, + *, + api_url: str = DEFAULT_CUSTOM_SEARCH_URL, + auth_token: str | None = None, + auth_env: str = DEFAULT_CUSTOM_SEARCH_AUTH_ENV, + provider: str = DEFAULT_CUSTOM_SEARCH_PROVIDER, + max_num_results: int = DEFAULT_CUSTOM_SEARCH_MAX_RESULTS, + timeout: int = DEFAULT_CUSTOM_SEARCH_TIMEOUT, + max_retries: int = DEFAULT_CUSTOM_SEARCH_MAX_RETRIES, + initial_backoff_seconds: float = DEFAULT_CUSTOM_SEARCH_INITIAL_BACKOFF_SECONDS, +) -> str: + query = str(query or "").strip() + if not query: + raise ValueError("custom_search query must be non-empty") + token = str(auth_token or os.environ.get(auth_env, "")).strip() + if not token: + raise ValueError(f"custom_search auth token missing; set {auth_env}") + payload = json.dumps( + { + "query": query, + "max_num_results": int(max_num_results), + "provider": provider, + }, + ensure_ascii=False, + ).encode("utf-8") + req = Request( + api_url, + data=payload, + headers={ + "Authorization": token, + "Content-Type": "application/json", + "User-Agent": DEFAULT_USER_AGENT, + }, + method="POST", + ) + attempts = max(1, int(max_retries) + 1) + last_error: RuntimeError | None = None + for attempt in range(1, attempts + 1): + try: + with urlopen(req, timeout=timeout) as response: + raw_body = response.read().decode("utf-8", errors="ignore") + break + except HTTPError as exc: + detail = exc.read().decode("utf-8", errors="ignore") + last_error = RuntimeError(f"custom_search HTTP {exc.code}: {detail[:1000]}") + if attempt >= attempts or not _is_retryable_search_http_error(exc.code): + raise last_error from exc + except (URLError, TimeoutError, socket.timeout) as exc: + last_error = RuntimeError(f"custom_search connection error: {exc}") + if attempt >= attempts: + raise last_error from exc + backoff_seconds = max(0.0, float(initial_backoff_seconds)) * (2 ** (attempt - 1)) + if backoff_seconds > 0: + time.sleep(backoff_seconds) + else: + raise last_error or RuntimeError("custom_search failed without a captured error") + try: + parsed = json.loads(raw_body) + except json.JSONDecodeError: + return f"Query: {query}\n\n{raw_body.strip() or '[empty response]'}" + return _format_search_payload(query, parsed) + + +def run_tool(name: str, arguments: dict, *, allowed_roots: list[str], allowed_files: list[str]) -> tuple[str, str]: + if name == "glob": + pattern = str(arguments.get("pattern") or "*") + matches: list[str] = [] + for root in allowed_roots: + for dirpath, _, filenames in os.walk(root): + for filename in filenames: + if allowed_files and filename not in allowed_files: + continue + rel = os.path.relpath(os.path.join(dirpath, filename), root) + if fnmatch.fnmatch(rel, pattern) or fnmatch.fnmatch(filename, pattern): + matches.append(os.path.join(dirpath, filename)) + if len(matches) >= _MAX_GLOB_MATCHES: + break + if len(matches) >= _MAX_GLOB_MATCHES: + break + return f"glob(pattern={pattern!r})", "\n".join(matches) if matches else "[no matches]" + + if name == "read": + path = str(arguments.get("path") or "") + if not path: + return "read(path='')", "[read error: missing path]" + if not _is_allowed(path, allowed_roots, allowed_files): + return f"read(path={path!r})", "[read error: path not allowed]" + start = max(int(arguments.get("start") or 1), 1) + limit = max(int(arguments.get("limit") or 80), 1) + with open(path, encoding="utf-8") as f: + lines = f.readlines() + excerpt = "".join(lines[start - 1:start - 1 + limit]) + return f"read(path={path!r}, start={start}, limit={limit})", excerpt[:_MAX_READ_CHARS] or "[empty file]" + + if name == "grep": + pattern = str(arguments.get("pattern") or "").lower() + path = str(arguments.get("path") or "") + if not pattern or not path: + return f"grep(pattern={pattern!r}, path={path!r})", "[grep error: missing pattern or path]" + if not _is_allowed(path, allowed_roots, allowed_files): + return f"grep(pattern={pattern!r}, path={path!r})", "[grep error: path not allowed]" + matches: list[str] = [] + with open(path, encoding="utf-8") as f: + for idx, line in enumerate(f, start=1): + if pattern in line.lower(): + matches.append(f"{idx}: {line.rstrip()}") + if len(matches) >= _MAX_GREP_MATCHES: + break + return f"grep(pattern={pattern!r}, path={path!r})", "\n".join(matches) if matches else "[no matches]" + + return name, f"[tool error: unknown tool {name}]" diff --git a/skillopt/envs/searchqa/__init__.py b/skillopt/envs/searchqa/__init__.py new file mode 100644 index 0000000..d60fc5f --- /dev/null +++ b/skillopt/envs/searchqa/__init__.py @@ -0,0 +1 @@ +"""SearchQA environment package for ReflACT.""" diff --git a/skillopt/envs/searchqa/adapter.py b/skillopt/envs/searchqa/adapter.py new file mode 100644 index 0000000..d173b96 --- /dev/null +++ b/skillopt/envs/searchqa/adapter.py @@ -0,0 +1,96 @@ +"""SearchQA environment adapter for ReflACT.""" +from __future__ import annotations + +import json + +from skillopt.datasets.base import BatchSpec +from skillopt.envs.base import EnvAdapter +from skillopt.envs.searchqa.dataloader import SearchQADataLoader +from skillopt.envs.searchqa.rollout import run_batch +from skillopt.model import get_target_backend + + +class SearchQAAdapter(EnvAdapter): + """SearchQA environment adapter.""" + + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "ratio", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + max_turns: int = 1, + exec_timeout: int = 120, + workers: int = 64, + analyst_workers: int = 16, + failure_only: bool = False, + minibatch_size: int = 8, + edit_budget: int = 4, + seed: int = 42, + limit: int = 0, + max_completion_tokens: int = 16384, + ) -> None: + self.max_turns = max_turns + self.exec_timeout = exec_timeout + self.workers = workers + self.max_completion_tokens = int(max_completion_tokens) + self.analyst_workers = analyst_workers + self.failure_only = failure_only + self.minibatch_size = minibatch_size + self.edit_budget = edit_budget + self.dataloader = SearchQADataLoader( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + seed=seed, + limit=limit, + ) + + def setup(self, cfg: dict) -> None: + super().setup(cfg) + self.dataloader.setup(cfg) + + def get_dataloader(self): + return self.dataloader + + def build_env_from_batch(self, batch: BatchSpec, **kwargs): + return list(batch.payload or []) + + def build_train_env(self, batch_size: int, seed: int, **kwargs): + batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs): + batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def rollout( + self, + env_manager, # actually list[dict] for SearchQA + skill_content: str, + out_dir: str, + **kwargs, + ) -> list[dict]: + """Run QA agent on items. Resume-aware.""" + items: list[dict] = env_manager # type alias for clarity + return run_batch( + items=items, + out_root=out_dir, + skill_content=skill_content, + max_turns=self.max_turns, + exec_timeout=self.exec_timeout, + workers=self.workers, + max_completion_tokens=self.max_completion_tokens, + diagnostic_mode=kwargs.get("diagnostic_mode", False), + diagnostic_instruction=kwargs.get("diagnostic_instruction", ""), + diagnostic_trace_context_by_id=kwargs.get("diagnostic_trace_context_by_id"), + task_timeout=self.exec_timeout, + ) + + def get_task_types(self) -> list[str]: + return ["qa"] diff --git a/skillopt/envs/searchqa/dataloader.py b/skillopt/envs/searchqa/dataloader.py new file mode 100644 index 0000000..2dc1c1e --- /dev/null +++ b/skillopt/envs/searchqa/dataloader.py @@ -0,0 +1,42 @@ +"""SearchQA task dataloader.""" +from __future__ import annotations + +import json + +from skillopt.datasets.base import SplitDataLoader + + +# ── Raw data loading utilities (for preprocessing / standalone eval) ───── + +def _load_items(path: str) -> list[dict]: + """Load items from JSON or JSONL file.""" + with open(path) as f: + content = f.read().strip() + try: + data = json.loads(content) + if isinstance(data, list): + return data + if isinstance(data, dict): + return data.get("data") or list(data.values()) + except json.JSONDecodeError: + pass + + items = [] + for line in content.splitlines(): + line = line.strip() + if line: + items.append(json.loads(line)) + return items + + +# ── Dataloader ─────────────────────────────────────────────────────────── + +class SearchQADataLoader(SplitDataLoader): + """SearchQA dataloader. + + Each split directory (train/, val/, test/) contains a .json file — + a JSON array of question items. + """ + + def load_raw_items(self, data_path: str) -> list[dict]: + return _load_items(data_path) diff --git a/skillopt/envs/searchqa/evaluator.py b/skillopt/envs/searchqa/evaluator.py new file mode 100644 index 0000000..8c6c488 --- /dev/null +++ b/skillopt/envs/searchqa/evaluator.py @@ -0,0 +1,100 @@ +"""SearchQA evaluation metrics: Exact Match, F1, and Substring Match. + +Normalization follows the SQuAD convention: + - lowercase + - remove punctuation + - remove articles (a, an, the) + - collapse whitespace + +Answer extraction looks for ... XML tags, +falling back to the last non-empty line of the response. +""" +from __future__ import annotations + +import re +import string +from collections import Counter + + +def normalize_answer(s: str) -> str: + """Normalize answer string (SQuAD convention).""" + s = s.lower() + s = "".join(ch for ch in s if ch not in string.punctuation) + s = re.sub(r"\b(a|an|the)\b", " ", s) + s = " ".join(s.split()) + return s.strip() + + +def extract_answer(text: str) -> str: + """Extract answer from ... tags. + + Fallback: last non-empty line, then full response stripped. + """ + matches = re.findall(r"(.*?)", text, re.DOTALL | re.IGNORECASE) + if matches: + return matches[-1].strip() + lines = [ln.strip() for ln in text.strip().splitlines() if ln.strip()] + if lines: + return lines[-1] + return text.strip() + + +def exact_match(prediction: str, gold_answers: list[str]) -> float: + norm_pred = normalize_answer(prediction) + for gold in gold_answers: + if normalize_answer(gold) == norm_pred: + return 1.0 + return 0.0 + + +def f1_score(prediction: str, gold_answers: list[str]) -> float: + """Token-level F1 (SQuAD-style), max across all gold answers.""" + norm_pred = normalize_answer(prediction) + pred_tokens = norm_pred.split() + + if not pred_tokens: + for gold in gold_answers: + if not normalize_answer(gold).split(): + return 1.0 + return 0.0 + + best_f1 = 0.0 + for gold in gold_answers: + gold_tokens = normalize_answer(gold).split() + if not gold_tokens: + continue + common = Counter(pred_tokens) & Counter(gold_tokens) + n_common = sum(common.values()) + if n_common == 0: + continue + precision = n_common / len(pred_tokens) + recall = n_common / len(gold_tokens) + f1 = 2 * precision * recall / (precision + recall) + best_f1 = max(best_f1, f1) + + return best_f1 + + +def sub_em(prediction: str, gold_answers: list[str]) -> float: + """1.0 if any normalized gold is a substring of prediction, or vice versa.""" + norm_pred = normalize_answer(prediction) + for gold in gold_answers: + norm_gold = normalize_answer(gold) + if norm_gold in norm_pred or norm_pred in norm_gold: + return 1.0 + return 0.0 + + +def evaluate(prediction_text: str, gold_answers: list[str]) -> dict: + """Evaluate a single QA prediction against gold answers. + + Returns dict with: em, f1, sub_em, predicted_answer, gold_answers. + """ + answer = extract_answer(prediction_text) + return { + "em": exact_match(answer, gold_answers), + "f1": f1_score(answer, gold_answers), + "sub_em": sub_em(answer, gold_answers), + "predicted_answer": answer, + "gold_answers": gold_answers, + } diff --git a/skillopt/envs/searchqa/prompts/analyst_error.md b/skillopt/envs/searchqa/prompts/analyst_error.md new file mode 100644 index 0000000..a60e73d --- /dev/null +++ b/skillopt/envs/searchqa/prompts/analyst_error.md @@ -0,0 +1,46 @@ +You are an expert failure-analysis agent for question answering tasks. + +You will be given MULTIPLE failed QA agent responses from a single minibatch +and the current skill document. Each trajectory includes the agent's response +and an evaluation result showing the predicted answer vs. the gold answer(s). + +Your job is to identify the most important COMMON failure patterns across +the batch and propose a concise set of skill edits. + +## Failure Type Categories +- **rule_missing**: the skill lacks a relevant rule for this type of question +- **rule_wrong**: an existing skill rule is misleading or incorrect +- **rule_ignored**: the skill has the right rule but the agent did not follow it +- **answer_format**: the agent found the right information but formatted it incorrectly +- **other**: none of the above + +## Analysis Process +1. Read ALL failed trajectories in the minibatch. +2. Carefully compare each predicted answer against the gold answer(s) — + understand exactly WHY the Exact Match failed. +3. Identify the most prevalent, systematic failure patterns across them. +4. For each pattern, classify its failure type. +5. Propose skill edits that address the COMMON patterns — not individual edge cases. +6. Edits must be generalizable; do not hardcode question-specific values. +7. Only patch gaps in the skill — do not duplicate existing content. + +You will be told the maximum number of edits (the budget L). Produce AT MOST L edits, +focusing on the highest-impact patterns. You may produce fewer if warranted. + +Respond ONLY with a valid JSON object (no markdown fences, no extra text): +{ + "batch_size": , + "failure_summary": [ + {"failure_type": "", "count": , "description": ""} + ], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +Only include edits that are needed. "edits" can be an empty list if no patch is warranted. diff --git a/skillopt/envs/searchqa/prompts/analyst_success.md b/skillopt/envs/searchqa/prompts/analyst_success.md new file mode 100644 index 0000000..6476d94 --- /dev/null +++ b/skillopt/envs/searchqa/prompts/analyst_success.md @@ -0,0 +1,32 @@ +You are an expert success-pattern analyst for AI question answering agents. + +You will be given MULTIPLE successful QA agent responses from a single minibatch +and the current skill document. Your job is to identify generalizable behavior +patterns that are COMMON across the batch and worth encoding in the skill. + +## Rules +- Only propose patches for patterns NOT already covered in the skill. +- Focus on patterns that appear across MULTIPLE trajectories in the batch. +- Be concise. Patterns must generalize beyond specific questions. +- Prefer reinforcing existing sections over adding new top-level sections. +- If the agents' success involved a smart reading strategy or disambiguation + approach, consider reinforcing that in the patch. + +You will be told the maximum number of edits (the budget L). Produce AT MOST L edits, +focusing on the most broadly applicable patterns. You may produce fewer if warranted. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "success_patterns": ["", ""], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +"edits" may be empty if the skill already covers all observed patterns. diff --git a/skillopt/envs/searchqa/prompts/rollout_system.md b/skillopt/envs/searchqa/prompts/rollout_system.md new file mode 100644 index 0000000..1befe4e --- /dev/null +++ b/skillopt/envs/searchqa/prompts/rollout_system.md @@ -0,0 +1,13 @@ +You are an expert question answering agent. + +{skill_section}## Task Format +You will receive a CONTEXT containing document passages and a QUESTION. +Read the context carefully and answer the question based on the information provided. + +## Answer Format +Think step by step, then provide your final answer inside ... tags. +Keep your answer concise — typically a few words or a short phrase. +Do not repeat the question. Do not include unnecessary explanation in the answer tags. + +Example: +Abraham Lincoln diff --git a/skillopt/envs/searchqa/reflect.py b/skillopt/envs/searchqa/reflect.py new file mode 100644 index 0000000..7a99207 --- /dev/null +++ b/skillopt/envs/searchqa/reflect.py @@ -0,0 +1,4 @@ +"""SearchQA Reflect stage. + +Prompts are now loaded from .md files by the base adapter. +""" diff --git a/skillopt/envs/searchqa/rollout.py b/skillopt/envs/searchqa/rollout.py new file mode 100644 index 0000000..83165b5 --- /dev/null +++ b/skillopt/envs/searchqa/rollout.py @@ -0,0 +1,494 @@ +"""SearchQA rollout — single-turn QA agent + batch execution. + +The QA agent receives a skill document, question, and context passages, +then produces an answer in ... tags. + +Public API +---------- +- :func:`process_one` — run + evaluate one QA item +- :func:`run_batch` — parallel execution of a list of items +""" +from __future__ import annotations + +import json +import os +import time +from collections import Counter +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait + +from skillopt.envs.searchqa.evaluator import evaluate +from skillopt.model import chat_target, is_target_exec_backend +from skillopt.model.codex_harness import prepare_workspace, render_skill_md, run_target_exec +from skillopt.prompts import load_prompt + +# ── Prompt templates ───────────────────────────────────────────────────────── + +_MAX_CONTEXT_CHARS = 6000 + + +def _raise_on_systemic_failure(results: list[dict]) -> None: + """Abort when all rollout rows failed before any agent response.""" + if not results or not all(row.get("agent_ok") is False for row in results): + return + reasons = Counter(str(row.get("fail_reason") or "unknown error") for row in results) + common_reason, count = reasons.most_common(1)[0] + raise RuntimeError( + f"SearchQA rollout failed for all {len(results)} items before an agent " + f"response ({count}x): {common_reason}" + ) + + +def _truncate_context(context: str, max_chars: int = _MAX_CONTEXT_CHARS) -> str: + """Truncate context at [DOC] boundaries to stay within budget.""" + if len(context) <= max_chars: + return context + docs = context.split("[DOC]") + result = "" + for doc in docs: + candidate = result + "[DOC]" + doc if result else doc + if len(candidate) > max_chars: + break + result = candidate + if not result: + result = context[:max_chars] + "\n...[truncated]" + return result + + +def _build_system(skill_content: str) -> str: + if skill_content.strip(): + skill_section = f"## Skill\n{skill_content.strip()}\n\n" + else: + skill_section = "" + return load_prompt("rollout_system", env="searchqa").format(skill_section=skill_section) + + +def _build_user( + question: str, + context: str, + *, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", +) -> str: + context = _truncate_context(context) + parts = [ + f"## Context\n{context}", + f"## Question\n{question}", + ] + if diagnostic_trace_context.strip(): + parts.append( + "## Previous Codex Trace Snapshot\n" + "This is a partial transcript from an earlier attempt. Use it as your current reasoning context.\n\n" + f"{diagnostic_trace_context.strip()}" + ) + if diagnostic_mode and diagnostic_instruction.strip(): + parts.append(f"## Training Readout\n{diagnostic_instruction.strip()}") + return "\n\n".join(parts) + + +def _build_codex_skill(skill_content: str) -> str: + return render_skill_md( + skill_content, + description="Dynamic ReflACT skill for solving the current SearchQA example.", + preamble=( + "Use this skill when solving the current SearchQA task.\n" + "Read the provided context carefully, ground the answer in that context,\n" + "and return the final answer inside ...." + ), + ) + + +def _run_codex_once( + *, + pred_dir: str, + skill_content: str, + question: str, + context: str, + model: str, + timeout: int, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", + previous_response: str = "", +) -> tuple[str, str, str, str]: + user = _build_user( + question, + context, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + task_parts = [user] + if previous_response: + task_parts.append( + "## Previous Attempt\n" + f"{previous_response}\n\n" + "Review it against the same context and question. If needed, correct it." + ) + task_text = "\n\n".join(task_parts) + skill_md = _build_codex_skill(skill_content) + work_dir = os.path.join(pred_dir, "codex_exec") + prepare_workspace( + work_dir=work_dir, + skill_md=skill_md, + task_text=task_text, + ) + prompt = ( + "Use the `skillopt-target` skill available in this workspace.\n" + "Read `task.md` and answer the SearchQA question.\n" + "Return the final answer inside ...." + ) + final_message, raw = run_target_exec( + work_dir=work_dir, + prompt=prompt, + model=model, + timeout=timeout, + ) + return final_message or raw, raw, skill_md, task_text + + +# ── Single-item execution ─────────────────────────────────────────────────── + + +def process_one( + item: dict, + out_root: str, + skill_content: str, + max_turns: int = 1, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", + exec_timeout: int = 120, + max_completion_tokens: int = 16384, +) -> dict: + """Process a single QA item: run agent + evaluate. + + Parameters + ---------- + item : dict + Must have keys: ``id``, ``question``, ``context``, ``answers``. + out_root : str + Output directory (predictions saved under ``predictions//``). + skill_content : str + Current skill document text. + max_turns : int + Max reasoning turns (1 = single-turn QA). + + Returns + ------- + dict + Result with ``hard`` (EM as int), ``soft`` (F1), etc. + """ + item_id = str(item["id"]) + question = item["question"] + context = item.get("context", "") + gold_answers = item.get("answers", []) + + result = { + "id": item_id, + "question": question, + "em": 0.0, + "f1": 0.0, + "sub_em": 0.0, + "hard": 0, + "soft": 0.0, + "predicted_answer": "", + "gold_answers": gold_answers, + "response": "", + "fail_reason": "", + "agent_ok": False, + "n_turns": 0, + } + + try: + pred_dir = os.path.join(out_root, "predictions", item_id) + os.makedirs(pred_dir, exist_ok=True) + + if is_target_exec_backend(): + from skillopt.model import azure_openai as _llm + + conversation: list[dict] = [] + response = "" + system = "" + user = "" + for turn in range(max_turns): + response, raw, system, user = _run_codex_once( + pred_dir=pred_dir, + skill_content=skill_content, + question=question, + context=context, + model=_llm.TARGET_DEPLOYMENT, + timeout=exec_timeout, + diagnostic_mode=diagnostic_mode if turn == 0 else False, + diagnostic_instruction=diagnostic_instruction if turn == 0 else "", + diagnostic_trace_context=diagnostic_trace_context if turn == 0 else "", + previous_response=response if turn > 0 else "", + ) + conversation.append({"type": "message", "turn": turn + 1, "content": response}) + if turn > 0 and "" in response.lower(): + break + + result["response"] = response + result["agent_ok"] = True + result["n_turns"] = len(conversation) + + with open(os.path.join(pred_dir, "target_system_prompt.txt"), "w") as f: + f.write(system) + with open(os.path.join(pred_dir, "target_user_prompt.txt"), "w") as f: + f.write(user) + with open(os.path.join(pred_dir, "conversation.json"), "w") as f: + json.dump(conversation, f, ensure_ascii=False, indent=2) + + eval_result = evaluate(response, gold_answers) + result["em"] = eval_result["em"] + result["f1"] = eval_result["f1"] + result["sub_em"] = eval_result["sub_em"] + result["predicted_answer"] = eval_result["predicted_answer"] + result["hard"] = int(eval_result["em"]) + result["soft"] = eval_result["f1"] + if eval_result["em"] < 1.0: + result["fail_reason"] = ( + f"EM=0: predicted '{eval_result['predicted_answer']}' " + f"but expected {gold_answers}" + ) + eval_detail = ( + f"[EVALUATION RESULT]\n" + f"Question: {question}\n" + f"Predicted answer: {eval_result['predicted_answer']!r}\n" + f"Gold answers: {gold_answers!r}\n" + f"Exact Match: {eval_result['em']}\n" + f"F1: {eval_result['f1']:.4f}" + ) + conversation.append({"role": "system", "content": eval_detail}) + with open(os.path.join(pred_dir, "conversation.json"), "w") as f: + json.dump(conversation, f, ensure_ascii=False, indent=2) + return result + + system = _build_system(skill_content) + user = _build_user( + question, + context, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + + conversation: list[dict] = [] + response = "" + + for turn in range(max_turns): + if turn == 0: + resp_text, _ = chat_target( + system=system, user=user, + max_completion_tokens=max_completion_tokens, + retries=5, stage="rollout", + timeout=exec_timeout, + ) + else: + refinement = ( + f"Your previous answer was:\n{response}\n\n" + f"Review it against the context and question. " + f"If correct, repeat it. If wrong, provide a corrected answer.\n" + f"Use ... tags for your final answer." + ) + resp_text, _ = chat_target( + system=system, user=refinement, + max_completion_tokens=max_completion_tokens, + retries=5, stage="rollout", + timeout=exec_timeout, + ) + + response = resp_text + conversation.append({"type": "message", "turn": turn + 1, "content": resp_text}) + + if turn > 0 and "" in resp_text.lower(): + break + + result["response"] = response + result["agent_ok"] = True + result["n_turns"] = len(conversation) + + # Save conversation + with open(os.path.join(pred_dir, "target_system_prompt.txt"), "w") as f: + f.write(system) + with open(os.path.join(pred_dir, "target_user_prompt.txt"), "w") as f: + f.write(user) + with open(os.path.join(pred_dir, "conversation.json"), "w") as f: + json.dump(conversation, f, ensure_ascii=False, indent=2) + + # Evaluate + eval_result = evaluate(response, gold_answers) + result["em"] = eval_result["em"] + result["f1"] = eval_result["f1"] + result["sub_em"] = eval_result["sub_em"] + result["predicted_answer"] = eval_result["predicted_answer"] + result["hard"] = int(eval_result["em"]) + result["soft"] = eval_result["f1"] + + if eval_result["em"] < 1.0: + result["fail_reason"] = ( + f"EM=0: predicted '{eval_result['predicted_answer']}' " + f"but expected {gold_answers}" + ) + + # Append eval details to conversation for the analyst + eval_detail = ( + f"[EVALUATION RESULT]\n" + f"Question: {question}\n" + f"Predicted answer: {eval_result['predicted_answer']!r}\n" + f"Gold answers: {gold_answers!r}\n" + f"Exact Match: {eval_result['em']}\n" + f"F1: {eval_result['f1']:.4f}" + ) + conversation.append({ + "role": "system", + "content": eval_detail, + }) + # Re-save enriched conversation + with open(os.path.join(pred_dir, "conversation.json"), "w") as f: + json.dump(conversation, f, ensure_ascii=False, indent=2) + + except Exception as e: # noqa: BLE001 + result["fail_reason"] = f"error: {e}" + + return result + + +# ── Batch execution ────────────────────────────────────────────────────────── + + +def run_batch( + items: list[dict], + out_root: str, + skill_content: str, + max_turns: int = 1, + exec_timeout: int = 120, + workers: int = 64, + max_completion_tokens: int = 16384, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context_by_id: dict[str, str] | None = None, + task_timeout: int = 600, +) -> list[dict]: + """Run QA agent on all items with ThreadPoolExecutor. Resume-aware.""" + task_timeout = max(int(task_timeout), int(exec_timeout) + 60) + results_path = os.path.join(out_root, "results.jsonl") + os.makedirs(out_root, exist_ok=True) + + # Resume: load already-done + done_ids: set[str] = set() + existing: list[dict] = [] + if os.path.exists(results_path): + with open(results_path) as f: + for line in f: + try: + r = json.loads(line) + done_ids.add(str(r["id"])) + existing.append(r) + except Exception: + pass + + pending = [it for it in items if str(it["id"]) not in done_ids] + if not pending: + _raise_on_systemic_failure(existing) + return existing + + total = len(existing) + len(pending) + completed = len(existing) + correct_count = sum(1 for r in existing if r.get("hard", 0)) + if existing: + print(f" [rollout] resuming: {completed}/{total} already done", flush=True) + + results = list(existing) + + def _timeout_result(item: dict) -> dict: + return { + "id": str(item["id"]), + "question": item.get("question", ""), + "task_description": item.get("question", ""), + "task_type": item.get("task_type") or "searchqa", + "hard": 0, + "soft": 0.0, + "predicted_answer": "", + "response": "", + "fail_reason": f"task-timeout-{task_timeout}s", + "agent_ok": False, + "n_turns": 0, + "gold_answer": item.get("answers", []), + "phase": "timeout", + } + + def _error_result(item: dict, exc: Exception) -> dict: + row = _timeout_result(item) + row["phase"] = "error" + row["fail_reason"] = f"unexpected: {type(exc).__name__}: {exc}" + return row + + started_at: dict[str, float] = {} + + def _run_one(item: dict) -> dict: + started_at[str(item["id"])] = time.time() + return process_one( + item, + out_root, + skill_content, + max_turns, + diagnostic_mode, + diagnostic_instruction, + (diagnostic_trace_context_by_id or {}).get(str(item["id"]), ""), + exec_timeout, + max_completion_tokens, + ) + + with open(results_path, "a") as outf: + ex = ThreadPoolExecutor(max_workers=workers) + try: + futs = {ex.submit(_run_one, it): it for it in pending} + pending_futs = set(futs) + while pending_futs: + done, _ = wait(pending_futs, timeout=5, return_when=FIRST_COMPLETED) + now = time.time() + timed_out = [ + fut for fut in pending_futs - done + if str(futs[fut]["id"]) in started_at + and now - started_at[str(futs[fut]["id"])] >= task_timeout + ] + for fut in done: + pending_futs.remove(fut) + item = futs[fut] + try: + res = fut.result() + except Exception as exc: # noqa: BLE001 + res = _error_result(item, exc) + results.append(res) + completed += 1 + if res.get("hard", 0): + correct_count += 1 + acc = correct_count / completed if completed else 0 + print( + f" [rollout] {completed}/{total} " + f"(acc={acc:.3f}) id={res['id']} " + f"hard={res.get('hard', '?')}", + flush=True, + ) + outf.write(json.dumps(res, ensure_ascii=False) + "\n") + outf.flush() + for fut in timed_out: + pending_futs.remove(fut) + fut.cancel() + res = _timeout_result(futs[fut]) + results.append(res) + completed += 1 + acc = correct_count / completed if completed else 0 + print( + f" [rollout] {completed}/{total} " + f"(acc={acc:.3f}) id={res['id']} TIMEOUT", + flush=True, + ) + outf.write(json.dumps(res, ensure_ascii=False) + "\n") + outf.flush() + finally: + ex.shutdown(wait=False, cancel_futures=True) + + _raise_on_systemic_failure(results) + return results diff --git a/skillopt/envs/searchqa/skills/initial.md b/skillopt/envs/searchqa/skills/initial.md new file mode 100644 index 0000000..6bc64d8 --- /dev/null +++ b/skillopt/envs/searchqa/skills/initial.md @@ -0,0 +1,3 @@ +# Question Answering Skill + +(No learned rules yet. Rules will be added through the reflection process.) diff --git a/skillopt/envs/spreadsheetbench/__init__.py b/skillopt/envs/spreadsheetbench/__init__.py new file mode 100644 index 0000000..3db374b --- /dev/null +++ b/skillopt/envs/spreadsheetbench/__init__.py @@ -0,0 +1,5 @@ +"""SpreadsheetBench environment adapter for ReflACT.""" + +from skillopt.envs.spreadsheetbench.adapter import SpreadsheetBenchAdapter + +__all__ = ["SpreadsheetBenchAdapter"] diff --git a/skillopt/envs/spreadsheetbench/adapter.py b/skillopt/envs/spreadsheetbench/adapter.py new file mode 100644 index 0000000..16e7856 --- /dev/null +++ b/skillopt/envs/spreadsheetbench/adapter.py @@ -0,0 +1,159 @@ +"""SpreadsheetBench environment adapter for ReflACT. + +Connects the ReflACT training loop to SpreadsheetBench by implementing +:class:`~skillopt.envs.base.EnvAdapter`. +""" +from __future__ import annotations + +import json +import os + +from skillopt.datasets.base import BatchSpec +from skillopt.envs.base import EnvAdapter +from skillopt.envs.spreadsheetbench.dataloader import SpreadsheetBenchDataLoader +from skillopt.envs.spreadsheetbench.rollout import ( + process_one, + run_spreadsheet_batch, + run_spreadsheet_batch_codegen, +) +from skillopt.model import get_target_backend, is_target_exec_backend + + +# Task types used for per-category breakdowns +TASK_TYPES = ["cell_level", "sheet_level"] + + +class SpreadsheetBenchAdapter(EnvAdapter): + """SpreadsheetBench environment adapter.""" + + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "ratio", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + data_root: str = "", + mode: str = "single", + max_turns: int = 30, + exec_timeout: int = 600, + workers: int = 64, + analyst_workers: int = 16, + failure_only: bool = False, + minibatch_size: int = 8, + edit_budget: int = 4, + seed: int = 42, + max_completion_tokens: int = 16384, + ) -> None: + self.data_root = data_root + self.mode = mode # "single", "multi", or "react" + self.max_turns = max_turns + self.exec_timeout = exec_timeout + self.workers = workers + self.max_completion_tokens = int(max_completion_tokens) + self.analyst_workers = analyst_workers + self.failure_only = failure_only + self.minibatch_size = minibatch_size + self.edit_budget = edit_budget + self.dataloader = SpreadsheetBenchDataLoader( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + data_root=data_root, + seed=seed, + ) + + def setup(self, cfg: dict) -> None: + super().setup(cfg) + if is_target_exec_backend() and self.mode != "single": + raise NotImplementedError( + "Exec target backends are currently supported only for SpreadsheetBench mode=single." + ) + self.dataloader.setup(cfg) + + def get_dataloader(self): + return self.dataloader + + def build_env_from_batch(self, batch: BatchSpec, **kwargs): + return list(batch.payload or []) + + def build_train_env(self, batch_size: int, seed: int, **kwargs): + batch = self.dataloader.build_train_batch(batch_size=batch_size, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def build_eval_env(self, env_num: int, split: str, seed: int, **kwargs): + batch = self.dataloader.build_eval_batch(env_num=env_num, split=split, seed=seed, **kwargs) + return self.build_env_from_batch(batch, **kwargs) + + def rollout( + self, + env_manager, + skill_content: str, + out_dir: str, + **kwargs, + ) -> list[dict]: + """Run agent on all items and return results. + + Dispatches based on ``self.mode``: + - ``"single"`` / ``"multi"``: codegen agent (no tool-call) + - ``"react"``: ReAct agent with tool-call (legacy) + """ + items = env_manager # For static datasets, env_manager is a list of items + results_path = os.path.join(out_dir, "results.jsonl") + os.makedirs(out_dir, exist_ok=True) + + # Resume support + if os.path.exists(results_path): + existing: list[dict] = [] + with open(results_path) as f: + for line in f: + try: + existing.append(json.loads(line)) + except Exception: + pass + if existing: + return existing + + if self.mode in ("single", "multi"): + results = run_spreadsheet_batch_codegen( + items=items, + data_root=self.data_root, + out_root=out_dir, + skill_content=skill_content, + mode=self.mode, + max_turns=self.max_turns, + max_completion_tokens=self.max_completion_tokens, + max_api_workers=self.workers, + task_timeout=self.exec_timeout, + use_eval_feedback=kwargs.get("use_eval_feedback", False), + diagnostic_mode=kwargs.get("diagnostic_mode", False), + diagnostic_instruction=kwargs.get("diagnostic_instruction", ""), + diagnostic_trace_context_by_id=kwargs.get("diagnostic_trace_context_by_id"), + ) + else: + results = run_spreadsheet_batch( + items=items, + data_root=self.data_root, + out_root=out_dir, + skill_content=skill_content, + max_turns=self.max_turns, + max_completion_tokens=self.max_completion_tokens, + max_api_workers=self.workers, + task_timeout=max(600, int(self.exec_timeout) + 60), + diagnostic_mode=kwargs.get("diagnostic_mode", False), + diagnostic_instruction=kwargs.get("diagnostic_instruction", ""), + diagnostic_trace_context_by_id=kwargs.get("diagnostic_trace_context_by_id"), + ) + + with open(results_path, "w") as f: + for r in results: + f.write(json.dumps(r, ensure_ascii=False) + "\n") + + return results + + def get_task_types(self) -> list[str]: + return list(TASK_TYPES) diff --git a/skillopt/envs/spreadsheetbench/codegen_agent.py b/skillopt/envs/spreadsheetbench/codegen_agent.py new file mode 100644 index 0000000..a4948f1 --- /dev/null +++ b/skillopt/envs/spreadsheetbench/codegen_agent.py @@ -0,0 +1,731 @@ +"""Codegen agent for SpreadsheetBench — no tool-call, pure code generation. + +Two modes: + - **single**: One LLM call → extract ```python``` block → done. + - **multi**: Up to max_turns LLM calls; after each, execute code and + feed errors back for correction. + +This matches the official SpreadsheetBench evaluation setting (LLM generates +a Python code block, no function-calling / tool-use). +""" +from __future__ import annotations + +import json +import os +import random +import signal +import time + +import openpyxl + + +# ── Timeout helper ────────────────────────────────────────────────────────── + +class TaskTimeout(Exception): + """Raised when a task exceeds its time budget.""" + + +def _timeout_handler(signum, frame): + raise TaskTimeout("Task timed out") + +from skillopt.model.azure_openai import ( + get_reasoning_effort, + get_target_client, + _needs_responses_api, + tracker, +) +from skillopt.model import get_codex_exec_config, get_target_backend, is_target_exec_backend +from skillopt.model.codex_harness import prepare_workspace, render_skill_md, run_target_exec +from skillopt.prompts import load_prompt +from skillopt.envs.spreadsheetbench.executor import run_generated_code +from skillopt.envs.spreadsheetbench.evaluator import evaluate + + +# ── Eval feedback helper (no golden value leakage) ───────────────────────── + +def _build_eval_feedback(verify_report: str) -> str: + """Build Target feedback from a verify report, hiding expected values. + + The verify report contains lines like: + Sheet1!D2: got=None, expected=0 ✗ + Sheet1!D10: got=None, expected=None ✓ + + We strip the ``expected=...`` part so the Target sees only its own + output and whether each cell is correct or wrong. + """ + import re + wrong_lines = [] + n_correct = 0 + for raw_line in verify_report.splitlines(): + raw_line = raw_line.strip() + if not raw_line: + continue + # Match enrichment lines like " Sheet1!D2: got=None, expected=0 ✗" + m = re.match( + r"(\S+!?\w+):\s*got=(.+?),\s*expected=.+?\s*(✓|✗)$", + raw_line, + ) + if m: + cell, got_val, mark = m.groups() + if mark == "✗": + wrong_lines.append(f" {cell}: your output = {got_val} (WRONG)") + else: + n_correct += 1 + lines = ["Your code executed successfully but produced incorrect results.", + "The following cells have wrong values:"] + lines.extend(wrong_lines) + if n_correct: + lines.append(f" ({n_correct} other cells are correct.)") + lines.append( + "\nPlease analyze the spreadsheet data more carefully and fix the code. " + "Return a complete corrected Python script inside a ```python``` block." + ) + return "\n".join(lines) + + +# ── Workbook preview (same as official prompt.py) ──────────────────────────── + +def _preview_workbook(path: str, max_rows: int = 5, max_cols: int = 20) -> str: + """Generate a text preview of the first few rows of each sheet.""" + wb = openpyxl.load_workbook(path, data_only=False) + chunks: list[str] = [] + for sheet_name in wb.sheetnames: + ws = wb[sheet_name] + chunks.append( + f"## Sheet: {sheet_name} " + f"(dim={ws.dimensions}, max_row={ws.max_row}, max_col={ws.max_column})" + ) + for row in ws.iter_rows( + min_row=1, + max_row=min(ws.max_row, max_rows), + max_col=min(ws.max_column, max_cols), + values_only=False, + ): + cells = [] + for cell in row: + v = cell.value + if v is None: + cells.append(f"{cell.coordinate}=") + else: + s = str(v) + if len(s) > 40: + s = s[:37] + "..." + cells.append(f"{cell.coordinate}={s}") + chunks.append(" | ".join(cells)) + if ws.max_row > max_rows: + chunks.append(f"... ({ws.max_row - max_rows} more rows)") + chunks.append("") + wb.close() + return "\n".join(chunks) + + +# ── Code extraction (same as official prompt.py) ──────────────────────────── + +def extract_code(text: str) -> str: + """Extract the first ```python``` fenced code block from LLM output.""" + if "```" not in text: + return text.strip() + start = text.find("```") + nl = text.find("\n", start) + end = text.find("```", nl + 1) + if nl == -1 or end == -1: + return text.strip() + return text[nl + 1 : end].strip() + + +# ── Prompt construction (official SpreadsheetBench prompts) ───────────────── + + +def _build_system(skill_content: str) -> str: + base = load_prompt("codegen_system", env="spreadsheetbench") + if skill_content.strip(): + base += f"\n\n## Skill\n{skill_content.strip()}" + return base + + +def _build_user( + instruction: str, + input_xlsx: str, + instruction_type: str = "", + answer_position: str = "", + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", +) -> str: + try: + preview = _preview_workbook(input_xlsx) + except Exception as e: # noqa: BLE001 + preview = f"(failed to preview workbook: {e})" + extra = "" + if instruction_type: + extra += f"\nInstruction type: {instruction_type}" + if answer_position: + extra += f"\nExpected answer position: {answer_position}" + task_suffix = "Return only a ```python``` code block." + diagnostic = "" + if diagnostic_mode and diagnostic_instruction.strip(): + task_suffix = ( + "First provide a short diagnostic readout that follows the training " + "instruction below, then return a single complete ```python``` code block." + ) + diagnostic = f"\n\n# Training readout\n{diagnostic_instruction.strip()}" + prefix = "" + if diagnostic_trace_context.strip(): + prefix = ( + "# Previous Codex Trace Snapshot\n" + "This is a partial transcript from an earlier attempt. Use it as your current reasoning context.\n\n" + f"{diagnostic_trace_context.strip()}\n\n" + ) + return ( + f"{prefix}" + f"# Instruction\n{instruction}\n{extra}\n\n" + f"# Input spreadsheet preview\n{preview}\n\n" + "# Task\n" + "Write a Python script that reads the workbook from the variable `INPUT_PATH`, " + "applies the instruction, and writes the modified workbook to `OUTPUT_PATH`. " + "Preserve all other cells unchanged. " + "The preview may be truncated — do not hardcode row counts or assume the data ends at the last previewed row; " + "iterate over all actual rows in the workbook instead. " + f"{task_suffix}" + f"{diagnostic}" + ) + + +# ── LLM call with retry ──────────────────────────────────────────────────── + +def _llm_call_with_retry(call_fn, *, retries: int = 5, timeout: int | None = 120): + """Wrap an LLM API call with retry and per-call timeout.""" + last_err = None + for attempt in range(retries): + try: + return call_fn(timeout=timeout) + except Exception as e: # noqa: BLE001 + last_err = e + sleep = min(2 ** attempt + random.random(), 60) + time.sleep(sleep) + raise RuntimeError(f"LLM call failed after {retries} retries: {last_err}") + + +def _get_deployment() -> str: + from skillopt.model import azure_openai as _llm + return _llm.TARGET_DEPLOYMENT + + +def _build_codex_skill(skill_content: str) -> str: + return render_skill_md( + skill_content, + description="Dynamic ReflACT skill for solving the current SpreadsheetBench task.", + preamble=( + "Use this skill when solving the current SpreadsheetBench task in this workspace.\n" + "Write a single self-contained Python solution to `solution.py`.\n" + "The solution must operate on the provided `INPUT_PATH` and `OUTPUT_PATH` variables.\n" + "You may inspect `input.xlsx` and run `python run_solution.py` to validate locally,\n" + "but do not hardcode values from the preview or from one specific workbook." + ), + ) + + +def _build_codex_task( + instruction: str, + input_xlsx: str, + instruction_type: str, + answer_position: str, + *, + diagnostic_mode: bool, + diagnostic_instruction: str, + diagnostic_trace_context: str, +) -> str: + prompt = _build_user( + instruction, + input_xlsx, + instruction_type, + answer_position, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + return ( + f"{prompt}\n\n" + "## Codex Harness Task\n" + "- Read `.agents/skills/skillopt-target/SKILL.md` before writing code; do not call a Skill tool.\n" + "- Read and optionally inspect `input.xlsx` in this workspace.\n" + "- Write the final Python solution to `solution.py`.\n" + "- The script should use the provided `INPUT_PATH` and `OUTPUT_PATH` variables.\n" + "- If you want to validate locally, run `python run_solution.py`.\n" + "- Do not return a code fence as the primary artifact; the source of truth is `solution.py`.\n" + ) + + +def _build_codex_driver() -> str: + return ( + "import pathlib\n" + "import re\n" + "import sys\n" + "import traceback\n\n" + 'INPUT_PATH = "input.xlsx"\n' + 'OUTPUT_PATH = "output.xlsx"\n' + "code = pathlib.Path('solution.py').read_text(encoding='utf-8')\n" + "code = re.sub(r'^\\s*(INPUT_PATH|OUTPUT_PATH)\\s*=\\s*.+$', '', code, flags=re.MULTILINE)\n" + "globals_dict = {'__name__': '__main__', 'INPUT_PATH': INPUT_PATH, 'OUTPUT_PATH': OUTPUT_PATH}\n" + "try:\n" + " exec(compile(code, 'solution.py', 'exec'), globals_dict, globals_dict)\n" + "except Exception:\n" + " traceback.print_exc()\n" + " sys.exit(2)\n" + ) + + +def _prepare_codex_workspace( + *, + instruction: str, + input_xlsx: str, + output_path: str, + instruction_type: str, + answer_position: str, + skill_content: str, + diagnostic_mode: bool, + diagnostic_instruction: str, + diagnostic_trace_context: str, + workspace_name: str = "codex_single", +) -> tuple[str, str, str, str]: + task_out_dir = os.path.dirname(output_path) + work_dir = os.path.join(task_out_dir, workspace_name) + skill_md = _build_codex_skill(skill_content) + task_md = _build_codex_task( + instruction, + input_xlsx, + instruction_type, + answer_position, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + prompt = ( + "Read `.agents/skills/skillopt-target/SKILL.md` directly; do not call a Skill tool.\n" + "Read `task.md`, inspect `input.xlsx` if useful, and write the final solution to `solution.py`.\n" + "You may run `python run_solution.py` to validate the script locally.\n" + "In your final response, briefly confirm whether `solution.py` was written and summarize the approach." + ) + prepare_workspace( + work_dir=work_dir, + skill_md=skill_md, + task_text=task_md, + extra_files={"run_solution.py": _build_codex_driver()}, + copy_files=[(input_xlsx, "input.xlsx")], + ) + + return work_dir, skill_md, task_md, prompt + + +def _run_exec_backend( + *, + work_dir: str, + prompt: str, + model: str, + timeout: int, +) -> tuple[str, str]: + return run_target_exec( + work_dir=work_dir, + prompt=prompt, + model=model, + timeout=timeout, + allow_file_edits=True, + ) + + +# ── Chat (no tools) ──────────────────────────────────────────────────────── + +def _chat_call( + client, + deployment: str, + messages: list[dict], + max_output_tokens: int, + llm_timeout: int | None = 120, +) -> str: + """Single LLM call, no tools. Returns raw text.""" + reasoning_effort = get_reasoning_effort() + if _needs_responses_api(deployment): + # Responses API + system = "" + api_input = [] + for m in messages: + if m["role"] == "system": + system = m["content"] + else: + api_input.append({"role": m["role"], "content": m["content"]}) + resp = _llm_call_with_retry(lambda timeout: client.responses.create( + model=deployment, + instructions=system, + input=api_input, + max_output_tokens=max_output_tokens, + **({"reasoning": {"effort": reasoning_effort}} if reasoning_effort else {}), + timeout=timeout, + ), timeout=llm_timeout) + if hasattr(resp, "usage") and resp.usage: + tracker.record( + "rollout", + getattr(resp.usage, "input_tokens", 0) or 0, + getattr(resp.usage, "output_tokens", 0) or 0, + ) + text = getattr(resp, "output_text", None) or "" + if text: + return text + for item in getattr(resp, "output", None) or []: + for part in getattr(item, "content", []): + if getattr(part, "type", "") == "output_text": + return part.text or "" + return "" + else: + # Chat Completions API — no tools + kwargs = { + "model": deployment, + "messages": messages, + "max_completion_tokens": max_output_tokens, + } + if reasoning_effort is not None: + kwargs["reasoning_effort"] = reasoning_effort + resp = _llm_call_with_retry(lambda timeout: client.chat.completions.create( + **kwargs, + timeout=timeout, + ), timeout=llm_timeout) + if resp.usage: + tracker.record( + "rollout", + resp.usage.prompt_tokens or 0, + resp.usage.completion_tokens or 0, + ) + return resp.choices[0].message.content or "" + + +# ── Public API ────────────────────────────────────────────────────────────── + +def run_single( + instruction: str, + input_xlsx: str, + output_path: str, + instruction_type: str = "", + answer_position: str = "", + skill_content: str = "", + max_output_tokens: int = 16384, + llm_timeout: int | None = 120, + task_timeout: int | None = 300, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", +) -> dict: + """Single-round code generation. One LLM call, no tools. + + Args: + llm_timeout: Per-LLM-call timeout in seconds (default 120). + task_timeout: Total task timeout in seconds (default 300). + + Returns ``{"code": str, "raw": str, "n_turns": 1}``. + """ + no_task_timeout = task_timeout is None or task_timeout <= 0 + if is_target_exec_backend(): + deadline = None if no_task_timeout else time.time() + task_timeout + deployment = _get_deployment() + work_dir, skill_md, task_md, prompt = _prepare_codex_workspace( + instruction=instruction, + input_xlsx=input_xlsx, + output_path=output_path, + instruction_type=instruction_type, + answer_position=answer_position, + skill_content=skill_content, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + if deadline is None: + effective_timeout = 10**9 + else: + remaining = max(10, int(deadline - time.time())) + effective_timeout = min(task_timeout, remaining) + final_message, raw = _run_exec_backend( + work_dir=work_dir, + prompt=prompt, + model=deployment, + timeout=effective_timeout, + ) + solution_path = os.path.join(work_dir, "solution.py") + if os.path.exists(solution_path): + with open(solution_path, encoding="utf-8") as f: + code = f.read() + else: + code = extract_code(final_message or raw) + return { + "code": code, + "raw": raw or final_message, + "n_turns": 1, + "conversation": [{"role": "assistant", "content": final_message or raw}], + "target_system_prompt": skill_md, + "target_user_prompt": f"{prompt}\n\n## Task File\n\n{task_md}", + } + + deadline = None if no_task_timeout else time.time() + task_timeout + client = get_target_client() + deployment = _get_deployment() + system = _build_system(skill_content) + user = _build_user( + instruction, + input_xlsx, + instruction_type, + answer_position, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + + messages = [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + + if deadline is None: + effective_timeout = None + else: + remaining = max(10, int(deadline - time.time())) + effective_timeout = min(llm_timeout or remaining, remaining) + raw = _chat_call(client, deployment, messages, max_output_tokens, llm_timeout=effective_timeout) + time.sleep(3) # Rate-limit cooldown after successful LLM call + code = extract_code(raw) + + return { + "code": code, + "raw": raw, + "n_turns": 1, + "conversation": [{"role": "assistant", "content": raw}], + "target_system_prompt": system, + "target_user_prompt": user, + } + + +def run_multi( + instruction: str, + input_xlsx: str, + output_path: str, + instruction_type: str = "", + answer_position: str = "", + skill_content: str = "", + max_turns: int = 5, + max_output_tokens: int = 16384, + llm_timeout: int | None = 120, + task_timeout: int | None = 600, + gold_path: str = "", + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", +) -> dict: + """Multi-round code generation with execution feedback. No tools. + + Each round: LLM generates code → execute → if error, feed back and retry. + + Args: + llm_timeout: Per-LLM-call timeout in seconds (default 120). + task_timeout: Total task timeout in seconds (default 600). + gold_path: Path to golden answer xlsx for eval feedback during + training. When non-empty, a successful execution is followed + by an eval check; if the output is wrong the agent receives + cell-level feedback (without revealing expected values) and + gets another turn. Leave empty for eval/test to avoid + data leakage. + + Returns ``{"code": str, "raw": str, "n_turns": int, "conversation": [...]}``. + """ + no_task_timeout = task_timeout is None or task_timeout <= 0 + if is_target_exec_backend(): + deadline = None if no_task_timeout else time.time() + task_timeout + deployment = _get_deployment() + work_dir, skill_md, task_md, initial_prompt = _prepare_codex_workspace( + instruction=instruction, + input_xlsx=input_xlsx, + output_path=output_path, + instruction_type=instruction_type, + answer_position=answer_position, + skill_content=skill_content, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + workspace_name="codex_multi", + ) + prompt = ( + f"{initial_prompt}\n\n" + "## Multi-Turn Repair Mode\n" + "- This is turn 1. Write or overwrite `solution.py`.\n" + "- After each turn, the harness will execute your `solution.py`; if it fails, you will receive feedback and may revise it.\n" + "- Keep the script general: use `INPUT_PATH` and `OUTPUT_PATH`, and do not hardcode one workbook's values." + ) + conversation: list[dict] = [] + code = "" + raw = "" + final_message = "" + solution_path = os.path.join(work_dir, "solution.py") + + for turn in range(max_turns): + if deadline is None: + effective_timeout = 10**9 + else: + remaining = deadline - time.time() + if remaining <= 10: + break + effective_timeout = max(10, int(remaining)) + final_message, raw = _run_exec_backend( + work_dir=work_dir, + prompt=prompt, + model=deployment, + timeout=effective_timeout, + ) + conversation.append({"role": "assistant", "content": final_message or raw}) + + if os.path.exists(solution_path): + with open(solution_path, encoding="utf-8") as f: + code = f.read() + else: + code = extract_code(final_message or raw) + if code.strip(): + with open(solution_path, "w", encoding="utf-8") as f: + f.write(code) + + if not code.strip(): + feedback = ( + "No usable `solution.py` or Python code block was produced. " + "Write a complete `solution.py` that reads `INPUT_PATH` and saves `OUTPUT_PATH`." + ) + else: + ok, err = run_generated_code( + code, + input_xlsx, + output_path, + timeout=None if no_task_timeout else 120, + ) + if ok: + if gold_path and answer_position: + from skillopt.envs.spreadsheetbench.rollout import _auto_verify_output + eval_result = evaluate( + output_path, gold_path, instruction_type, answer_position, + ) + if eval_result["ok"]: + break + verify = _auto_verify_output(output_path, gold_path, answer_position) + feedback = _build_eval_feedback(verify) + else: + break + else: + feedback = ( + "The current `solution.py` raised an error during harness execution:\n\n" + f"```\n{err[:3000]}\n```\n\n" + "Revise `solution.py` to fix the error. Keep using `INPUT_PATH` and `OUTPUT_PATH`." + ) + + feedback_path = os.path.join(work_dir, f"feedback_turn_{turn + 1:02d}.md") + with open(feedback_path, "w", encoding="utf-8") as f: + f.write(feedback) + conversation.append({"role": "user", "content": feedback}) + prompt = ( + f"The previous `solution.py` was evaluated and needs another revision.\n" + f"Read `{os.path.basename(feedback_path)}` and update `solution.py` accordingly.\n" + "You may run `python run_solution.py` for a local syntax/runtime check, but the harness will run the final code separately.\n" + "Do not hardcode workbook-specific answers; preserve unrelated cells." + ) + + return { + "code": code, + "raw": raw or final_message, + "n_turns": len([m for m in conversation if m["role"] == "assistant"]), + "conversation": conversation, + "target_system_prompt": skill_md, + "target_user_prompt": f"{initial_prompt}\n\n## Task File\n\n{task_md}", + } + + deadline = None if no_task_timeout else time.time() + task_timeout + client = get_target_client() + deployment = _get_deployment() + system = _build_system(skill_content) + user = _build_user( + instruction, + input_xlsx, + instruction_type, + answer_position, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + + messages: list[dict] = [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + conversation: list[dict] = [] + code = "" + raw = "" + + for turn in range(max_turns): + if deadline is None: + effective_timeout = None + else: + remaining = deadline - time.time() + if remaining <= 10: + # Not enough time for another round + break + effective_timeout = min(llm_timeout or int(remaining), int(remaining)) + raw = _chat_call(client, deployment, messages, max_output_tokens, llm_timeout=effective_timeout) + time.sleep(3) # Rate-limit cooldown after successful LLM call + code = extract_code(raw) + conversation.append({"role": "assistant", "content": raw}) + messages.append({"role": "assistant", "content": raw}) + + if not code.strip(): + # No code extracted — ask again + feedback = ( + "No Python code block was found in your response. " + "Please return a complete Python script inside a ```python``` block." + ) + messages.append({"role": "user", "content": feedback}) + conversation.append({"role": "user", "content": feedback}) + continue + + # Execute the code + ok, err = run_generated_code( + code, + input_xlsx, + output_path, + timeout=None if no_task_timeout else 120, + ) + if ok: + # Execution succeeded — check correctness if gold_path available + if gold_path and answer_position: + from skillopt.envs.spreadsheetbench.rollout import _auto_verify_output + eval_result = evaluate( + output_path, gold_path, instruction_type, answer_position, + ) + if eval_result["ok"]: + break # Genuinely correct — stop + + # Output is wrong — build feedback without leaking golden values + verify = _auto_verify_output(output_path, gold_path, answer_position) + feedback = _build_eval_feedback(verify) + messages.append({"role": "user", "content": feedback}) + conversation.append({"role": "user", "content": feedback}) + continue + else: + # No gold path (eval/test) — accept execution success + break + + # Execution failed — feed error back + feedback = ( + f"The code raised an error during execution:\n\n" + f"```\n{err[:3000]}\n```\n\n" + f"Please fix the code and return a complete corrected Python script " + f"inside a ```python``` block." + ) + messages.append({"role": "user", "content": feedback}) + conversation.append({"role": "user", "content": feedback}) + + return { + "code": code, + "raw": raw, + "n_turns": turn + 1, + "conversation": conversation, + "target_system_prompt": system, + "target_user_prompt": user, + } diff --git a/skillopt/envs/spreadsheetbench/dataloader.py b/skillopt/envs/spreadsheetbench/dataloader.py new file mode 100644 index 0000000..542ecc5 --- /dev/null +++ b/skillopt/envs/spreadsheetbench/dataloader.py @@ -0,0 +1,37 @@ +"""SpreadsheetBench task dataloader.""" +from __future__ import annotations + +from skillopt.datasets.base import SplitDataLoader + + +class SpreadsheetBenchDataLoader(SplitDataLoader): + """SpreadsheetBench dataloader. + + Each split directory contains a .json file (JSON array of task items). + Spreadsheet files referenced by items live under a separate ``data_root``. + """ + + def __init__( + self, + split_dir: str = "", + data_path: str = "", + split_mode: str = "ratio", + split_ratio: str = "2:1:7", + split_seed: int = 42, + split_output_dir: str = "", + data_root: str = "", + seed: int = 42, + limit: int = 0, + **kwargs, + ) -> None: + super().__init__( + split_dir=split_dir, + data_path=data_path, + split_mode=split_mode, + split_ratio=split_ratio, + split_seed=split_seed, + split_output_dir=split_output_dir, + seed=seed, + limit=limit, + ) + self.data_root = data_root diff --git a/skillopt/envs/spreadsheetbench/evaluator.py b/skillopt/envs/spreadsheetbench/evaluator.py new file mode 100644 index 0000000..3d8b84a --- /dev/null +++ b/skillopt/envs/spreadsheetbench/evaluator.py @@ -0,0 +1,158 @@ +"""Cell-value evaluator faithful to the official SpreadsheetBench +`evaluation/evaluation.py` (https://github.com/RUCKBReasoning/SpreadsheetBench). + +Key rules (copied from the official `transform_value` / `compare_cell_value`): + * numeric values (int/float and numeric strings) are compared after + ``round(float(v), 2)`` — a fixed 2-decimal quantization (NOT a tolerance); + * ``datetime.time`` is stringified and the trailing microseconds stripped; + * ``datetime.datetime`` is converted to an Excel serial day and rounded + to an integer day; + * an empty string ``""`` and ``None`` are considered equal, but otherwise + ``type(v1) != type(v2)`` fails the comparison. + +Format/style comparison is deliberately NOT performed — the official +reference evaluator also skips it (the relevant lines are commented out +in `cell_level_compare`). soft vs hard is defined at the run_bench level +across a task's multiple test cases, not here. +""" +from __future__ import annotations + +import datetime +import os +import re + +import openpyxl + + +# ---------- value transform / compare (official port) ---------- + +def _datetime_to_float(dt: datetime.datetime) -> float: + excel_start_date = datetime.datetime(1899, 12, 30) + delta = dt - excel_start_date + return delta.days + delta.seconds / 86400.0 + + +def _transform_value(v): + if isinstance(v, bool): + # openpyxl can return Python bool; official code doesn't special-case + # bools, but round(float(True), 2) == 1.0 which breaks 1 vs True. Keep + # parity with the official transform by promoting bool -> float. + return round(float(v), 2) + if isinstance(v, (int, float)): + return round(float(v), 2) + if isinstance(v, datetime.time): + return str(v)[:-3] + if isinstance(v, datetime.datetime): + return round(_datetime_to_float(v), 0) + if isinstance(v, str): + try: + return round(float(v), 2) + except ValueError: + return v + return v + + +def _compare_cell_value(v1, v2) -> bool: + v1 = _transform_value(v1) + v2 = _transform_value(v2) + if (v1 == "" and v2 is None) or (v1 is None and v2 == ""): + return True + if (v1 == "" and v2 == "") or (v1 is None and v2 is None): + return True + if type(v1) is not type(v2): + return False + return v1 == v2 + + +# ---------- range parsing (official port) ---------- + +def _col_num2name(n: int) -> str: + name = "" + while n > 0: + n, r = divmod(n - 1, 26) + name = chr(65 + r) + name + return name + + +def _col_name2num(name: str) -> int: + num = 0 + for c in name: + num = num * 26 + (ord(c) - ord("A") + 1) + return num + + +def _parse_range(range_str: str): + start_cell, end_cell = range_str.split(":") + sc = "".join(ch for ch in start_cell if ch.isalpha()) + sr = "".join(ch for ch in start_cell if ch.isdigit()) + ec = "".join(ch for ch in end_cell if ch.isalpha()) + er = "".join(ch for ch in end_cell if ch.isdigit()) + return (_col_name2num(sc), int(sr)), (_col_name2num(ec), int(er)) + + +def _generate_cell_names(range_str: str): + if ":" not in range_str: + return [range_str] + (sc, sr), (ec, er) = _parse_range(range_str) + cols = [_col_num2name(i) for i in range(sc, ec + 1)] + return [f"{c}{r}" for c in cols for r in range(sr, er + 1)] + + +def _cell_level_compare(wb_gt, wb_proc, sheet_name: str, cell_range: str): + if sheet_name not in wb_proc.sheetnames: + return False, f"worksheet not found: {sheet_name}" + ws_gt = wb_gt[sheet_name] + ws_proc = wb_proc[sheet_name] + for cn in _generate_cell_names(cell_range): + cg = ws_gt[cn] + cp = ws_proc[cn] + if not _compare_cell_value(cg.value, cp.value): + return False, f"value@{sheet_name}!{cn}: gt={cg.value!r} pred={cp.value!r}" + return True, "" + + +# ---------- public API ---------- + +def compare_workbooks(gt_file: str, proc_file: str, answer_position: str) -> tuple[bool, str]: + """Return (ok, msg). Single test-case comparison, official semantics.""" + if not os.path.exists(proc_file): + return False, "file not exist" + try: + wb_gt = openpyxl.load_workbook(filename=gt_file, data_only=True) + wb_proc = openpyxl.load_workbook(filename=proc_file, data_only=True) + except Exception as e: # noqa: BLE001 + return False, f"load error: {e}" + try: + ok_all = True + msg_first = "" + for scr in (answer_position or "").split(","): + scr = scr.strip() + if not scr: + continue + if "!" in scr: + sheet_name, cell_range = scr.split("!", 1) + sheet_name = sheet_name.strip().strip("'\"") + else: + sheet_name = wb_gt.sheetnames[0] + cell_range = scr + cell_range = cell_range.strip().strip("'\"") + ok, msg = _cell_level_compare(wb_gt, wb_proc, sheet_name, cell_range) + if not ok: + ok_all = False + if not msg_first: + msg_first = msg + return ok_all, msg_first + finally: + wb_gt.close() + wb_proc.close() + + +def evaluate(pred_path: str, gold_path: str, + instruction_type: str, answer_position: str) -> dict: + """Single test-case evaluate. soft/hard aggregation happens in run_bench.""" + ok, msg = compare_workbooks(gold_path, pred_path, answer_position) + return { + "ok": ok, + "reason": msg, + "instruction_type": instruction_type, + } diff --git a/skillopt/envs/spreadsheetbench/executor.py b/skillopt/envs/spreadsheetbench/executor.py new file mode 100644 index 0000000..24421f9 --- /dev/null +++ b/skillopt/envs/spreadsheetbench/executor.py @@ -0,0 +1,67 @@ +"""Execute LLM-generated Python code against an input xlsx to produce an output xlsx.""" +from __future__ import annotations + +import os +import re +import subprocess +import sys +import tempfile +import textwrap + + +RUNNER_TEMPLATE = textwrap.dedent( + """ + import os, sys, traceback + INPUT_PATH = {input_path!r} + OUTPUT_PATH = {output_path!r} + try: + {user_code_indented} + except Exception: + traceback.print_exc() + sys.exit(2) + """ +) + +# Regex to strip user-defined INPUT_PATH / OUTPUT_PATH assignments, +# since the runner template injects the correct values. +_PATH_ASSIGN_RE = re.compile( + r'^\s*(INPUT_PATH|OUTPUT_PATH)\s*=\s*.+$', re.MULTILINE +) + + +def _strip_path_assignments(code: str) -> str: + """Remove INPUT_PATH/OUTPUT_PATH assignments from user code.""" + return _PATH_ASSIGN_RE.sub("", code) + + +def run_generated_code(code: str, input_path: str, output_path: str, timeout: int | None = 120) -> tuple[bool, str]: + os.makedirs(os.path.dirname(output_path), exist_ok=True) + cleaned = _strip_path_assignments(code) + indented = textwrap.indent(cleaned, " ") + script = RUNNER_TEMPLATE.format( + input_path=input_path, + output_path=output_path, + user_code_indented=indented, + ) + with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as f: + f.write(script) + tmp = f.name + try: + proc = subprocess.run( + [sys.executable, tmp], + capture_output=True, + text=True, + timeout=timeout if timeout and timeout > 0 else None, + ) + if proc.returncode != 0: + return False, (proc.stdout + "\n" + proc.stderr).strip() + if not os.path.exists(output_path): + return False, "output file was not created" + return True, "" + except subprocess.TimeoutExpired: + return False, f"timeout after {timeout}s" + finally: + try: + os.unlink(tmp) + except OSError: + pass diff --git a/skillopt/envs/spreadsheetbench/prompts/analyst_error.md b/skillopt/envs/spreadsheetbench/prompts/analyst_error.md new file mode 100644 index 0000000..dc7f352 --- /dev/null +++ b/skillopt/envs/spreadsheetbench/prompts/analyst_error.md @@ -0,0 +1,46 @@ +You are an expert failure-analysis agent for spreadsheet manipulation tasks. + +You will be given MULTIPLE failed agent trajectories from a single minibatch +and the current skill document. +Your job is to identify the most important COMMON failure patterns across +the batch and propose a concise set of skill edits. + +## Failure Type Categories +- **rule_missing**: the skill lacks a relevant rule for this type of task +- **rule_wrong**: an existing skill rule is misleading or incorrect +- **rule_ignored**: the skill has the right rule but the agent did not follow it +- **data_exploration**: the agent did not read enough data from the spreadsheet +- **code_error**: the agent's code has a bug unrelated to the skill +- **other**: none of the above + +## Analysis Process +1. Read ALL failed trajectories in the minibatch. +2. Identify the most prevalent, systematic failure patterns across them. +3. For each pattern, classify its failure type. +4. Propose skill edits that address the COMMON patterns — not individual edge cases. +5. Edits must be generalizable; do not hardcode task-specific values + (file paths, cell addresses, expected values). +6. Only patch gaps in the skill — do not duplicate existing content. +7. If the failure is because the agent did not read enough spreadsheet rows/columns + to understand the data, propose a patch encouraging broader data exploration. + +You will be told the maximum number of edits (the budget L). Produce AT MOST L edits, +focusing on the highest-impact patterns. You may produce fewer if warranted. + +Respond ONLY with a valid JSON object (no markdown fences, no extra text): +{ + "batch_size": , + "failure_summary": [ + {"failure_type": "", "count": , "description": ""} + ], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +Only include edits that are needed. "edits" can be an empty list if no patch is warranted. diff --git a/skillopt/envs/spreadsheetbench/prompts/analyst_success.md b/skillopt/envs/spreadsheetbench/prompts/analyst_success.md new file mode 100644 index 0000000..7e97330 --- /dev/null +++ b/skillopt/envs/spreadsheetbench/prompts/analyst_success.md @@ -0,0 +1,32 @@ +You are an expert success-pattern analyst for AI spreadsheet agents. + +You will be given MULTIPLE successful agent trajectories from a single minibatch +and the current skill document. Your job is to identify generalizable behavior +patterns that are COMMON across the batch and worth encoding in the skill. + +## Rules +- Only propose patches for patterns NOT already covered in the skill. +- Focus on patterns that appear across MULTIPLE trajectories in the batch. +- Be concise. Patterns must generalize beyond specific tasks. +- Prefer reinforcing existing sections over adding new top-level sections. +- If the agents' success involved reading enough data rows or using a smart + exploration strategy, consider reinforcing that in the patch. + +You will be told the maximum number of edits (the budget L). Produce AT MOST L edits, +focusing on the most broadly applicable patterns. You may produce fewer if warranted. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "success_patterns": ["", ""], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +"edits" may be empty if the skill already covers all observed patterns. diff --git a/skillopt/envs/spreadsheetbench/prompts/codegen_system.md b/skillopt/envs/spreadsheetbench/prompts/codegen_system.md new file mode 100644 index 0000000..1ec7134 --- /dev/null +++ b/skillopt/envs/spreadsheetbench/prompts/codegen_system.md @@ -0,0 +1 @@ +You are an expert Python programmer specializing in spreadsheet manipulation. You will be given a user instruction together with a preview of an input .xlsx file. Your job is to write a single self-contained Python script that reads the input file at the path stored in the variable INPUT_PATH, performs the requested manipulation, and saves the result to OUTPUT_PATH. Use only the standard library, openpyxl, and pandas. Do not print anything. Do not use input(). Do not hardcode file paths. Return ONLY the Python code inside a single ```python ... ``` fenced block. \ No newline at end of file diff --git a/skillopt/envs/spreadsheetbench/prompts/critical_rules.md b/skillopt/envs/spreadsheetbench/prompts/critical_rules.md new file mode 100644 index 0000000..822da17 --- /dev/null +++ b/skillopt/envs/spreadsheetbench/prompts/critical_rules.md @@ -0,0 +1,9 @@ +## Critical Rules (MUST follow) +1. NEVER write Excel formulas to cells that will be graded on their displayed value. + openpyxl does NOT compute formulas -- the evaluator will see None. + Instead, compute results in Python and write literal values (numbers/strings). +2. After saving the workbook, ALWAYS reopen and verify the written values: + `wb2 = openpyxl.load_workbook(OUTPUT_PATH); print(wb2[sheet][cell].value)` +3. Use the `write_file` tool to create solution.py -- it avoids shell escaping issues. + Do NOT use `echo "..." > solution.py` for multi-line scripts. + diff --git a/skillopt/envs/spreadsheetbench/prompts/react_system.md b/skillopt/envs/spreadsheetbench/prompts/react_system.md new file mode 100644 index 0000000..afbbffb --- /dev/null +++ b/skillopt/envs/spreadsheetbench/prompts/react_system.md @@ -0,0 +1,21 @@ +You are an expert spreadsheet manipulation agent. + +{critical_rules}{skill_section}## Tools +You have two tools: +- `bash` -- execute any shell command and receive its output. +- `write_file` -- write content to a file (path, content). Use this for solution.py. + +## Protocol +1. Explore the input spreadsheet to understand its structure (sheets, headers, row count). +2. Use the `write_file` tool to create `solution.py` in the current directory. + solution.py MUST start with: + INPUT_PATH = "" + OUTPUT_PATH = "" + Then perform the manipulation and save the result to OUTPUT_PATH. + Use only: standard library, openpyxl, pandas. +3. Run `python solution.py` via `bash` and verify the output was created. +4. Fix any errors and re-run until the output is correct. +5. Once OUTPUT_PATH exists and is correct, stop calling tools. + +Do NOT use any libraries other than standard library, openpyxl, and pandas. +Do NOT hardcode cell values from the preview -- iterate over actual rows. diff --git a/skillopt/envs/spreadsheetbench/react_agent.py b/skillopt/envs/spreadsheetbench/react_agent.py new file mode 100644 index 0000000..2e72953 --- /dev/null +++ b/skillopt/envs/spreadsheetbench/react_agent.py @@ -0,0 +1,395 @@ +"""ReAct agent with bash tool for SpreadsheetBench evaluation. + +Adapted from the original SpreadsheetBench react agent implementation. + +Uses the unified ``skillopt.model`` router so SpreadsheetBench follows the same +backend selection as the rest of the framework. +""" +from __future__ import annotations + +import json +import os +import subprocess + +from skillopt.model import chat_target_messages +from skillopt.prompts import load_prompt + +# ── Tool schemas ───────────────────────────────────────────────────────────── + +BASH_TOOL_CHAT = { + "type": "function", + "function": { + "name": "bash", + "description": ( + "Execute a bash command and receive stdout+stderr (truncated to 4000 chars). " + "Use Python to read / write Excel files." + ), + "parameters": { + "type": "object", + "properties": { + "cmd": {"type": "string", "description": "Bash command to execute."} + }, + "required": ["cmd"], + }, + }, +} + +BASH_TOOL_RESPONSES = { + "type": "function", + "name": "bash", + "description": ( + "Execute a bash command and receive stdout+stderr (truncated to 4000 chars). " + "Use Python to read / write Excel files." + ), + "parameters": { + "type": "object", + "properties": { + "cmd": {"type": "string", "description": "Bash command to execute."} + }, + "required": ["cmd"], + }, +} + +WRITE_FILE_TOOL_CHAT = { + "type": "function", + "function": { + "name": "write_file", + "description": ( + "Write content to a file. Use this instead of echo/cat for multi-line " + "Python scripts to avoid shell escaping issues." + ), + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "File path to write (relative to working directory).", + }, + "content": { + "type": "string", + "description": "File content to write.", + }, + }, + "required": ["path", "content"], + }, + }, +} + +WRITE_FILE_TOOL_RESPONSES = { + "type": "function", + "name": "write_file", + "description": ( + "Write content to a file. Use this instead of echo/cat for multi-line " + "Python scripts to avoid shell escaping issues." + ), + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "File path to write (relative to working directory).", + }, + "content": { + "type": "string", + "description": "File content to write.", + }, + }, + "required": ["path", "content"], + }, +} + +# ── System prompt ───────────────────────────────────────────────────────────── + + +def _build_system(skill_content: str) -> str: + if skill_content.strip(): + skill_section = f"## Skill\n{skill_content.strip()}\n\n" + else: + skill_section = "" + return load_prompt("react_system", env="spreadsheetbench").format( + critical_rules=load_prompt("critical_rules", env="spreadsheetbench"), + skill_section=skill_section, + ) + + +def _build_user( + instruction: str, + input_path: str, + output_path: str, + instruction_type: str, + answer_position: str, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", +) -> str: + parts = [] + if diagnostic_trace_context.strip(): + parts.append( + "# Previous Codex Trace Snapshot\n" + "This is a partial transcript from an earlier attempt. Use it as your current reasoning context.\n\n" + f"{diagnostic_trace_context.strip()}" + ) + parts.extend([ + f"# Instruction\n{instruction}", + f"# Input file\n{input_path}", + f"# Output file\n{output_path}", + ]) + if instruction_type: + parts.append(f"# Instruction type\n{instruction_type}") + if answer_position: + parts.append(f"# Answer position\n{answer_position}") + if diagnostic_mode and diagnostic_instruction.strip(): + parts.append(f"# Training readout\n{diagnostic_instruction.strip()}") + parts.append( + "Manipulate the input spreadsheet according to the instruction " + "and save the result to the output file." + ) + return "\n\n".join(parts) + + +# ── File write (bypass shell escaping) ──────────────────────────────────────── + +def _write_file(path: str, content: str, work_dir: str) -> str: + """Write content to a file, bypassing shell escaping issues.""" + try: + full_path = os.path.join(work_dir, path) if not os.path.isabs(path) else path + parent = os.path.dirname(full_path) + if parent: + os.makedirs(parent, exist_ok=True) + with open(full_path, "w") as f: + f.write(content) + return f"File written: {full_path} ({len(content)} chars)" + except Exception as e: # noqa: BLE001 + return f"[write_file error: {e}]" + + +# ── Auto-verification ───────────────────────────────────────────────────────── + +def _auto_verify(work_dir: str) -> str: + """Auto-verify output xlsx after solution.py runs.""" + import glob as _glob + + sol_path = os.path.join(work_dir, "solution.py") + output_path = None + if os.path.exists(sol_path): + with open(sol_path) as f: + for line in f: + stripped = line.strip() + if stripped.startswith("OUTPUT_PATH"): + try: + val = stripped.split("=", 1)[1].strip() + output_path = val.strip("'\"").strip() + except Exception: # noqa: BLE001 + pass + break + + if not output_path or not os.path.exists(output_path): + xlsx_files = [ + f for f in _glob.glob(os.path.join(work_dir, "*.xlsx")) + if "_pred" in os.path.basename(f) + ] + if xlsx_files: + output_path = xlsx_files[0] + + if not output_path or not os.path.exists(output_path): + return ( + "\n\n[AUTO-VERIFY] WARNING: Output file not found! " + "Make sure OUTPUT_PATH is correct and wb.save(OUTPUT_PATH) is called." + ) + + try: + import openpyxl + + wb_formula = openpyxl.load_workbook(output_path, data_only=False) + wb_value = openpyxl.load_workbook(output_path, data_only=True) + lines = [f"\n\n[AUTO-VERIFY] Output file exists: {output_path}"] + + sn = wb_formula.sheetnames[0] + ws_f = wb_formula[sn] + ws_v = wb_value[sn] + lines.append(f" Sheet '{sn}': {ws_f.dimensions}") + + for row in ws_v.iter_rows( + min_row=1, max_row=min(5, ws_v.max_row), values_only=True, + ): + lines.append(f" {list(row)}") + + none_cells: list[str] = [] + for row_f, row_v in zip( + ws_f.iter_rows(min_row=1, max_row=min(30, ws_f.max_row)), + ws_v.iter_rows(min_row=1, max_row=min(30, ws_v.max_row)), + ): + for cf, cv in zip(row_f, row_v): + formula_val = cf.value + cached_val = cv.value + if ( + isinstance(formula_val, str) + and formula_val.startswith("=") + and cached_val is None + ): + none_cells.append(cf.coordinate) + + if none_cells: + lines.append( + f" WARNING: {len(none_cells)} cells have formulas but NO cached " + f"value -- evaluator will see None: {none_cells[:10]}" + ) + lines.append( + " FIX: Compute values in Python and write literal " + "numbers/strings instead of formulas." + ) + else: + lines.append(" All cells have concrete values. Looks good.") + + wb_formula.close() + wb_value.close() + return "\n".join(lines) + except Exception as e: # noqa: BLE001 + return f"\n\n[AUTO-VERIFY] Could not inspect output: {e}" + + +# ── Bash execution ──────────────────────────────────────────────────────────── + +def _run_bash(cmd: str, work_dir: str, timeout: int = 60) -> str: + try: + proc = subprocess.run( + cmd, + shell=True, + capture_output=True, + text=True, + timeout=timeout, + cwd=work_dir, + ) + out = (proc.stdout + proc.stderr).strip() + except subprocess.TimeoutExpired: + return f"[timeout after {timeout}s]" + except Exception as e: # noqa: BLE001 + return f"[error: {e}]" + if len(out) > 4000: + out = out[:3800] + f"\n...[truncated, {len(out)} total chars]" + result = out or "(no output)" + + if "solution.py" in cmd and "python" in cmd.lower(): + result += _auto_verify(work_dir) + + return result + + +def _assistant_tool_calls(message) -> list[dict]: + tool_calls = getattr(message, "tool_calls", None) or [] + return [ + tool_call.model_dump() if hasattr(tool_call, "model_dump") else dict(tool_call) + for tool_call in tool_calls + ] + + +def _react_loop( + system: str, + user: str, + work_dir: str, + max_turns: int, + max_output_tokens: int, +) -> dict: + messages: list[dict] = [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + conversation: list[dict] = [] + n_turns = 0 + + for _ in range(max_turns): + message, _ = chat_target_messages( + messages=messages, + tools=[BASH_TOOL_CHAT, WRITE_FILE_TOOL_CHAT], + tool_choice="auto", + max_completion_tokens=max_output_tokens, + retries=5, + stage="rollout", + return_message=True, + ) + + assistant_text = str(getattr(message, "content", "") or "") + tool_calls = _assistant_tool_calls(message) + assistant_payload: dict = {"role": "assistant", "content": assistant_text} + if tool_calls: + assistant_payload["tool_calls"] = tool_calls + messages.append(assistant_payload) + + if not tool_calls: + conversation.append({"type": "message", "content": assistant_text}) + break + + for tool_call in tool_calls: + n_turns += 1 + function = tool_call.get("function", {}) or {} + try: + args = json.loads(str(function.get("arguments", "{}") or "{}")) + except json.JSONDecodeError: + args = {} + + if function.get("name") == "write_file": + obs = _write_file( + args.get("path", ""), + args.get("content", ""), + work_dir, + ) + conversation.append({ + "type": "tool_call", + "cmd": f"[write_file] {args.get('path', '')}", + "obs": obs, + }) + else: + cmd = args.get("cmd", "") + obs = _run_bash(cmd, work_dir) + conversation.append({"type": "tool_call", "cmd": cmd, "obs": obs}) + + messages.append( + { + "role": "tool", + "tool_call_id": tool_call.get("id", ""), + "content": obs, + } + ) + + return {"conversation": conversation, "n_turns": n_turns} + + +# ── Public API ──────────────────────────────────────────────────────────────── + +def run_react( + instruction: str, + input_path: str, + output_path: str, + work_dir: str, + instruction_type: str = "", + answer_position: str = "", + skill_content: str = "", + max_turns: int = 30, + max_output_tokens: int = 16384, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", +) -> dict: + """Run the ReAct agent for one task. + + Returns: + { + "conversation": [...], # list of {type, cmd/content, obs?} + "n_turns": int, # number of bash tool calls made + } + """ + system = _build_system(skill_content) + user = _build_user( + instruction, + input_path, + output_path, + instruction_type, + answer_position, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + result = _react_loop(system, user, work_dir, max_turns, max_output_tokens) + result["target_system_prompt"] = system + result["target_user_prompt"] = user + return result diff --git a/skillopt/envs/spreadsheetbench/reflect.py b/skillopt/envs/spreadsheetbench/reflect.py new file mode 100644 index 0000000..cdfeaf6 --- /dev/null +++ b/skillopt/envs/spreadsheetbench/reflect.py @@ -0,0 +1,4 @@ +"""SpreadsheetBench Reflect stage. + +Prompts are now loaded from .md files by the base adapter. +""" diff --git a/skillopt/envs/spreadsheetbench/rollout.py b/skillopt/envs/spreadsheetbench/rollout.py new file mode 100644 index 0000000..4667775 --- /dev/null +++ b/skillopt/envs/spreadsheetbench/rollout.py @@ -0,0 +1,979 @@ +"""SpreadsheetBench rollout — codegen & ReAct batch execution. + +Provides: + - process_one_codegen(): single/multi-round code generation (no tool-call) + - run_spreadsheet_batch_codegen(): batch wrapper for codegen + - process_one(): ReAct agent with tool-call (legacy) + - run_spreadsheet_batch(): batch wrapper for ReAct (legacy) + - load_items(): load benchmark .json/.jsonl files +""" +from __future__ import annotations + +import glob as _glob +import json +import os +import shutil +import tempfile +import time +import traceback +from concurrent.futures import ( + FIRST_COMPLETED, + ThreadPoolExecutor, + wait, + TimeoutError as FuturesTimeoutError, +) + +import openpyxl + +from skillopt.envs.spreadsheetbench.react_agent import run_react +from skillopt.envs.spreadsheetbench.evaluator import ( + evaluate, _generate_cell_names, _compare_cell_value, +) +from skillopt.envs.spreadsheetbench.executor import run_generated_code + + +# ── Data loading ───────────────────────────────────────────────────────────── + + +def load_items(path: str) -> list[dict]: + """Load a benchmark file. Supports both .jsonl and .json (list of dicts).""" + if path.endswith(".json"): + with open(path) as f: + data = json.load(f) + if isinstance(data, dict): + data = data.get("data") or list(data.values()) + return list(data) + items = [] + with open(path) as f: + for line in f: + line = line.strip() + if line: + items.append(json.loads(line)) + return items + + +# ── Test case discovery ────────────────────────────────────────────────────── + + +def _find_test_cases(task_dir: str) -> list[tuple[str, str, str]]: + """Return [(case_no, input_path, answer_path), ...] sorted by case_no. + + Supports naming conventions used by SpreadsheetBench releases: + * ``{no}_{id}_input.xlsx`` + ``{no}_{id}_answer.xlsx`` (original) + * ``{no}_{id}_init.xlsx`` + ``{no}_{id}_golden.xlsx`` (verified_400) + * ``initial.xlsx`` + ``golden.xlsx`` (verified_400, no prefix) + """ + cases: list[tuple[str, str, str]] = [] + inputs = sorted(_glob.glob(os.path.join(task_dir, "*_input.xlsx"))) + for ip in inputs: + no = os.path.basename(ip).split("_", 1)[0] + ap = ip.replace("_input.xlsx", "_answer.xlsx") + if os.path.exists(ap): + cases.append((no, ip, ap)) + inits = sorted(_glob.glob(os.path.join(task_dir, "*_init.xlsx"))) + for ip in inits: + no = os.path.basename(ip).split("_", 1)[0] + ap = ip.replace("_init.xlsx", "_golden.xlsx") + if os.path.exists(ap): + cases.append((no, ip, ap)) + + # Fallback: bare initial.xlsx + golden.xlsx (no numbered prefix) + if not cases: + bare_init = os.path.join(task_dir, "initial.xlsx") + bare_gold = os.path.join(task_dir, "golden.xlsx") + if os.path.exists(bare_init) and os.path.exists(bare_gold): + cases.append(("1", bare_init, bare_gold)) + + return cases + + +# ── Auto-verify helper ────────────────────────────────────────────────────── + +# The official SpreadsheetBench evaluator never serialises cells to text — it +# compares in memory and returns only a pass/fail bool. The per-cell report +# below is a repo-local training aid (fed back to the model on retry and saved +# into the trajectory for reflection). On most tasks the answer range is a +# handful of cells, so the full report is tiny. But a few tasks have answer +# ranges spanning tens of thousands of cells (e.g. 80-42 = +# 'Consolidate_ALL'!A2:L8000 ≈ 96k cells); dumping every cell explodes the +# report to several MB, floods the model's context and bloats conversation +# files. We therefore apply the same head+tail character truncation the rest of +# the codebase uses for oversized trajectory text (cf. reflect.py / slow_update.py +# `text[:half] + "...[truncated]...\n" + text[-half:]`): keep the first and last +# `_MAX_REPORT_CHARS // 2` chars so both the leading and trailing wrong cells +# stay visible. Small reports are unchanged. +_MAX_REPORT_CHARS = 12000 # head+tail char budget (~6000 head + 6000 tail) + + +def _auto_verify_output( + pred_path: str, + gold_path: str, + answer_position: str, +) -> str: + """Reopen the predicted xlsx and compare cells at answer_position with gold. + + Returns a human-readable verification report that can be appended to the + trajectory so the error analyst can see exactly what went wrong (e.g. + ``cell A1: got=None, expected=420``). Oversized reports are head+tail + truncated to `_MAX_REPORT_CHARS` chars, matching the rest of the codebase. + """ + if not os.path.exists(pred_path): + return "Verification: output file does not exist." + try: + wb_pred = openpyxl.load_workbook(pred_path, data_only=True) + wb_gold = openpyxl.load_workbook(gold_path, data_only=True) + except Exception as e: + return f"Verification: could not open workbooks: {e}" + + lines = ["## Output Verification"] + try: + for scr in (answer_position or "").split(","): + scr = scr.strip() + if not scr: + continue + if "!" in scr: + sheet_name, cell_range = scr.split("!", 1) + sheet_name = sheet_name.strip().strip("'\"") + else: + sheet_name = wb_gold.sheetnames[0] + cell_range = scr + cell_range = cell_range.strip().strip("'\"") + + cell_names = _generate_cell_names(cell_range) + ws_pred = wb_pred[sheet_name] if sheet_name in wb_pred.sheetnames else None + ws_gold = wb_gold[sheet_name] if sheet_name in wb_gold.sheetnames else None + + if ws_pred is None: + lines.append(f" Sheet '{sheet_name}' NOT FOUND in output.") + continue + + n_empty_correct = 0 # empty-on-both correct cells collapsed to a count + for cn in cell_names: + gv = ws_gold[cn].value if ws_gold else "N/A" + pv = ws_pred[cn].value + # Use the official cell comparator so this report's ✓/✗ agrees + # with the real scorer (evaluate). repr() equality would wrongly + # flag e.g. 5 vs 5.0 or None vs "" as mismatches and mislead the + # model into "fixing" cells that already pass scoring. + ok_cell = ws_gold is not None and _compare_cell_value(gv, pv) + # Collapse only cells that are correct AND empty on both sides + # (got=None, expected=None ✓): pure noise. Every other cell — + # including non-empty correct cells — is listed in full; the + # final head+tail char cap keeps the report bounded. + if ok_cell and gv in (None, "") and pv in (None, ""): + n_empty_correct += 1 + continue + match = "✓" if ok_cell else "✗" + lines.append(f" {sheet_name}!{cn}: got={pv!r}, expected={gv!r} {match}") + if n_empty_correct: + lines.append( + f" (+{n_empty_correct} empty cells correct, omitted)" + ) + + # Also check if any cells in the output contain formula strings + formula_cells = [] + for sn in wb_pred.sheetnames: + ws = wb_pred[sn] + for row in ws.iter_rows(max_row=min(ws.max_row, 200), values_only=False): + for cell in row: + if isinstance(cell.value, str) and cell.value.startswith("="): + formula_cells.append(f"{sn}!{cell.coordinate}={cell.value}") + if len(formula_cells) >= 10: + break + if len(formula_cells) >= 10: + break + if len(formula_cells) >= 10: + break + if formula_cells: + lines.append(f"\n WARNING: {len(formula_cells)} cells contain Excel formulas (openpyxl cannot evaluate them):") + for fc in formula_cells[:5]: + lines.append(f" {fc}") + if len(formula_cells) > 5: + lines.append(f" ... and {len(formula_cells) - 5} more") + finally: + wb_pred.close() + wb_gold.close() + + report = "\n".join(lines) + # Head+tail truncation, matching reflect.py / slow_update.py: keep the first + # and last half so both leading and trailing wrong cells remain visible. + if len(report) > _MAX_REPORT_CHARS: + half = _MAX_REPORT_CHARS // 2 + report = ( + report[:half] + + f"\n ...[verification report truncated, {len(report)} chars total]...\n" + + report[-half:] + ) + return report + + +# ── Per-task worker ────────────────────────────────────────────────────────── + + +def process_one( + item: dict, + data_root: str, + out_root: str, + skill_content: str, + max_turns: int, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", + max_completion_tokens: int = 16384, +) -> dict: + """Run the ReAct agent on a single SpreadsheetBench task. + + Returns a result dict compatible with ``compute_score()``. + """ + task_id = str(item["id"]) + instruction = item["instruction"] + instruction_type = item.get("instruction_type", "") + answer_position = item.get("answer_position", "") + answer_sheet = item.get("answer_sheet", "") + if answer_position and answer_sheet and "!" not in answer_position: + answer_position_eval = f"{answer_sheet}!{answer_position}" + else: + answer_position_eval = answer_position + + # Determine task_type from instruction_type + itype_lower = (instruction_type or "").lower() + if "cell" in itype_lower: + task_type = "cell_level" + elif "sheet" in itype_lower: + task_type = "sheet_level" + else: + task_type = "other" + + sp = item.get("spreadsheet_path", f"spreadsheet/{task_id}") + task_dir = sp if os.path.isabs(sp) else os.path.join(data_root, sp) + + result = { + "id": task_id, + "ok": False, + "instruction_type": instruction_type, + "task_type": task_type, + "task_description": instruction, + "phase": "setup", + "fail_reason": "", + "agent_ok": False, + "exec_ok": False, + "n_cases": 0, + "n_exec_pass": 0, + "n_pass": 0, + "soft": 0.0, + "hard": 0, + "n_turns": 0, + "cases": [], + "error": "", + } + + try: + cases = _find_test_cases(task_dir) + result["n_cases"] = len(cases) + if not cases: + result["fail_reason"] = "no-test-cases" + return result + + task_out_dir = os.path.join(out_root, "predictions", task_id) + os.makedirs(task_out_dir, exist_ok=True) + + no1, ip1, _ = cases[0] + pred_path_1 = os.path.join(task_out_dir, f"{no1}_pred.xlsx") + target_prompt_parts = [ + f"# Instruction\n{instruction}", + f"# Input file\n{ip1}", + f"# Output file\n{pred_path_1}", + ] + if instruction_type: + target_prompt_parts.append(f"# Instruction type\n{instruction_type}") + if answer_position_eval: + target_prompt_parts.append(f"# Answer position\n{answer_position_eval}") + if diagnostic_trace_context.strip(): + target_prompt_parts.insert( + 0, + "# Previous Codex Trace Snapshot\n" + "This is a partial transcript from an earlier attempt. Use it as your current reasoning context.\n\n" + f"{diagnostic_trace_context.strip()}", + ) + if diagnostic_mode and diagnostic_instruction.strip(): + target_prompt_parts.append(f"# Training readout\n{diagnostic_instruction.strip()}") + target_user_prompt = "\n\n".join(target_prompt_parts) + try: + from skillopt.envs.spreadsheetbench.react_agent import _build_system + target_system_prompt = _build_system(skill_content) + except Exception: + target_system_prompt = "" + if target_system_prompt: + with open(os.path.join(task_out_dir, "target_system_prompt.txt"), "w") as f: + f.write(target_system_prompt) + result["target_system_prompt"] = target_system_prompt + with open(os.path.join(task_out_dir, "target_user_prompt.txt"), "w") as f: + f.write(target_user_prompt) + result["target_user_prompt"] = target_user_prompt + + # ── Stage 1: run ReAct agent on test case 1 ───────────────────── + result["phase"] = "agent" + + work_dir = tempfile.mkdtemp(prefix=f"react_{task_id}_") + try: + # Copy input so agent works in an isolated directory + work_input = os.path.join(work_dir, os.path.basename(ip1)) + shutil.copy2(ip1, work_input) + + agent_result = run_react( + instruction=instruction, + input_path=work_input, + output_path=pred_path_1, + work_dir=work_dir, + instruction_type=instruction_type, + answer_position=answer_position_eval, + skill_content=skill_content, + max_turns=max_turns, + max_output_tokens=max_completion_tokens, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + result["n_turns"] = agent_result.get("n_turns", 0) + if agent_result.get("target_system_prompt"): + with open(os.path.join(task_out_dir, "target_system_prompt.txt"), "w") as f: + f.write(agent_result["target_system_prompt"]) + result["target_system_prompt"] = agent_result["target_system_prompt"] + if agent_result.get("target_user_prompt"): + with open(os.path.join(task_out_dir, "target_user_prompt.txt"), "w") as f: + f.write(agent_result["target_user_prompt"]) + result["target_user_prompt"] = agent_result["target_user_prompt"] + + # Save conversation log + with open(os.path.join(task_out_dir, "conversation.json"), "w") as f: + json.dump( + agent_result.get("conversation", []), + f, ensure_ascii=False, indent=2, + ) + + # Copy solution.py if the agent wrote one + solution_src = os.path.join(work_dir, "solution.py") + solution_dst = os.path.join(task_out_dir, "solution.py") + if os.path.exists(solution_src): + shutil.copy2(solution_src, solution_dst) + + except Exception as e: + result["fail_reason"] = f"agent-error: {type(e).__name__}: {e}" + result["error"] = traceback.format_exc() + return result + finally: + shutil.rmtree(work_dir, ignore_errors=True) + + result["agent_ok"] = True + + # ── Stage 2: evaluate all test cases ───────────────────────────── + result["phase"] = "eval" + solution_path = os.path.join(task_out_dir, "solution.py") + all_exec = True + + for i, (no, ip, ap) in enumerate(cases): + pred_path = os.path.join(task_out_dir, f"{no}_pred.xlsx") + + if i > 0: + # Re-apply solution.py to subsequent test cases + if not os.path.exists(solution_path): + all_exec = False + result["cases"].append( + {"no": no, "stage": "exec", "ok": False, "error": "no-solution-py"} + ) + if not result["fail_reason"]: + result["fail_reason"] = "no-solution-py-for-other-cases" + continue + + with open(solution_path) as f: + code = f.read() + + # Prepend new INPUT_PATH / OUTPUT_PATH + preamble = ( + f"INPUT_PATH = {ip!r}\n" + f"OUTPUT_PATH = {pred_path!r}\n" + ) + full_code = preamble + code + + ok_exec, err = run_generated_code(full_code, ip, pred_path) + if not ok_exec: + all_exec = False + result["cases"].append( + {"no": no, "stage": "exec", "ok": False, "error": err[:500]} + ) + if not result["fail_reason"]: + tail = err.strip().splitlines()[-1][:200] if err.strip() else "unknown" + result["fail_reason"] = f"exec-error: {tail}" + continue + + # ── Evaluate ───────────────────────────────────────────────── + if not os.path.exists(pred_path): + all_exec = False + result["cases"].append( + {"no": no, "stage": "exec", "ok": False, "error": "output-not-found"} + ) + if not result["fail_reason"]: + result["fail_reason"] = "output-not-found" + continue + + result["n_exec_pass"] += 1 + try: + ev = evaluate(pred_path, ap, instruction_type, answer_position_eval) + except Exception as e: # noqa: BLE001 + ev = {"ok": False, "reason": f"eval-exception: {type(e).__name__}: {e}"} + + if ev["ok"]: + result["n_pass"] += 1 + else: + if not result["fail_reason"]: + result["fail_reason"] = f"eval-mismatch: {ev['reason'][:200]}" + result["cases"].append( + {"no": no, "stage": "eval", "ok": ev["ok"], "reason": ev.get("reason", "")} + ) + + result["exec_ok"] = all_exec + n_cases = result["n_cases"] + n_pass = result["n_pass"] + result["soft"] = (n_pass / n_cases) if n_cases else 0.0 + result["hard"] = 1 if (n_cases > 0 and n_pass == n_cases) else 0 + result["ok"] = bool(result["hard"]) + if result["ok"]: + result["fail_reason"] = "" + return result + + except Exception as e: # noqa: BLE001 + result["fail_reason"] = f"unexpected: {type(e).__name__}: {e}" + result["error"] = traceback.format_exc() + return result + + +# ── Batch runner ───────────────────────────────────────────────────────────── + + +def run_spreadsheet_batch( + items: list[dict], + data_root: str, + out_root: str, + skill_content: str, + max_turns: int = 30, + max_completion_tokens: int = 16384, + max_api_workers: int = 64, + task_timeout: int = 600, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context_by_id: dict[str, str] | None = None, +) -> list[dict]: + """Run the ReAct agent on all items with ThreadPoolExecutor. + + Returns list of result dicts compatible with ``compute_score()``. + """ + os.makedirs(out_root, exist_ok=True) + + # Check for already-done items (resume support) + results_path = os.path.join(out_root, "results.jsonl") + done_ids: set[str] = set() + existing: list[dict] = [] + if os.path.exists(results_path): + with open(results_path) as f: + for line in f: + try: + r = json.loads(line) + done_ids.add(str(r["id"])) + existing.append(r) + except Exception: + pass + + pending = [it for it in items if str(it["id"]) not in done_ids] + print( + f" [spreadsheet rollout] total={len(items)} done={len(done_ids)} " + f"pending={len(pending)} workers={max_api_workers} task_timeout={task_timeout}s" + ) + + if not pending: + return existing + + t0 = time.time() + results = list(existing) + started_at: dict[str, float] = {} + + def _timeout_result(item: dict) -> dict: + return { + "id": str(item["id"]), + "ok": False, + "phase": "timeout", + "fail_reason": f"task-timeout-{task_timeout}s", + "n_cases": 0, "n_pass": 0, "soft": 0.0, "hard": 0, + "n_turns": 0, "cases": [], "error": "timeout", + } + + def _error_result(item: dict, exc: Exception) -> dict: + return { + "id": str(item["id"]), + "ok": False, + "phase": "error", + "fail_reason": f"unexpected: {type(exc).__name__}: {exc}", + "n_cases": 0, "n_pass": 0, "soft": 0.0, "hard": 0, + "n_turns": 0, "cases": [], "error": str(exc), + } + + def _run_one(it: dict) -> dict: + started_at[str(it["id"])] = time.time() + return process_one( + it, + data_root, + out_root, + skill_content, + max_turns, + diagnostic_mode, + diagnostic_instruction, + (diagnostic_trace_context_by_id or {}).get(str(it["id"]), ""), + max_completion_tokens, + ) + + ex = ThreadPoolExecutor(max_workers=max_api_workers) + try: + futs = {ex.submit(_run_one, it): it for it in pending} + pending_futs = set(futs) + finished = 0 + while pending_futs: + done, _ = wait(pending_futs, timeout=5, return_when=FIRST_COMPLETED) + now = time.time() + timed_out = [ + fut for fut in pending_futs - done + if str(futs[fut]["id"]) in started_at + and now - started_at[str(futs[fut]["id"])] >= task_timeout + ] + for fut in done: + pending_futs.remove(fut) + item = futs[fut] + try: + res = fut.result() + except FuturesTimeoutError: + res = _timeout_result(item) + except Exception as e: # noqa: BLE001 + res = _error_result(item, e) + results.append(res) + finished += 1 + status = "PASS" if res.get("hard") else ("TIMEOUT" if res.get("phase") == "timeout" else "FAIL") + dt = time.time() - t0 + print( + f" {finished}/{len(pending)} id={res['id']:<10} {status} " + f"turns={res.get('n_turns', 0):<3} " + f"cases={res.get('n_pass', 0)}/{res.get('n_cases', 0)} " + f"dt={dt:.0f}s" + ) + for fut in timed_out: + pending_futs.remove(fut) + res = _timeout_result(futs[fut]) + results.append(res) + finished += 1 + status = "TIMEOUT" + dt = time.time() - t0 + print( + f" {finished}/{len(pending)} id={res['id']:<10} {status} " + f"turns={res.get('n_turns', 0):<3} " + f"cases={res.get('n_pass', 0)}/{res.get('n_cases', 0)} " + f"dt={dt:.0f}s" + ) + finally: + ex.shutdown(wait=False, cancel_futures=True) + + return results + + +# ── Codegen per-task worker (no tool-call) ────────────────────────────────── + + +def process_one_codegen( + item: dict, + data_root: str, + out_root: str, + skill_content: str, + mode: str = "single", + max_turns: int = 5, + max_completion_tokens: int = 16384, + task_timeout: int = 600, + use_eval_feedback: bool = False, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context: str = "", +) -> dict: + """Run codegen agent (single or multi-round) on one SpreadsheetBench task. + + This matches the official evaluation setting: LLM generates a Python code + block, no function-calling / tool-use. + """ + from skillopt.envs.spreadsheetbench.codegen_agent import run_single, run_multi + + task_id = str(item["id"]) + instruction = item["instruction"] + instruction_type = item.get("instruction_type", "") + answer_position = item.get("answer_position", "") + answer_sheet = item.get("answer_sheet", "") + if answer_position and answer_sheet and "!" not in answer_position: + answer_position_eval = f"{answer_sheet}!{answer_position}" + else: + answer_position_eval = answer_position + + itype_lower = (instruction_type or "").lower() + if "cell" in itype_lower: + task_type = "cell_level" + elif "sheet" in itype_lower: + task_type = "sheet_level" + else: + task_type = "other" + + sp = item.get("spreadsheet_path", f"spreadsheet/{task_id}") + task_dir = sp if os.path.isabs(sp) else os.path.join(data_root, sp) + + result = { + "id": task_id, + "ok": False, + "instruction_type": instruction_type, + "task_type": task_type, + "task_description": instruction, + "phase": "setup", + "fail_reason": "", + "llm_ok": False, + "code_ok": False, + "exec_ok": False, + "n_cases": 0, + "n_exec_pass": 0, + "n_pass": 0, + "soft": 0.0, + "hard": 0, + "n_turns": 0, + "cases": [], + "error": "", + } + + try: + cases = _find_test_cases(task_dir) + result["n_cases"] = len(cases) + if not cases: + result["fail_reason"] = "no-test-cases" + return result + + task_out_dir = os.path.join(out_root, "predictions", task_id) + os.makedirs(task_out_dir, exist_ok=True) + + # ── Save context for Optimizer (Reflect stage) ────────────────── + from skillopt.envs.spreadsheetbench.codegen_agent import ( + _preview_workbook, _build_system, _build_user, + ) + first_input_for_preview = cases[0][1] + try: + preview_text = _preview_workbook(first_input_for_preview) + except Exception: + preview_text = "(preview failed)" + target_system = _build_system(skill_content) + target_user = _build_user( + instruction, + first_input_for_preview, + instruction_type, + answer_position_eval, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + + with open(os.path.join(task_out_dir, "spreadsheet_preview.txt"), "w") as f: + f.write(preview_text) + with open(os.path.join(task_out_dir, "target_system_prompt.txt"), "w") as f: + f.write(target_system) + with open(os.path.join(task_out_dir, "target_user_prompt.txt"), "w") as f: + f.write(target_user) + + result["spreadsheet_preview"] = preview_text + result["target_system_prompt"] = target_system + result["target_user_prompt"] = target_user + + # ── LLM phase ────────────────────────────────────────────────── + result["phase"] = "llm" + first_input = cases[0][1] + first_gold = cases[0][2] + first_pred = os.path.join(task_out_dir, f"{cases[0][0]}_pred.xlsx") + + try: + if mode == "multi": + agent_result = run_multi( + instruction=instruction, + input_xlsx=first_input, + output_path=first_pred, + instruction_type=instruction_type, + answer_position=answer_position_eval, + skill_content=skill_content, + max_turns=max_turns, + max_output_tokens=max_completion_tokens, + task_timeout=task_timeout, + gold_path=first_gold if use_eval_feedback else "", + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + else: + agent_result = run_single( + instruction=instruction, + input_xlsx=first_input, + output_path=first_pred, + instruction_type=instruction_type, + answer_position=answer_position_eval, + skill_content=skill_content, + max_output_tokens=max_completion_tokens, + task_timeout=task_timeout, + diagnostic_mode=diagnostic_mode, + diagnostic_instruction=diagnostic_instruction, + diagnostic_trace_context=diagnostic_trace_context, + ) + except Exception as e: # noqa: BLE001 + result["fail_reason"] = f"llm-call-failed: {type(e).__name__}: {e}" + result["error"] = traceback.format_exc() + return result + + result["llm_ok"] = True + result["n_turns"] = agent_result.get("n_turns", 1) + code = agent_result.get("code", "") + raw = agent_result.get("raw", "") + + # Save artifacts + with open(os.path.join(task_out_dir, "code.py"), "w") as f: + f.write(code) + with open(os.path.join(task_out_dir, "raw.txt"), "w") as f: + f.write(raw) + if agent_result.get("conversation"): + with open(os.path.join(task_out_dir, "conversation.json"), "w") as f: + json.dump(agent_result["conversation"], f, ensure_ascii=False, indent=2) + + if not code.strip(): + result["phase"] = "extract" + result["fail_reason"] = "empty-code-block" + return result + result["code_ok"] = True + + # ── Exec + eval per test case ────────────────────────────────── + result["phase"] = "exec" + all_exec = True + # Collect enrichment info for the conversation/trajectory + enrichment_parts: list[str] = [] + + for no, ip, ap in cases: + pred_path = os.path.join(task_out_dir, f"{no}_pred.xlsx") + + # For multi mode, the first case may already be produced + if not os.path.exists(pred_path): + ok_exec, err = run_generated_code(code, ip, pred_path) + if not ok_exec: + all_exec = False + result["cases"].append( + {"no": no, "stage": "exec", "ok": False, "error": err[:500]} + ) + if not result["fail_reason"]: + tail = err.strip().splitlines()[-1][:200] if err.strip() else "unknown" + result["fail_reason"] = f"exec-error: {tail}" + enrichment_parts.append( + f"## Execution (case {no})\nERROR: {err[:500]}" + ) + continue + + if not os.path.exists(pred_path): + all_exec = False + result["cases"].append( + {"no": no, "stage": "exec", "ok": False, "error": "output-not-found"} + ) + if not result["fail_reason"]: + result["fail_reason"] = "output-not-found" + continue + + result["n_exec_pass"] += 1 + try: + ev = evaluate(pred_path, ap, instruction_type, answer_position_eval) + except Exception as e: # noqa: BLE001 + ev = {"ok": False, "reason": f"eval-exception: {type(e).__name__}: {e}"} + + if ev["ok"]: + result["n_pass"] += 1 + else: + if not result["fail_reason"]: + result["fail_reason"] = f"eval-mismatch: {ev['reason'][:200]}" + result["cases"].append( + {"no": no, "stage": "eval", "ok": ev["ok"], "reason": ev.get("reason", "")} + ) + + # Auto-verify: reopen output and compare cells at answer_position + if answer_position_eval: + verify_report = _auto_verify_output(pred_path, ap, answer_position_eval) + enrichment_parts.append( + f"## Eval Result (case {no}): {'PASS' if ev['ok'] else 'FAIL'}\n" + f"{ev.get('reason', '')}\n\n{verify_report}" + ) + + result["exec_ok"] = all_exec + + # ── Enrich conversation with eval details ────────────────────── + if enrichment_parts: + enrichment_msg = "\n\n---\n\n".join(enrichment_parts) + conversation = agent_result.get("conversation", []) + conversation.append({ + "role": "system", + "content": f"[POST-EXECUTION VERIFICATION]\n\n{enrichment_msg}", + }) + # Re-save the enriched conversation + with open(os.path.join(task_out_dir, "conversation.json"), "w") as f: + json.dump(conversation, f, ensure_ascii=False, indent=2) + n_cases = result["n_cases"] + n_pass = result["n_pass"] + result["soft"] = (n_pass / n_cases) if n_cases else 0.0 + result["hard"] = 1 if (n_cases > 0 and n_pass == n_cases) else 0 + result["ok"] = bool(result["hard"]) + if result["ok"]: + result["fail_reason"] = "" + return result + + except Exception as e: # noqa: BLE001 + result["fail_reason"] = f"unexpected: {type(e).__name__}: {e}" + result["error"] = traceback.format_exc() + return result + + +# ── Codegen batch runner ──────────────────────────────────────────────────── + + +def run_spreadsheet_batch_codegen( + items: list[dict], + data_root: str, + out_root: str, + skill_content: str, + mode: str = "single", + max_turns: int = 5, + max_completion_tokens: int = 16384, + max_api_workers: int = 32, + task_timeout: int = 0, + use_eval_feedback: bool = False, + diagnostic_mode: bool = False, + diagnostic_instruction: str = "", + diagnostic_trace_context_by_id: dict[str, str] | None = None, +) -> list[dict]: + """Run codegen agent on all items (no tool-call). + + Args: + mode: "single" or "multi". + task_timeout: Hard per-task timeout in seconds at the future level. + 0 or negative disables the per-task timeout. + """ + no_task_timeout = task_timeout <= 0 + task_timeout_label = "none" if no_task_timeout else f"{task_timeout}s" + + os.makedirs(out_root, exist_ok=True) + + results_path = os.path.join(out_root, "results.jsonl") + done_ids: set[str] = set() + existing: list[dict] = [] + if os.path.exists(results_path): + with open(results_path) as f: + for line in f: + try: + r = json.loads(line) + done_ids.add(str(r["id"])) + existing.append(r) + except Exception: + pass + + pending = [it for it in items if str(it["id"]) not in done_ids] + print( + f" [spreadsheet codegen-{mode}] total={len(items)} done={len(done_ids)} " + f"pending={len(pending)} workers={max_api_workers} task_timeout={task_timeout_label}" + ) + + if not pending: + return existing + + t0 = time.time() + results = list(existing) + + started_at: dict[str, float] = {} + + def _run_one(it: dict) -> dict: + started_at[str(it["id"])] = time.time() + return process_one_codegen( + it, + data_root, + out_root, + skill_content, + mode, + max_turns, + max_completion_tokens, + task_timeout, + use_eval_feedback, + diagnostic_mode, + diagnostic_instruction, + (diagnostic_trace_context_by_id or {}).get(str(it["id"]), ""), + ) + + def _timeout_result(item: dict) -> dict: + return { + "id": str(item["id"]), + "ok": False, + "instruction_type": item.get("instruction_type", ""), + "task_type": "other", + "phase": "timeout", + "fail_reason": f"task-timeout-{task_timeout}s", + "n_cases": 0, "n_pass": 0, "soft": 0.0, "hard": 0, + "n_turns": 0, "cases": [], "error": "timeout", + } + + def _error_result(item: dict, e: Exception) -> dict: + return { + "id": str(item["id"]), + "ok": False, + "instruction_type": item.get("instruction_type", ""), + "task_type": "other", + "phase": "error", + "fail_reason": f"unexpected: {type(e).__name__}: {e}", + "n_cases": 0, "n_pass": 0, "soft": 0.0, "hard": 0, + "n_turns": 0, "cases": [], "error": str(e), + } + + def _record(res: dict, i: int) -> None: + results.append(res) + status = "PASS" if res.get("hard") else ("TIMEOUT" if res.get("phase") == "timeout" else "FAIL") + dt = time.time() - t0 + print( + f" {i}/{len(pending)} id={res['id']:<10} {status} " + f"turns={res.get('n_turns', 0):<3} " + f"cases={res.get('n_pass', 0)}/{res.get('n_cases', 0)} " + f"dt={dt:.0f}s" + ) + + ex = ThreadPoolExecutor(max_workers=max_api_workers) + try: + futs = {ex.submit(_run_one, it): it for it in pending} + pending_futs = set(futs) + finished = 0 + while pending_futs: + done, _ = wait(pending_futs, timeout=5, return_when=FIRST_COMPLETED) + now = time.time() + timed_out = [] if no_task_timeout else [ + fut for fut in pending_futs - done + if str(futs[fut]["id"]) in started_at + and now - started_at[str(futs[fut]["id"])] >= task_timeout + ] + for fut in done: + pending_futs.remove(fut) + item = futs[fut] + try: + res = fut.result() + except FuturesTimeoutError: + res = _timeout_result(item) + except Exception as e: # noqa: BLE001 + res = _error_result(item, e) + finished += 1 + _record(res, finished) + for fut in timed_out: + pending_futs.remove(fut) + fut.cancel() + finished += 1 + _record(_timeout_result(futs[fut]), finished) + finally: + ex.shutdown(wait=False, cancel_futures=True) + + return results diff --git a/skillopt/envs/spreadsheetbench/skills/initial.md b/skillopt/envs/spreadsheetbench/skills/initial.md new file mode 100644 index 0000000..17eb7cd --- /dev/null +++ b/skillopt/envs/spreadsheetbench/skills/initial.md @@ -0,0 +1,56 @@ +# Spreadsheet Manipulation Skill (xlsx) + +## Overview +This skill guides agents in manipulating Excel (.xlsx) spreadsheets using Python. + +**Primary libraries**: `openpyxl` (structure-preserving read/write), `pandas` (data transformation). +Never use any other third-party libraries. + +--- + +## Common Workflow + +1. **Explore** the input file: list sheets, inspect headers, check dimensions. +2. **Write `solution.py`** with `INPUT_PATH` and `OUTPUT_PATH` defined at the top. +3. **Execute** `python solution.py` and verify the output file was created. +4. **Confirm** the target cells/range contain the expected values. + +--- + +## Library Selection + +| Use case | Library | +|----------|---------| +| Preserve formulas, formatting, named ranges | `openpyxl` | +| Bulk data transformation, aggregation, sorting | `pandas` → write back with `openpyxl` | +| Simple cell read/write | `openpyxl` | + +**Warning**: `pandas.to_excel()` silently destroys existing formulas and named ranges. +When writing back to a spreadsheet that contains formulas, always use `openpyxl.save()`. + +--- + +## solution.py Template + +```python +import openpyxl +import pandas as pd + +INPUT_PATH = "..." # set to the actual input path +OUTPUT_PATH = "..." # set to the actual output path + +wb = openpyxl.load_workbook(INPUT_PATH) +ws = wb.active # or wb["SheetName"] + +# --- perform manipulation --- + +wb.save(OUTPUT_PATH) +``` + +--- + +## Output Requirements + +- Save the result to `OUTPUT_PATH`. +- Do not hardcode row counts or column letters — iterate over actual rows in the workbook. +- Preserve sheets and cells not mentioned in the instruction. diff --git a/skillopt/evaluation/__init__.py b/skillopt/evaluation/__init__.py new file mode 100644 index 0000000..bb89670 --- /dev/null +++ b/skillopt/evaluation/__init__.py @@ -0,0 +1,13 @@ +"""ReflACT Evaluation -- candidate skill validation and model selection. + +Analogous to validation-based early stopping and model selection in neural +network training: evaluates candidate skills on held-out selection sets and +decides whether to accept or reject proposed updates. +""" +from skillopt.evaluation.gate import ( # noqa: F401 + GateAction, + GateMetric, + GateResult, + evaluate_gate, + select_gate_score, +) diff --git a/skillopt/evaluation/gate.py b/skillopt/evaluation/gate.py new file mode 100644 index 0000000..10461a9 --- /dev/null +++ b/skillopt/evaluation/gate.py @@ -0,0 +1,225 @@ +"""Validation gate — accept / reject candidate skills. + +Analogous to validation-based early stopping and model selection in neural +network training: compares the candidate's score against the current and +best scores, then returns an accept/reject decision. + +The trainer owns side-effects (cache lookup, rollout, printing, state +mutation). This module is the pure decision function. + +Metric selection +---------------- +Three gate metrics are supported: + +* ``"hard"`` (default, backward-compatible): + Compare candidate vs current/best using *hard* exact-match accuracy. +* ``"soft"``: + Compare using *soft* per-item score (F1 / partial credit / etc.). + Use this when a small held-out selection set has too few items for + hard accuracy to be sensitive to incremental skill improvements. +* ``"mixed"``: + Compare using a weighted average ``(1 - w) * hard + w * soft``. + ``w`` is configurable via ``mixed_weight`` (default ``0.5``). +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + + +GateAction = Literal["accept_new_best", "accept", "reject"] +GateMetric = Literal["hard", "soft", "mixed"] + + +@dataclass(frozen=True) +class GateResult: + """Immutable outcome of the validation gate.""" + + action: GateAction + current_skill: str + current_score: float + best_skill: str + best_score: float + best_step: int + + +def compute_semantic_density( + skill_content: str, + leading_words: list[str] | None = None, +) -> float: + """Compute the semantic density of leading words in a skill document.""" + if not skill_content or not skill_content.strip(): + return 0.0 + if leading_words is None: + leading_words = [ + "MUST", "ALWAYS", "NEVER", "ONLY", "CRITICAL", "IMPORTANT", + "RESOLVE", "PREFER", "ENSURE", "STRICT", "VERIFY" + ] + + # Strip metadata comments to focus purely on instruction text + skill = skill_content + for start, end in [ + ("", ""), + ("", "") + ]: + while True: + s_idx = skill.find(start) + if s_idx == -1: + break + e_idx = skill.find(end, s_idx) + if e_idx == -1: + skill = skill[:s_idx] + skill[s_idx + len(start):] + break + skill = skill[:s_idx] + skill[e_idx + len(end):] + + import re + words = re.findall(r'[a-zA-Z0-9]+', skill.lower()) + if not words: + return 0.0 + + leading_set = {w.lower() for w in leading_words} + leading_count = sum(1 for w in words if w in leading_set) + return leading_count / len(words) + + +def select_gate_score( + hard: float, + soft: float, + metric: GateMetric = "hard", + mixed_weight: float = 0.5, + *, + skill_content: str = "", + use_semantic_density: bool = False, + semantic_density_weight: float = 0.05, + leading_words: list[str] | None = None, +) -> float: + """Project (hard, soft) onto a single comparison metric. + + Parameters + ---------- + hard, soft + Aggregate hard / soft scores from a rollout batch (both 0..1). + metric + Which metric to compare on. + mixed_weight + For ``"mixed"``: weight given to ``soft``. Must be in ``[0, 1]``. + Ignored for ``"hard"`` / ``"soft"``. + skill_content + The raw skill document content. + use_semantic_density + Whether to adjust the score based on semantic density of leading words. + semantic_density_weight + Scaling weight for the semantic density bonus. + leading_words + Optional custom list of high-influence words to prioritize. + """ + if metric == "hard": + score = float(hard) + elif metric == "soft": + score = float(soft) + elif metric == "mixed": + w = max(0.0, min(1.0, float(mixed_weight))) + score = (1.0 - w) * float(hard) + w * float(soft) + else: + raise ValueError( + f"unknown gate metric {metric!r}; expected 'hard', 'soft', or 'mixed'" + ) + + if use_semantic_density: + density = compute_semantic_density(skill_content, leading_words) + score += float(semantic_density_weight) * density + + return score + + +def evaluate_gate( + candidate_skill: str, + cand_hard: float, + current_skill: str, + current_score: float, + best_skill: str, + best_score: float, + best_step: int, + global_step: int, + *, + cand_soft: float = 0.0, + metric: GateMetric = "hard", + mixed_weight: float = 0.5, + use_semantic_density: bool = False, + semantic_density_weight: float = 0.05, + leading_words: list[str] | None = None, +) -> GateResult: + """Pure gate decision: compare candidate score to current/best. + + Parameters + ---------- + candidate_skill + The candidate skill content being evaluated. + cand_hard, cand_soft + Aggregate hard / soft scores of the candidate on the selection set. + current_skill, current_score + The currently-active skill and its *metric-space* score. + best_skill, best_score, best_step + The best-so-far skill, its *metric-space* score, and the step + at which it was accepted. + global_step + Current global training step (recorded if a new best is accepted). + cand_soft + Soft score of the candidate; only consulted when ``metric != "hard"``. + Defaults to ``0.0`` for backward compatibility with callers that + previously passed only ``cand_hard``. + metric + Which metric to compare on. Defaults to ``"hard"`` to preserve + the original gate behavior. + mixed_weight + Weight on ``soft`` when ``metric == "mixed"``. + use_semantic_density + Whether to adjust the score based on semantic density of leading words. + semantic_density_weight + Scaling weight for the semantic density bonus. + leading_words + Optional custom list of high-influence words to prioritize. + + Returns + ------- + GateResult + Updated state; the caller decides what to do with it (print, + mutate trainer state, log, etc.). + """ + cand_score = select_gate_score( + cand_hard, + cand_soft, + metric, + mixed_weight, + skill_content=candidate_skill, + use_semantic_density=use_semantic_density, + semantic_density_weight=semantic_density_weight, + leading_words=leading_words, + ) + + if cand_score > current_score: + if cand_score > best_score: + return GateResult( + action="accept_new_best", + current_skill=candidate_skill, + current_score=cand_score, + best_skill=candidate_skill, + best_score=cand_score, + best_step=global_step, + ) + return GateResult( + action="accept", + current_skill=candidate_skill, + current_score=cand_score, + best_skill=best_skill, + best_score=best_score, + best_step=best_step, + ) + return GateResult( + action="reject", + current_skill=current_skill, + current_score=current_score, + best_skill=best_skill, + best_score=best_score, + best_step=best_step, + ) diff --git a/skillopt/gradient/__init__.py b/skillopt/gradient/__init__.py new file mode 100644 index 0000000..0b05ef8 --- /dev/null +++ b/skillopt/gradient/__init__.py @@ -0,0 +1,15 @@ +"""SkillOpt Gradient -- trajectory analysis and patch generation. + +Analogous to gradient computation in neural network training: analyzes +minibatch rollout trajectories to produce skill-edit patches (the "gradient" +that drives skill updates). + +Modules +------- +- reflect: minibatch trajectory analysis (gradient computation) +- aggregate: hierarchical patch merging (gradient aggregation) +""" +from skillopt.gradient.reflect import ( # noqa: F401 + run_minibatch_reflect, +) +from skillopt.gradient.aggregate import merge_patches # noqa: F401 diff --git a/skillopt/gradient/aggregate.py b/skillopt/gradient/aggregate.py new file mode 100644 index 0000000..841f08f --- /dev/null +++ b/skillopt/gradient/aggregate.py @@ -0,0 +1,253 @@ +"""ReflACT Aggregate stage — hierarchical patch merging. + +The Aggregate stage takes independently-generated patches from the Reflect +stage and merges them into a single coherent patch via hierarchical LLM calls. +Failure-driven patches take priority over success-driven ones. +""" +from __future__ import annotations + +import json +from concurrent.futures import ThreadPoolExecutor, as_completed + +from skillopt.model import chat_optimizer +from skillopt.optimizer.meta_skill import format_meta_skill_context +from skillopt.optimizer.update_modes import ( + get_payload_items, + is_full_rewrite_minibatch_mode, + is_rewrite_mode, + normalize_update_mode, + payload_key, + payload_label, +) +from skillopt.prompts import load_prompt +from skillopt.utils import extract_json + + +# ── Internal helpers ────────────────────────────────────────────────────────── + +def _merge_batch( + skill_content: str, + patches: list[dict], + system_prompt: str, + update_mode: str, + meta_skill_context: str = "", + level: int = 1, +) -> dict: + """Call optimizer LLM to merge a batch of patches into one.""" + patches_text = json.dumps(patches, ensure_ascii=False, indent=2) + user = ( + f"## Current Skill\n{skill_content}\n\n" + f"## Patches to merge ({len(patches)} total, merge level {level})\n{patches_text}" + ) + optimizer_ctx = format_meta_skill_context(meta_skill_context) + if optimizer_ctx: + user = f"{optimizer_ctx}\n\n{user}" + try: + response, _ = chat_optimizer( + system=system_prompt, + user=user, + max_completion_tokens=64000 if is_full_rewrite_minibatch_mode(update_mode) else 16384, + retries=3, + stage="merge", + ) + merged = extract_json(response) + key = payload_key(update_mode) + if merged and key in merged: + for e in merged.get(key, []): + e["merge_level"] = level + return merged + except Exception: # noqa: BLE001 + pass + # Fallback: concatenate all edits + all_edits = [] + for p in patches: + for e in get_payload_items(p, update_mode): + e.setdefault("merge_level", level) + all_edits.append(e) + return {"reasoning": "fallback concatenation", payload_key(update_mode): all_edits} + + +def _hierarchical_merge( + skill_content: str, + patches: list[dict], + system_prompt: str, + update_mode: str, + batch_size: int, + verbose: bool, + label: str = "", + workers: int = 16, + meta_skill_context: str = "", +) -> dict: + """Hierarchically merge N patches using the given system prompt. + + Same-level batches are executed in PARALLEL via ThreadPoolExecutor. + """ + if not patches: + return {"reasoning": "no patches", payload_key(update_mode): []} + if len(patches) == 1: + return patches[0] + + current = list(patches) + level = 0 + while len(current) > 1: + level += 1 + batches: list[tuple[int, list[dict]]] = [] + for i in range(0, len(current), batch_size): + batch = current[i : i + batch_size] + batches.append((i, batch)) + + if verbose: + print( + f" [aggregate {label}] level={level} " + f"{len(current)} patches → {len(batches)} batches " + f"(parallel, batch_size={batch_size})" + ) + + next_level: list[dict | None] = [None] * len(batches) + + to_merge: list[tuple[int, list[dict]]] = [] + for idx, (i, batch) in enumerate(batches): + if len(batch) == 1: + next_level[idx] = batch[0] + else: + to_merge.append((idx, batch)) + + if to_merge: + with ThreadPoolExecutor(max_workers=workers) as ex: + futs = { + ex.submit( + _merge_batch, skill_content, batch, system_prompt, update_mode, + meta_skill_context, level, + ): idx + for idx, batch in to_merge + } + for fut in as_completed(futs): + idx = futs[fut] + next_level[idx] = fut.result() + if verbose: + batch_i, batch_data = batches[idx] + n_edits = len(get_payload_items(next_level[idx], update_mode)) + print( + f" [aggregate {label}] level={level} " + f"batch [{batch_i}:{batch_i+len(batch_data)}] " + f"→ 1 patch ({n_edits} {payload_label(update_mode)})" + ) + + current = [x for x in next_level if x is not None] + + return current[0] + + +# ── Public API ──────────────────────────────────────────────────────────────── + +def merge_patches( + skill_content: str, + failure_patches: list[dict], + success_patches: list[dict], + batch_size: int = 8, + verbose: bool = True, + workers: int = 16, + update_mode: str = "patch", + meta_skill_context: str = "", +) -> dict: + """Failure-first hierarchical merge with support count tracking. + + 1. Merge failure patches independently (parallel) + 2. Merge success patches independently (parallel) + 3. Final merge: combine both groups with failure priority + + Returns a merged :class:`~skillopt.types.Patch` dict (``edits`` + ``reasoning``). + """ + if verbose: + print( + f" [3/6 AGGREGATE] " + f"failure={len(failure_patches)} success={len(success_patches)} " + f"(parallel, workers={workers})" + ) + + update_mode = normalize_update_mode(update_mode) + if is_full_rewrite_minibatch_mode(update_mode): + merge_failure_prompt = load_prompt("merge_failure_full_rewrite") + merge_success_prompt = load_prompt("merge_success_full_rewrite") + merge_final_prompt = load_prompt("merge_final_full_rewrite") + elif is_rewrite_mode(update_mode): + merge_failure_prompt = load_prompt("merge_failure_rewrite") + merge_success_prompt = load_prompt("merge_success_rewrite") + merge_final_prompt = load_prompt("merge_final_rewrite") + else: + merge_failure_prompt = load_prompt("merge_failure") + merge_success_prompt = load_prompt("merge_success") + merge_final_prompt = load_prompt("merge_final") + + failure_merged = _hierarchical_merge( + skill_content, failure_patches, merge_failure_prompt, update_mode, + batch_size, verbose, label="failure", workers=workers, + meta_skill_context=meta_skill_context, + ) + + success_merged = _hierarchical_merge( + skill_content, success_patches, merge_success_prompt, update_mode, + batch_size, verbose, label="success", workers=workers, + meta_skill_context=meta_skill_context, + ) + + f_edits = get_payload_items(failure_merged, update_mode) + s_edits = get_payload_items(success_merged, update_mode) + + if not f_edits and not s_edits: + return {"reasoning": "no updates from either group", payload_key(update_mode): []} + if not s_edits: + return failure_merged + if not f_edits: + return success_merged + + combined_patches = [failure_merged, success_merged] + combined_text = json.dumps(combined_patches, ensure_ascii=False, indent=2) + if is_full_rewrite_minibatch_mode(update_mode): + item_label = payload_label(update_mode) + user = ( + f"## Current Skill\n{skill_content}\n\n" + f"## Two pre-merged candidate groups to combine\n" + f"Group 1 (from failed trajectories): " + f"{len(f_edits)} {item_label}\n" + f"Group 2 (from successful trajectories): " + f"{len(s_edits)} {item_label}\n\n" + f"{combined_text}" + ) + else: + user = ( + f"## Current Skill\n{skill_content}\n\n" + f"## Two pre-merged patch groups to combine\n" + f"Group 1 (failure-driven, HIGH priority): " + f"{len(f_edits)} edits\n" + f"Group 2 (success-driven, lower priority): " + f"{len(s_edits)} edits\n\n" + f"{combined_text}" + ) + optimizer_ctx = format_meta_skill_context(meta_skill_context) + if optimizer_ctx: + user = f"{optimizer_ctx}\n\n{user}" + try: + response, _ = chat_optimizer( + system=merge_final_prompt, + user=user, + max_completion_tokens=64000 if is_full_rewrite_minibatch_mode(update_mode) else 16384, + retries=3, + stage="merge", + ) + final = extract_json(response) + key = payload_key(update_mode) + if final and key in final: + if verbose: + print( + f" [aggregate final] " + f"{len(f_edits)}+{len(s_edits)} → {len(final[key])} {payload_label(update_mode)}" + ) + return final + except Exception: # noqa: BLE001 + pass + + return { + "reasoning": "fallback: failure first, then success", + payload_key(update_mode): f_edits + s_edits, + } diff --git a/skillopt/gradient/reflect.py b/skillopt/gradient/reflect.py new file mode 100644 index 0000000..8078f85 --- /dev/null +++ b/skillopt/gradient/reflect.py @@ -0,0 +1,635 @@ +"""ReflACT core Reflect engine -- minibatch trajectory analysis. + +Provides environment-agnostic minibatch trajectory analysis: instead of +analyzing each trajectory independently, trajectories are grouped into +minibatches of size M and analyzed together -- analogous to minibatch SGD +vs per-sample SGD in neural network training. + +Two-level prompt priority system: + +1. **Custom prompt** (adapter returns non-None) -- used as-is. +2. **Generic default prompt** (adapter returns None) -- built-in defaults + that work for any environment without configuration. + +Public API +---------- +- :func:`fmt_trajectory` -- format one conversation into text +- :func:`fmt_minibatch_trajectories` -- format multiple trajectories for batch analysis +- :func:`run_error_analyst_minibatch` -- one optimizer call for a group of failures +- :func:`run_success_analyst_minibatch` -- one optimizer call for a group of successes +- :func:`run_minibatch_reflect` -- full reflect stage dispatcher +""" +from __future__ import annotations + +import json +import os +import random +import traceback +from concurrent.futures import ThreadPoolExecutor, as_completed + +from skillopt.model import chat_optimizer +from skillopt.optimizer.meta_skill import format_meta_skill_context +from skillopt.optimizer.skill_aware import ( + augment_error_prompt, + augment_success_prompt, + extract_appendix_notes, + get_skill_aware_appendix_source, + is_skill_aware_enabled, +) +from skillopt.optimizer.update_modes import ( + get_payload_items, + is_full_rewrite_minibatch_mode, + normalize_update_mode, + payload_key, + payload_label, + truncate_payload, +) +from skillopt.prompts import load_prompt +from skillopt.utils import extract_json + + +# ── Trajectory formatting ──────────────────────────────────────────────────── + + +def _clip_text(value, limit: int | None = None) -> str: + """Render optional trajectory fields. Truncation is disabled: the optimizer + is given the full content so it can see exactly what the agent saw/did. + + ``limit`` is accepted for backward compatibility but ignored. + """ + if value is None: + return "" + return str(value) + + +def fmt_trajectory( + conversation: list[dict], + max_chars: int | None = None, +) -> str: + """Format a conversation list into analyst-readable text. + + Accepts two common formats: + + 1. Tool-call records: ``{"type": "tool_call", "cmd": ..., "obs": ...}`` + 2. Step records: ``{"step": N, "action": ..., "env_feedback": ..., "reasoning": ...}`` + + Any other dict is rendered via its ``"content"`` key. + """ + lines: list[str] = [] + for item in conversation: + if not isinstance(item, dict): + lines.append(f"[agent] {_clip_text(item)}") + continue + if item.get("type") == "tool_call": + cmd = _clip_text(item.get("cmd")) + obs = _clip_text(item.get("obs")) + lines.append(f"[action] {cmd}") + lines.append(f"[obs] {obs}") + elif "action" in item and "env_feedback" in item: + step = item.get("step", "?") + reasoning = _clip_text(item.get("reasoning")) + action = _clip_text(item.get("action")) + feedback = _clip_text(item.get("env_feedback")) + if reasoning: + lines.append(f"[step {step} think] {reasoning}") + lines.append(f"[step {step} action] {action}") + lines.append(f"[step {step} obs] {feedback}") + elif item.get("role") == "system": + # Post-execution verification / enrichment info + msg = _clip_text(item.get("content")) + lines.append(f"[verification] {msg}") + else: + msg = _clip_text(item.get("content")) + role = item.get("role", "agent") + lines.append(f"[{role}] {msg}") + + return "\n".join(lines) + + +# ── Minibatch trajectory formatting ────────────────────────────────────────── + + +def fmt_minibatch_trajectories( + items: list[dict], + prediction_dir: str, +) -> str: + """Format multiple trajectories for minibatch analyst consumption. + + Each item is a rollout result dict with ``"id"``, ``"task_description"``, + ``"task_type"``, ``"fail_reason"``, etc. Reads ``conversation.json`` + for each and formats them together with trajectory headers. + + If available, includes the spreadsheet preview and target system prompt + so the analyst can see what the agent saw. + + Parameters + ---------- + items : list[dict] + Rollout result dicts belonging to one minibatch. + prediction_dir : str + Path to ``predictions/`` directory containing per-task + ``/conversation.json`` files. + + Returns + ------- + str + Formatted text with all trajectories separated by ``---``. + """ + parts: list[str] = [] + for idx, item in enumerate(items, 1): + tid = str(item["id"]) + conv_path = os.path.join(prediction_dir, tid, "conversation.json") + if not os.path.exists(conv_path): + continue + with open(conv_path) as f: + conversation = json.load(f) + if not conversation: + continue + + traj_text = fmt_trajectory(conversation) + header = ( + f"### Trajectory {idx} (id={tid})\n" + f"Task: {item.get('task_description', item.get('instruction', ''))}\n" + f"Task type: {item.get('task_type', item.get('instruction_type', ''))}\n" + ) + fail_reason = item.get("fail_reason", "") + if fail_reason: + header += f"Failure reason: {fail_reason}\n" + header += f"Steps: {item.get('n_turns', '?')}\n" + + reference_text = str(item.get("reference_text") or "").strip() + if reference_text: + header += ( + f"\n#### Hidden Reference\n" + f"{reference_text}\n" + ) + + # ── Append target context (what the agent saw) ────────────── + target_prompt = item.get("target_system_prompt", "") + if not target_prompt: + prompt_path = os.path.join(prediction_dir, tid, "target_system_prompt.txt") + if os.path.exists(prompt_path): + with open(prompt_path) as f: + target_prompt = f.read() + if target_prompt: + header += ( + f"\n#### Target System Prompt\n" + f"{target_prompt}\n" + ) + + user_prompt = item.get("target_user_prompt", "") + if not user_prompt: + user_prompt_path = os.path.join(prediction_dir, tid, "target_user_prompt.txt") + if os.path.exists(user_prompt_path): + with open(user_prompt_path) as f: + user_prompt = f.read() + if user_prompt: + header += ( + f"\n#### Target User Prompt\n" + f"{user_prompt}\n" + ) + + if os.environ.get("REFLACT_CODEX_TRACE_TO_OPTIMIZER", "0") == "1": + codex_trace_summary = item.get("codex_trace_summary", "") + if not codex_trace_summary: + codex_trace_summary_path = os.path.join(prediction_dir, tid, "codex_trace_summary.txt") + if os.path.exists(codex_trace_summary_path): + with open(codex_trace_summary_path) as f: + codex_trace_summary = f.read() + if codex_trace_summary: + header += ( + f"\n#### Codex Trace Summary\n" + f"{codex_trace_summary}\n" + ) + + codex_probe_trace_steps = str(item.get("codex_probe_trace_steps") or "").strip() + if codex_probe_trace_steps: + header += ( + f"\n#### Codex Trace Steps\n" + f"{codex_probe_trace_steps}\n" + ) + + preview = item.get("spreadsheet_preview", "") + if not preview: + preview_path = os.path.join(prediction_dir, tid, "spreadsheet_preview.txt") + if os.path.exists(preview_path): + with open(preview_path) as f: + preview = f.read() + if preview: + header += ( + f"\n#### Spreadsheet Preview\n" + f"{preview}\n" + ) + + parts.append(header + "\n" + traj_text) + + return "\n\n---\n\n".join(parts) + + +# ── Prompt resolution ─────────────────────────────────────────────────────── + + +def _resolve_prompt(custom: str | None, default_name: str, update_mode: str = "patch") -> str: + """Return *custom* if provided (non-None), otherwise load from file.""" + if custom is not None: + return custom + mode = normalize_update_mode(update_mode) + actual_name = default_name + if is_full_rewrite_minibatch_mode(mode): + full_name = f"{default_name}_full_rewrite" + try: + return load_prompt(full_name) + except FileNotFoundError: + actual_name = default_name + elif mode == "rewrite_from_suggestions": + rewrite_name = f"{default_name}_rewrite" + try: + return load_prompt(rewrite_name) + except FileNotFoundError: + actual_name = default_name + return load_prompt(actual_name) + + +# ── Minibatch analysts ────────────────────────────────────────────────────── + + +def run_error_analyst_minibatch( + skill_content: str, + items: list[dict], + prediction_dir: str, + edit_budget: int = 4, + *, + system_prompt: str | None = None, + rejection_context: str = "", + trajectory_memory_context: str = "", + step_buffer_context: str = "", + meta_skill_context: str = "", + update_mode: str = "patch", + skill_aware_reflection: bool = False, +) -> dict | None: + """Analyze a minibatch of failed trajectories in one optimizer call. + + Parameters + ---------- + skill_content : str + Current skill document text. + items : list[dict] + Rollout result dicts (all should have ``hard=0``). + prediction_dir : str + Path to ``predictions/`` directory. + edit_budget : int + Maximum number of edits (L). + system_prompt : str | None + Custom system prompt. ``None`` = use generic default. + rejection_context : str + *Deprecated* — use ``step_buffer_context``. + trajectory_memory_context : str + *Deprecated* — use ``step_buffer_context``. + step_buffer_context : str + Unified summary of previous steps (failure patterns + rejected edits). + + Returns + ------- + dict | None + Patch dict with ``source_type="failure"``, or ``None`` on error. + """ + mode = normalize_update_mode(update_mode) + actual_system = _resolve_prompt(system_prompt, "analyst_error", mode) + # Skill-aware reflection: augment the resolved prompt at runtime so both + # env-specific and generic analyst prompts get the defect/lapse instruction. + # When the toggle is off this is a no-op (prompt byte-identical to baseline). + if skill_aware_reflection and not is_full_rewrite_minibatch_mode(mode): + actual_system = augment_error_prompt(actual_system) + + trajectories_text = fmt_minibatch_trajectories(items, prediction_dir) + if not trajectories_text.strip(): + return None + + user = ( + f"## Current Skill\n{skill_content}\n\n" + ) + if is_full_rewrite_minibatch_mode(mode): + user += ( + f"## Update Format\n" + f"Produce one complete replacement skill candidate for this minibatch. " + f"Do not output edits, patches, or revise suggestions.\n\n" + ) + else: + user += ( + f"## {payload_label(mode, title=True)} Budget\n" + f"Produce at most L={edit_budget} {payload_label(mode)}.\n\n" + ) + # Unified step buffer context (preferred) + ctx = step_buffer_context or rejection_context or "" + if trajectory_memory_context: + ctx = f"{ctx}\n{trajectory_memory_context}" if ctx else trajectory_memory_context + if ctx.strip(): + user += f"## Previous Steps in This Epoch\n{ctx}\n\n" + optimizer_ctx = format_meta_skill_context(meta_skill_context) + if optimizer_ctx: + user += optimizer_ctx + "\n\n" + user += f"## Failed Trajectories ({len(items)} total)\n{trajectories_text}" + + try: + response, _ = chat_optimizer( + system=actual_system, user=user, + max_completion_tokens=64000 if is_full_rewrite_minibatch_mode(mode) else 16384, + retries=3, + stage="analyst", + ) + result = extract_json(response) + if not result: + return None + notes = extract_appendix_notes(result) if skill_aware_reflection else [] + if "patch" in result: + result["source_type"] = "failure" + if not is_full_rewrite_minibatch_mode(mode): + truncate_payload(result["patch"], edit_budget, mode) + if skill_aware_reflection: + result["appendix_notes"] = notes + return result + # Skill-aware: a batch may legitimately yield ONLY execution-lapse notes + # (no body edit). Return a no-op patch so the notes still reach the + # trainer via all_raw_patches; empty edits are dropped from the body + # pipeline by _normalise_patches, so body behavior is unchanged. + if skill_aware_reflection and notes: + return { + "source_type": "failure", + "patch": {"reasoning": "execution-lapse only", "edits": []}, + "appendix_notes": notes, + } + except Exception: # noqa: BLE001 + traceback.print_exc() + return None + + +def run_success_analyst_minibatch( + skill_content: str, + items: list[dict], + prediction_dir: str, + edit_budget: int = 4, + *, + system_prompt: str | None = None, + trajectory_memory_context: str = "", + step_buffer_context: str = "", + meta_skill_context: str = "", + update_mode: str = "patch", + skill_aware_reflection: bool = False, + emit_appendix_notes: bool = True, +) -> dict | None: + """Analyze a minibatch of successful trajectories in one optimizer call. + + Parameters + ---------- + system_prompt : str | None + Custom system prompt. ``None`` = use generic default. + trajectory_memory_context : str + *Deprecated* — use ``step_buffer_context``. + step_buffer_context : str + Unified summary of previous steps (failure patterns + rejected edits). + + Returns + ------- + dict | None + Patch dict with ``source_type="success"``, or ``None`` on error. + """ + mode = normalize_update_mode(update_mode) + actual_system = _resolve_prompt(system_prompt, "analyst_success", mode) + # Only augment + parse appendix notes on the success side when allowed. + # failure_only mode (paper-faithful S_app) suppresses success-side notes. + sa_emit = skill_aware_reflection and emit_appendix_notes + if sa_emit and not is_full_rewrite_minibatch_mode(mode): + actual_system = augment_success_prompt(actual_system) + + trajectories_text = fmt_minibatch_trajectories(items, prediction_dir) + if not trajectories_text.strip(): + return None + + user = ( + f"## Current Skill\n{skill_content}\n\n" + ) + if is_full_rewrite_minibatch_mode(mode): + user += ( + f"## Update Format\n" + f"Produce one complete replacement skill candidate for this minibatch. " + f"Do not output edits, patches, or revise suggestions.\n\n" + ) + else: + user += ( + f"## {payload_label(mode, title=True)} Budget\n" + f"Produce at most L={edit_budget} {payload_label(mode)}.\n\n" + ) + ctx = step_buffer_context or trajectory_memory_context or "" + if ctx.strip(): + user += f"## Previous Steps in This Epoch\n{ctx}\n\n" + optimizer_ctx = format_meta_skill_context(meta_skill_context) + if optimizer_ctx: + user += optimizer_ctx + "\n\n" + user += f"## Successful Trajectories ({len(items)} total)\n{trajectories_text}" + + try: + response, _ = chat_optimizer( + system=actual_system, user=user, + max_completion_tokens=64000 if is_full_rewrite_minibatch_mode(mode) else 16384, + retries=3, + stage="analyst", + ) + result = extract_json(response) + if result and "patch" in result: + result["source_type"] = "success" + if not is_full_rewrite_minibatch_mode(mode): + truncate_payload(result["patch"], edit_budget, mode) + if sa_emit: + result["appendix_notes"] = extract_appendix_notes(result) + return result + except Exception: # noqa: BLE001 + traceback.print_exc() + return None + + +# ── Minibatch reflect dispatcher ──────────────────────────────────────────── + + +def _split_minibatches(items: list, batch_size: int) -> list[list]: + """Split items into minibatches of at most *batch_size*.""" + return [items[i : i + batch_size] for i in range(0, len(items), batch_size)] + + +def _shuffle_for_minibatch(items: list, seed: int | None) -> list: + """Return items in minibatch order. + + Uses a deterministic shuffle when a seed is provided so resume runs keep + the same minibatch composition. Falls back to input order when no seed is + available. + """ + ordered = list(items) + if seed is None: + return ordered + random.Random(seed).shuffle(ordered) + return ordered + + +def run_minibatch_reflect( + results: list[dict], + skill_content: str, + prediction_dir: str, + patches_dir: str, + workers: int, + failure_only: bool, + minibatch_size: int = 8, + edit_budget: int = 4, + random_seed: int | None = None, + *, + error_system: str | None = None, + success_system: str | None = None, + rejection_context: str = "", + trajectory_memory_context: str = "", + step_buffer_context: str = "", + meta_skill_context: str = "", + update_mode: str = "patch", + skill_aware_reflection: bool | None = None, + skill_aware_appendix_source: str | None = None, +) -> list[dict | None]: + """Full minibatch reflect stage: group → parallel optimizer calls → patches. + + Separates failure and success trajectories, splits each into minibatches + of size M, runs all minibatches in parallel, and saves patch files. + + Parameters + ---------- + results : list[dict] + Rollout result dicts; see :class:`~skillopt.types.RolloutResult`. + skill_content : str + Current skill document. + prediction_dir : str + Path to ``predictions/`` with ``conversation.json`` files. + patches_dir : str + Path to save per-minibatch patch JSON files. + workers : int + Max parallel optimizer calls. + failure_only : bool + If True, skip success trajectories. + minibatch_size : int + Trajectories per group (M). + edit_budget : int + Max edits per minibatch (L). + random_seed : int | None + Optional seed used to shuffle trajectories before minibatch splitting. + error_system, success_system : str | None + Optional custom prompts. ``None`` = use generic defaults. + + Returns + ------- + list[dict | None] + Patch dicts (with ``source_type`` "failure" or "success"). + """ + # Resolve the skill-aware toggle: explicit kwargs win; otherwise fall back + # to the process-wide config switch set by the trainer, so the feature is + # env-independent and adapters need no per-benchmark wiring. + if skill_aware_reflection is None: + skill_aware_reflection = is_skill_aware_enabled() + if skill_aware_appendix_source is None: + skill_aware_appendix_source = get_skill_aware_appendix_source() + + os.makedirs(patches_dir, exist_ok=True) + + # Separate failure / success + failures = [r for r in results if not r.get("hard") or float(r.get("hard", 0)) < 1e-9] + successes = [r for r in results if r.get("hard")] if not failure_only else [] + + failures = _shuffle_for_minibatch(failures, random_seed) + successes = _shuffle_for_minibatch(successes, None if random_seed is None else random_seed + 1) + + # Split into minibatches + fail_batches = _split_minibatches(failures, minibatch_size) + succ_batches = _split_minibatches(successes, minibatch_size) + + n_fail_batches = len(fail_batches) + n_succ_batches = len(succ_batches) + print( + f" [2/6 REFLECT minibatch] " + f"failure={len(failures)}→{n_fail_batches} groups " + f"success={len(successes)}→{n_succ_batches} groups " + f"(M={minibatch_size}, L={edit_budget}, workers={workers})" + ) + + raw_patches: list[dict | None] = [] + + # Resume support: check for already-done minibatch patches + pending_fail: list[tuple[int, list[dict]]] = [] + for idx, batch in enumerate(fail_batches): + path = os.path.join(patches_dir, f"minibatch_fail_{idx:03d}.json") + if os.path.exists(path): + with open(path) as f: + raw_patches.append(json.load(f)) + else: + pending_fail.append((idx, batch)) + + pending_succ: list[tuple[int, list[dict]]] = [] + for idx, batch in enumerate(succ_batches): + path = os.path.join(patches_dir, f"minibatch_succ_{idx:03d}.json") + if os.path.exists(path): + with open(path) as f: + raw_patches.append(json.load(f)) + else: + pending_succ.append((idx, batch)) + + # ── Worker functions ────────────────────────────────────────────────── + def _do_fail(idx: int, batch: list[dict]) -> tuple[str, dict | None]: + patch = run_error_analyst_minibatch( + skill_content, batch, prediction_dir, + edit_budget=edit_budget, + system_prompt=error_system, + step_buffer_context=step_buffer_context, + # backward compat fallback + rejection_context=rejection_context, + trajectory_memory_context=trajectory_memory_context, + meta_skill_context=meta_skill_context, + update_mode=update_mode, + skill_aware_reflection=skill_aware_reflection, + ) + return f"minibatch_fail_{idx:03d}", patch + + def _do_succ(idx: int, batch: list[dict]) -> tuple[str, dict | None]: + patch = run_success_analyst_minibatch( + skill_content, batch, prediction_dir, + edit_budget=edit_budget, + system_prompt=success_system, + step_buffer_context=step_buffer_context, + trajectory_memory_context=trajectory_memory_context, + meta_skill_context=meta_skill_context, + update_mode=update_mode, + skill_aware_reflection=skill_aware_reflection, + emit_appendix_notes=(skill_aware_appendix_source != "failure_only"), + ) + return f"minibatch_succ_{idx:03d}", patch + + # Run all pending minibatches in parallel + all_pending = ( + [("fail", idx, batch) for idx, batch in pending_fail] + + [("succ", idx, batch) for idx, batch in pending_succ] + ) + + with ThreadPoolExecutor(max_workers=workers) as ex: + futs = {} + for kind, idx, batch in all_pending: + if kind == "fail": + futs[ex.submit(_do_fail, idx, batch)] = (kind, idx, len(batch)) + else: + futs[ex.submit(_do_succ, idx, batch)] = (kind, idx, len(batch)) + + for i, fut in enumerate(as_completed(futs), 1): + kind, idx, batch_len = futs[fut] + tag, patch = fut.result() + if patch: + path = os.path.join(patches_dir, f"{tag}.json") + with open(path, "w") as f: + json.dump(patch, f, ensure_ascii=False, indent=2) + raw_patches.append(patch) + n_edits = len(get_payload_items(patch.get("patch", {}) if patch else {}, update_mode)) + print( + f" [analyst] {i}/{len(all_pending)} {tag} " + f"({batch_len} trajs) → {n_edits} {payload_label(update_mode)}" + ) + + return raw_patches diff --git a/skillopt/model/__init__.py b/skillopt/model/__init__.py new file mode 100644 index 0000000..d332997 --- /dev/null +++ b/skillopt/model/__init__.py @@ -0,0 +1,542 @@ +"""ReflACT model API with runtime backend selection for the target path.""" + +from __future__ import annotations + +from typing import Any + +from skillopt.model import azure_openai as _openai +from skillopt.model import claude_backend as _claude +from skillopt.model import minimax_backend as _minimax +from skillopt.model import qwen_backend as _qwen +from skillopt.model.backend_config import ( # noqa: F401 + configure_claude_code_exec, + configure_codex_exec, + get_claude_code_exec_config, + get_codex_exec_config, + get_optimizer_backend, + get_target_backend, + is_optimizer_chat_backend, + is_target_chat_backend, + is_target_exec_backend, + set_optimizer_backend, + set_target_backend, +) + + +def set_backend(name: str | None) -> str: + """Backward-compatible global backend setter. + + Historically the codebase used one shared backend for both optimizer and + target. Keep that entry point so older scripts continue to work, while + mapping it onto the split optimizer/target backend model. + """ + normalized = str(name or "azure_openai").strip().lower() + if normalized in {"azure_openai", "openai_chat", "azure", "azure-openai"}: + set_optimizer_backend("openai_chat") + set_target_backend("openai_chat") + return "azure_openai" + if normalized in {"claude", "claude_chat", "anthropic"}: + set_optimizer_backend("claude_chat") + set_target_backend("claude_chat") + return "claude_chat" + if normalized == "codex": + set_optimizer_backend("openai_chat") + set_target_backend("codex_exec") + return "codex" + if normalized in {"codex_exec", "claude_code_exec"}: + set_optimizer_backend("openai_chat") + set_target_backend(normalized) + return normalized + if normalized in {"qwen", "qwen_chat"}: + set_optimizer_backend("openai_chat") + set_target_backend("qwen_chat") + return "qwen_chat" + if normalized in {"minimax", "minimax_chat"}: + set_optimizer_backend("openai_chat") + set_target_backend("minimax_chat") + return "minimax_chat" + raise ValueError(f"Unsupported legacy backend: {name!r}") + + +def get_backend_name() -> str: + """Best-effort backward-compatible backend summary.""" + optimizer = get_optimizer_backend() + target = get_target_backend() + if optimizer == "claude_chat" and target == "claude_chat": + return "claude_chat" + if optimizer == "qwen_chat" and target == "qwen_chat": + return "qwen_chat" + if optimizer == "openai_chat" and target == "openai_chat": + return "azure_openai" + if optimizer == "openai_chat" and target == "codex_exec": + return "codex" + if optimizer == "openai_chat" and target == "qwen_chat": + return "qwen_chat" + if optimizer == "openai_chat" and target == "minimax_chat": + return "minimax_chat" + return f"{optimizer}+{target}" + + +def chat_optimizer( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + reasoning_effort: str | None = None, + timeout: int | None = None, +) -> tuple[str, dict]: + if get_optimizer_backend() == "claude_chat": + return _claude.chat_optimizer( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + timeout=timeout, + ) + if get_optimizer_backend() == "qwen_chat": + return _qwen.chat_optimizer( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + timeout=timeout, + ) + if get_optimizer_backend() == "minimax_chat": + return _minimax.chat_optimizer( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + timeout=timeout, + ) + return _openai.chat_optimizer( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + timeout=timeout, + ) + + +def chat_target( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + reasoning_effort: str | None = None, + timeout: int | None = None, +) -> tuple[str, dict]: + if get_target_backend() == "claude_chat": + return _claude.chat_target( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + timeout=timeout, + ) + if get_target_backend() == "qwen_chat": + return _qwen.chat_target( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + timeout=timeout, + ) + if get_target_backend() == "minimax_chat": + return _minimax.chat_target( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + ) + if not is_target_chat_backend(): + raise NotImplementedError( + "chat_target is only supported with target_backend=openai_chat, claude_chat, qwen_chat, or minimax_chat. " + "Exec backends are handled in environment-specific rollout code." + ) + return _openai.chat_target( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + timeout=timeout, + ) + + +def chat_optimizer_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict]: + if get_optimizer_backend() == "claude_chat": + return _claude.chat_optimizer_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + if get_optimizer_backend() == "qwen_chat": + return _qwen.chat_optimizer_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + if get_optimizer_backend() == "minimax_chat": + return _minimax.chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + return _openai.chat_optimizer_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_target_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict]: + if get_target_backend() == "claude_chat": + return _claude.chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + if get_target_backend() == "qwen_chat": + return _qwen.chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + if get_target_backend() == "minimax_chat": + return _minimax.chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + ) + if not is_target_chat_backend(): + raise NotImplementedError( + "chat_target_messages is only supported with target_backend=openai_chat, claude_chat, qwen_chat, or minimax_chat. " + "Exec backends are handled in environment-specific rollout code." + ) + return _openai.chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_messages_with_deployment( + deployment: str, + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "custom", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict]: + return _openai.chat_messages_with_deployment( + deployment=deployment, + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_with_deployment( + deployment: str, + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "custom", + reasoning_effort: str | None = None, + timeout: int | None = None, +) -> tuple[str, dict]: + return _openai.chat_with_deployment( + deployment=deployment, + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + reasoning_effort=reasoning_effort, + timeout=timeout, + ) + + +def get_token_summary() -> dict: + summary = _openai.get_token_summary() + claude_summary = _claude.get_token_summary() + for stage, values in claude_summary.items(): + if stage == "_total": + continue + if stage not in summary: + summary[stage] = values + continue + summary[stage]["calls"] += values["calls"] + summary[stage]["prompt_tokens"] += values["prompt_tokens"] + summary[stage]["completion_tokens"] += values["completion_tokens"] + summary[stage]["total_tokens"] += values["total_tokens"] + qwen_summary = _qwen.get_token_summary() + for stage, values in qwen_summary.items(): + if stage == "_total": + continue + if stage not in summary: + summary[stage] = values + continue + summary[stage]["calls"] += values["calls"] + summary[stage]["prompt_tokens"] += values["prompt_tokens"] + summary[stage]["completion_tokens"] += values["completion_tokens"] + summary[stage]["total_tokens"] += values["total_tokens"] + minimax_summary = _minimax.get_token_summary() + for stage, values in minimax_summary.items(): + if stage == "_total": + continue + if stage not in summary: + summary[stage] = values + continue + summary[stage]["calls"] += values["calls"] + summary[stage]["prompt_tokens"] += values["prompt_tokens"] + summary[stage]["completion_tokens"] += values["completion_tokens"] + summary[stage]["total_tokens"] += values["total_tokens"] + total = { + "calls": 0, + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + } + for stage, values in summary.items(): + if stage == "_total": + continue + total["calls"] += values["calls"] + total["prompt_tokens"] += values["prompt_tokens"] + total["completion_tokens"] += values["completion_tokens"] + total["total_tokens"] += values["total_tokens"] + summary["_total"] = total + return summary + + +def reset_token_tracker() -> None: + _openai.reset_token_tracker() + _claude.reset_token_tracker() + _qwen.reset_token_tracker() + _minimax.reset_token_tracker() + + +def configure_azure_openai( + *, + endpoint: str | None = None, + api_version: str | None = None, + api_key: str | None = None, + auth_mode: str | None = None, + ad_scope: str | None = None, + managed_identity_client_id: str | None = None, + optimizer_endpoint: str | None = None, + optimizer_api_version: str | None = None, + optimizer_api_key: str | None = None, + optimizer_auth_mode: str | None = None, + optimizer_ad_scope: str | None = None, + optimizer_managed_identity_client_id: str | None = None, + target_endpoint: str | None = None, + target_api_version: str | None = None, + target_api_key: str | None = None, + target_auth_mode: str | None = None, + target_ad_scope: str | None = None, + target_managed_identity_client_id: str | None = None, +) -> None: + _openai.configure_azure_openai( + endpoint=endpoint, + api_version=api_version, + api_key=api_key, + auth_mode=auth_mode, + ad_scope=ad_scope, + managed_identity_client_id=managed_identity_client_id, + optimizer_endpoint=optimizer_endpoint, + optimizer_api_version=optimizer_api_version, + optimizer_api_key=optimizer_api_key, + optimizer_auth_mode=optimizer_auth_mode, + optimizer_ad_scope=optimizer_ad_scope, + optimizer_managed_identity_client_id=optimizer_managed_identity_client_id, + target_endpoint=target_endpoint, + target_api_version=target_api_version, + target_api_key=target_api_key, + target_auth_mode=target_auth_mode, + target_ad_scope=target_ad_scope, + target_managed_identity_client_id=target_managed_identity_client_id, + ) + + +def configure_qwen_chat( + *, + base_url: str | None = None, + api_key: str | None = None, + temperature: float | str | None = None, + timeout_seconds: float | str | None = None, + max_tokens: int | str | None = None, + enable_thinking: bool | str | None = None, + use_max_completion_tokens: bool | str | None = None, + optimizer_base_url: str | None = None, + optimizer_api_key: str | None = None, + optimizer_temperature: float | str | None = None, + optimizer_timeout_seconds: float | str | None = None, + optimizer_max_tokens: int | str | None = None, + optimizer_enable_thinking: bool | str | None = None, + optimizer_use_max_completion_tokens: bool | str | None = None, + target_base_url: str | None = None, + target_api_key: str | None = None, + target_temperature: float | str | None = None, + target_timeout_seconds: float | str | None = None, + target_max_tokens: int | str | None = None, + target_enable_thinking: bool | str | None = None, + target_use_max_completion_tokens: bool | str | None = None, +) -> None: + _qwen.configure_qwen_chat( + base_url=base_url, + api_key=api_key, + temperature=temperature, + timeout_seconds=timeout_seconds, + max_tokens=max_tokens, + enable_thinking=enable_thinking, + use_max_completion_tokens=use_max_completion_tokens, + optimizer_base_url=optimizer_base_url, + optimizer_api_key=optimizer_api_key, + optimizer_temperature=optimizer_temperature, + optimizer_timeout_seconds=optimizer_timeout_seconds, + optimizer_max_tokens=optimizer_max_tokens, + optimizer_enable_thinking=optimizer_enable_thinking, + optimizer_use_max_completion_tokens=optimizer_use_max_completion_tokens, + target_base_url=target_base_url, + target_api_key=target_api_key, + target_temperature=target_temperature, + target_timeout_seconds=target_timeout_seconds, + target_max_tokens=target_max_tokens, + target_enable_thinking=target_enable_thinking, + target_use_max_completion_tokens=target_use_max_completion_tokens, + ) + + +def configure_minimax_chat( + *, + base_url: str | None = None, + api_key: str | None = None, + temperature: float | str | None = None, + timeout_seconds: float | str | None = None, + max_tokens: int | str | None = None, + enable_thinking: bool | str | None = None, +) -> None: + _minimax.configure_minimax_chat( + base_url=base_url, + api_key=api_key, + temperature=temperature, + timeout_seconds=timeout_seconds, + max_tokens=max_tokens, + enable_thinking=enable_thinking, + ) + + +def set_reasoning_effort(effort: str | None) -> None: + _openai.set_reasoning_effort(effort) + _claude.set_reasoning_effort(effort) + _qwen.set_reasoning_effort(effort) + _minimax.set_reasoning_effort(effort) + + +def set_target_deployment(deployment: str) -> None: + _openai.set_target_deployment(deployment) + _claude.set_target_deployment(deployment) + _qwen.set_target_deployment(deployment) + _minimax.set_target_deployment(deployment) + + +def set_optimizer_deployment(deployment: str) -> None: + _openai.set_optimizer_deployment(deployment) + _claude.set_optimizer_deployment(deployment) + _qwen.set_optimizer_deployment(deployment) diff --git a/skillopt/model/azure_openai.py b/skillopt/model/azure_openai.py new file mode 100644 index 0000000..e7c139c --- /dev/null +++ b/skillopt/model/azure_openai.py @@ -0,0 +1,915 @@ +"""ReflACT Model backend — Azure OpenAI wrapper with token tracking. + +Provides optimizer/target dual-deployment chat functions and a global +TokenTracker for per-stage cost accounting. Previously llm/azure_openai.py. +""" +from __future__ import annotations + +import json +import os +import subprocess +import threading +import time +from types import SimpleNamespace +from typing import Any +from openai import AzureOpenAI, OpenAI + +# Sentinel value used as the api_version when the "openai_compatible" +# auth_mode is selected. Real Azure deployments never use this string, +# so it doubles as a marker for downstream type narrowing. +_OPENAI_COMPATIBLE_API_VERSION = "openai-compat" + +# ── Configuration ───────────────────────────────────────────────────────────── + +ENDPOINT = os.environ.get( + "AZURE_OPENAI_ENDPOINT", + "", # Set via env var or config: e.g. "https://your-resource.openai.azure.com/" +) +API_VERSION = os.environ.get("AZURE_OPENAI_API_VERSION", "2024-12-01-preview") +API_KEY = os.environ.get( + "AZURE_OPENAI_API_KEY", + "", +) +AUTH_MODE = os.environ.get("AZURE_OPENAI_AUTH_MODE", "azure_cli").strip().lower() +AD_SCOPE = os.environ.get( + "AZURE_OPENAI_AD_SCOPE", + "https://cognitiveservices.azure.com/.default", +) +MANAGED_IDENTITY_CLIENT_ID = os.environ.get( + "AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID", + "", +).strip() + +OPTIMIZER_ENDPOINT = ( + os.environ.get("OPTIMIZER_AZURE_OPENAI_ENDPOINT") + or os.environ.get("AZURE_OPENAI_OPTIMIZER_ENDPOINT") + or ENDPOINT +) +TARGET_ENDPOINT = ( + os.environ.get("TARGET_AZURE_OPENAI_ENDPOINT") + or os.environ.get("AZURE_OPENAI_TARGET_ENDPOINT") + or ENDPOINT +) +OPTIMIZER_API_VERSION = ( + os.environ.get("OPTIMIZER_AZURE_OPENAI_API_VERSION") + or os.environ.get("AZURE_OPENAI_OPTIMIZER_API_VERSION") + or API_VERSION +) +TARGET_API_VERSION = ( + os.environ.get("TARGET_AZURE_OPENAI_API_VERSION") + or os.environ.get("AZURE_OPENAI_TARGET_API_VERSION") + or API_VERSION +) +OPTIMIZER_API_KEY = ( + os.environ.get("OPTIMIZER_AZURE_OPENAI_API_KEY") + or os.environ.get("AZURE_OPENAI_OPTIMIZER_API_KEY") + or API_KEY +) +TARGET_API_KEY = ( + os.environ.get("TARGET_AZURE_OPENAI_API_KEY") + or os.environ.get("AZURE_OPENAI_TARGET_API_KEY") + or API_KEY +) +OPTIMIZER_AUTH_MODE = ( + os.environ.get("OPTIMIZER_AZURE_OPENAI_AUTH_MODE") + or os.environ.get("AZURE_OPENAI_OPTIMIZER_AUTH_MODE") + or AUTH_MODE +).strip().lower() +TARGET_AUTH_MODE = ( + os.environ.get("TARGET_AZURE_OPENAI_AUTH_MODE") + or os.environ.get("AZURE_OPENAI_TARGET_AUTH_MODE") + or AUTH_MODE +).strip().lower() +OPTIMIZER_AD_SCOPE = ( + os.environ.get("OPTIMIZER_AZURE_OPENAI_AD_SCOPE") + or os.environ.get("AZURE_OPENAI_OPTIMIZER_AD_SCOPE") + or AD_SCOPE +) +TARGET_AD_SCOPE = ( + os.environ.get("TARGET_AZURE_OPENAI_AD_SCOPE") + or os.environ.get("AZURE_OPENAI_TARGET_AD_SCOPE") + or AD_SCOPE +) +OPTIMIZER_MANAGED_IDENTITY_CLIENT_ID = ( + os.environ.get("OPTIMIZER_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID") + or os.environ.get("AZURE_OPENAI_OPTIMIZER_MANAGED_IDENTITY_CLIENT_ID") + or MANAGED_IDENTITY_CLIENT_ID +).strip() +TARGET_MANAGED_IDENTITY_CLIENT_ID = ( + os.environ.get("TARGET_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID") + or os.environ.get("AZURE_OPENAI_TARGET_MANAGED_IDENTITY_CLIENT_ID") + or MANAGED_IDENTITY_CLIENT_ID +).strip() + +OPTIMIZER_DEPLOYMENT = os.environ.get("OPTIMIZER_DEPLOYMENT", "gpt-4o") +TARGET_DEPLOYMENT = os.environ.get("TARGET_DEPLOYMENT", "gpt-4o") + +REASONING_EFFORT: str | None = None + +_AZ_CLI_TOKEN_CACHE: dict[str, dict[str, Any]] = {} + +# Deployments that require Responses API +_RESPONSES_API_MODELS = { + "gpt-5.3-codex", "gpt-5.1-codex", "gpt-5.2-codex", + "gpt-5-codex", "codex-mini", "gpt-5.4-pro", +} + + +# ── Token Tracker ───────────────────────────────────────────────────────────── + +class TokenTracker: + """Thread-safe per-stage token counter.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._data: dict[str, dict] = {} + + def record( + self, stage: str, prompt_tokens: int, completion_tokens: int, + ) -> None: + with self._lock: + if stage not in self._data: + self._data[stage] = { + "calls": 0, + "prompt_tokens": 0, + "completion_tokens": 0, + } + d = self._data[stage] + d["calls"] += 1 + d["prompt_tokens"] += prompt_tokens + d["completion_tokens"] += completion_tokens + + def summary(self) -> dict: + with self._lock: + out: dict = {} + total_p = total_c = total_calls = 0 + for stage, d in sorted(self._data.items()): + out[stage] = { + "calls": d["calls"], + "prompt_tokens": d["prompt_tokens"], + "completion_tokens": d["completion_tokens"], + "total_tokens": d["prompt_tokens"] + d["completion_tokens"], + } + total_p += d["prompt_tokens"] + total_c += d["completion_tokens"] + total_calls += d["calls"] + out["_total"] = { + "calls": total_calls, + "prompt_tokens": total_p, + "completion_tokens": total_c, + "total_tokens": total_p + total_c, + } + return out + + def reset(self) -> None: + with self._lock: + self._data.clear() + + def stage_snapshot(self, stage: str) -> dict: + """Return a copy of one stage's counters (or zeros if not tracked yet).""" + with self._lock: + d = self._data.get(stage, {}) + return { + "calls": d.get("calls", 0), + "prompt_tokens": d.get("prompt_tokens", 0), + "completion_tokens": d.get("completion_tokens", 0), + "total_tokens": d.get("prompt_tokens", 0) + d.get("completion_tokens", 0), + } + + +tracker = TokenTracker() + + +# ── Client management ───────────────────────────────────────────────────────── + +_optimizer_client: AzureOpenAI | OpenAI | None = None +_target_client: AzureOpenAI | OpenAI | None = None +_optimizer_lock = threading.Lock() +_target_lock = threading.Lock() + + +def _role_config(role: str) -> dict[str, str]: + if role == "optimizer": + return { + "endpoint": OPTIMIZER_ENDPOINT, + "api_version": OPTIMIZER_API_VERSION, + "api_key": OPTIMIZER_API_KEY, + "auth_mode": OPTIMIZER_AUTH_MODE, + "ad_scope": OPTIMIZER_AD_SCOPE, + "managed_identity_client_id": OPTIMIZER_MANAGED_IDENTITY_CLIENT_ID, + } + if role == "target": + return { + "endpoint": TARGET_ENDPOINT, + "api_version": TARGET_API_VERSION, + "api_key": TARGET_API_KEY, + "auth_mode": TARGET_AUTH_MODE, + "ad_scope": TARGET_AD_SCOPE, + "managed_identity_client_id": TARGET_MANAGED_IDENTITY_CLIENT_ID, + } + raise ValueError(f"Unknown Azure OpenAI client role: {role!r}") + + +def _make_token_provider( + auth_mode: str, + ad_scope: str, + managed_identity_client_id: str, +): + try: + from azure.identity import ( # type: ignore[import-not-found] + AzureCliCredential, + ManagedIdentityCredential, + get_bearer_token_provider, + ) + except ImportError as e: + if auth_mode == "azure_cli": + return _make_azure_cli_token_provider(ad_scope) + raise ImportError( + "Azure AD auth requires azure-identity. Install it with `pip install azure-identity`." + ) from e + + if auth_mode in {"managed_identity", "aad", "azure_ad"}: + if managed_identity_client_id: + credential = ManagedIdentityCredential(client_id=managed_identity_client_id) + else: + credential = ManagedIdentityCredential() + elif auth_mode == "azure_cli": + credential = AzureCliCredential() + else: + raise ValueError( + "Unsupported Azure OpenAI auth mode " + f"{auth_mode!r}; expected api_key, managed_identity, azure_ad, aad, or azure_cli." + ) + return get_bearer_token_provider(credential, ad_scope) + + +def _make_azure_cli_token_provider(ad_scope: str): + """Return an Azure CLI token provider compatible with AzureOpenAI. + + This fallback avoids requiring azure-identity in environments where `az` + is already logged in. The SDK calls this provider whenever it needs a + bearer token. + """ + + resource = ad_scope.removesuffix("/.default") + + def _provider() -> str: + now = int(time.time()) + cache = _AZ_CLI_TOKEN_CACHE.setdefault(resource, {"token": "", "expires_on": 0}) + cached = str(cache.get("token") or "") + expires_on = int(cache.get("expires_on") or 0) + if cached and expires_on - now > 300: + return cached + + raw = subprocess.check_output( + [ + "az", + "account", + "get-access-token", + "--resource", + resource, + "-o", + "json", + ], + text=True, + stderr=subprocess.STDOUT, + ) + payload = json.loads(raw) + token = str(payload["accessToken"]) + cache["token"] = token + cache["expires_on"] = int(payload.get("expires_on") or now + 3000) + return token + + return _provider + + +def _make_client(role: str) -> AzureOpenAI | OpenAI: + cfg = _role_config(role) + if not cfg["endpoint"]: + raise ValueError( + f"Azure OpenAI endpoint is not configured for {role}. " + "Pass --azure_openai_endpoint https://your-resource.openai.azure.com/ " + "or set AZURE_OPENAI_ENDPOINT in your environment." + ) + auth_mode = cfg["auth_mode"] + if auth_mode in {"openai_compatible", "compat", "openai"}: + return OpenAI( + base_url=cfg["endpoint"].rstrip("/"), + api_key=cfg["api_key"] or "dummy", + default_headers={"User-Agent": "SkillOpt"}, + ) + if auth_mode in {"api_key", "key"}: + if not cfg["api_key"]: + raise ValueError( + f"Azure OpenAI API key is not configured for {role}. " + "Set model.azure_openai_api_key in the config or export AZURE_OPENAI_API_KEY." + ) + return AzureOpenAI( + api_version=cfg["api_version"], + azure_endpoint=cfg["endpoint"], + api_key=cfg["api_key"], + ) + return AzureOpenAI( + api_version=cfg["api_version"], + azure_endpoint=cfg["endpoint"], + azure_ad_token_provider=_make_token_provider( + auth_mode, + cfg["ad_scope"], + cfg["managed_identity_client_id"], + ), + ) + + +def get_optimizer_client() -> AzureOpenAI | OpenAI: + global _optimizer_client + with _optimizer_lock: + if _optimizer_client is None: + _optimizer_client = _make_client("optimizer") + return _optimizer_client + + +def get_target_client() -> AzureOpenAI | OpenAI: + global _target_client + with _target_lock: + if _target_client is None: + # When using qwen_chat backend, return an OpenAI client pointing to vLLM + from skillopt.model.backend_config import get_target_backend + if get_target_backend() == "qwen_chat": + from skillopt.model import qwen_backend as _qwen + target_config = _qwen.TARGET_CONFIG + _target_client = OpenAI( + base_url=target_config.base_url, + api_key=target_config.api_key or "dummy", + ) + else: + _target_client = _make_client("target") + return _target_client + + +def _needs_responses_api(deployment: str) -> bool: + dep = deployment.lower() + return any(dep == m or dep.startswith(m + "-") for m in _RESPONSES_API_MODELS) + + +# ── Core chat function ──────────────────────────────────────────────────────── + +def _chat_impl( + client: AzureOpenAI | OpenAI, + deployment: str, + system: str, + user: str, + max_completion_tokens: int, + retries: int, + stage: str, + reasoning_effort: str | None = None, + timeout: int | None = None, +) -> tuple[str, dict]: + """Call LLM, track tokens, return (text, usage_dict).""" + last_err = None + usage_info = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + for attempt in range(retries): + try: + if _needs_responses_api(deployment): + kwargs: dict[str, Any] = { + "model": deployment, + "instructions": system, + "input": [{"role": "user", "content": user}], + "max_output_tokens": max_completion_tokens, + } + actual_effort = reasoning_effort or REASONING_EFFORT + if actual_effort: + kwargs["reasoning"] = {"effort": actual_effort} + if timeout is not None: + kwargs["timeout"] = timeout + resp = client.responses.create(**kwargs) + text = getattr(resp, "output_text", None) or "" + if not text: + for item in getattr(resp, "output", None) or []: + for part in getattr(item, "content", []): + if getattr(part, "type", "") == "output_text": + text = part.text or "" + if hasattr(resp, "usage") and resp.usage: + usage_info = { + "prompt_tokens": getattr(resp.usage, "input_tokens", 0) or 0, + "completion_tokens": getattr(resp.usage, "output_tokens", 0) or 0, + "total_tokens": ( + (getattr(resp.usage, "input_tokens", 0) or 0) + + (getattr(resp.usage, "output_tokens", 0) or 0) + ), + } + else: + kwargs: dict[str, Any] = dict( + model=deployment, + messages=[ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + max_completion_tokens=max_completion_tokens, + ) + actual_effort = reasoning_effort or REASONING_EFFORT + if actual_effort is not None: + kwargs["reasoning_effort"] = actual_effort + if timeout is not None: + kwargs["timeout"] = timeout + resp = client.chat.completions.create(**kwargs) + text = resp.choices[0].message.content or "" + if resp.usage: + usage_info = { + "prompt_tokens": resp.usage.prompt_tokens or 0, + "completion_tokens": resp.usage.completion_tokens or 0, + "total_tokens": resp.usage.total_tokens or 0, + } + + tracker.record( + stage, + usage_info["prompt_tokens"], + usage_info["completion_tokens"], + ) + return text, usage_info + + except Exception as e: # noqa: BLE001 + last_err = e + sleep = min(2 ** attempt, 30) + time.sleep(sleep) + + raise RuntimeError(f"LLM call failed after {retries} retries: {last_err}") + + +def _chat_messages_impl( + client: AzureOpenAI | OpenAI, + deployment: str, + messages: list[dict[str, Any]], + max_completion_tokens: int, + retries: int, + stage: str, + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict]: + """Call the model with a pre-built message list.""" + last_err = None + usage_info = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + for attempt in range(retries): + try: + if _needs_responses_api(deployment): + input_items, instructions = _messages_to_responses_input(messages) + kwargs: dict[str, Any] = { + "model": deployment, + "input": input_items, + "max_output_tokens": max_completion_tokens, + } + if instructions: + kwargs["instructions"] = instructions + actual_effort = reasoning_effort or REASONING_EFFORT + if actual_effort: + kwargs["reasoning"] = {"effort": actual_effort} + if tools: + kwargs["tools"] = [_chat_tool_to_responses_tool(tool) for tool in tools] + if tool_choice is not None: + kwargs["tool_choice"] = tool_choice + if timeout is not None: + kwargs["timeout"] = timeout + resp = client.responses.create(**kwargs) + message, text = _responses_to_chat_message(resp) + if hasattr(resp, "usage") and resp.usage: + usage_info = { + "prompt_tokens": getattr(resp.usage, "input_tokens", 0) or 0, + "completion_tokens": getattr(resp.usage, "output_tokens", 0) or 0, + "total_tokens": ( + (getattr(resp.usage, "input_tokens", 0) or 0) + + (getattr(resp.usage, "output_tokens", 0) or 0) + ), + } + else: + kwargs = dict( + model=deployment, + messages=messages, + max_completion_tokens=max_completion_tokens, + ) + actual_effort = reasoning_effort or REASONING_EFFORT + if tools: + kwargs["tools"] = tools + if tool_choice is not None: + kwargs["tool_choice"] = tool_choice + # Some models (e.g. gpt-5.5) don't support reasoning_effort with function tools + elif actual_effort is not None: + kwargs["reasoning_effort"] = actual_effort + if timeout is not None: + kwargs["timeout"] = timeout + resp = client.chat.completions.create(**kwargs) + message = resp.choices[0].message + text = message.content or "" + if resp.usage: + usage_info = { + "prompt_tokens": resp.usage.prompt_tokens or 0, + "completion_tokens": resp.usage.completion_tokens or 0, + "total_tokens": resp.usage.total_tokens or 0, + } + tracker.record( + stage, + usage_info["prompt_tokens"], + usage_info["completion_tokens"], + ) + return (message if return_message else text), usage_info + except Exception as e: # noqa: BLE001 + last_err = e + sleep = min(2 ** attempt, 30) + time.sleep(sleep) + + raise RuntimeError(f"LLM message call failed after {retries} retries: {last_err}") + + +def _chat_tool_to_responses_tool(tool: dict[str, Any]) -> dict[str, Any]: + """Convert a Chat Completions function tool to Responses API format.""" + if tool.get("type") == "function" and isinstance(tool.get("function"), dict): + fn = tool["function"] + return { + "type": "function", + "name": fn.get("name", ""), + "description": fn.get("description", ""), + "parameters": fn.get("parameters", {"type": "object", "properties": {}}), + } + return tool + + +def _messages_to_responses_input(messages: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], str]: + """Convert chat-style messages, including tool results, to Responses input.""" + instructions: list[str] = [] + input_items: list[dict[str, Any]] = [] + for message in messages: + role = message.get("role") + content = message.get("content") or "" + if role == "system": + if content: + instructions.append(str(content)) + continue + if role == "tool": + input_items.append({ + "type": "function_call_output", + "call_id": str(message.get("tool_call_id", "")), + "output": str(content), + }) + continue + if role == "assistant": + if content: + input_items.append({"role": "assistant", "content": str(content)}) + for tool_call in message.get("tool_calls") or []: + function = tool_call.get("function", {}) or {} + input_items.append({ + "type": "function_call", + "call_id": str(tool_call.get("id", "")), + "name": str(function.get("name", "")), + "arguments": str(function.get("arguments", "{}") or "{}"), + }) + continue + if role in {"user", "developer"}: + input_items.append({"role": "user", "content": str(content)}) + return input_items, "\n\n".join(instructions) + + +def _responses_to_chat_message(resp: Any) -> tuple[Any, str]: + """Convert Responses output into the subset of Chat message API we use.""" + text = getattr(resp, "output_text", None) or "" + tool_calls: list[dict[str, Any]] = [] + for item in getattr(resp, "output", None) or []: + item_type = getattr(item, "type", "") + if item_type == "function_call": + tool_calls.append({ + "id": getattr(item, "call_id", "") or getattr(item, "id", ""), + "type": "function", + "function": { + "name": getattr(item, "name", ""), + "arguments": getattr(item, "arguments", "") or "{}", + }, + }) + elif item_type == "message" and not text: + content_parts = getattr(item, "content", []) or [] + for part in content_parts: + if getattr(part, "type", "") == "output_text": + text += getattr(part, "text", "") or "" + return SimpleNamespace(content=text, tool_calls=tool_calls), text + + +# ── Public API ──────────────────────────────────────────────────────────────── + +def configure_azure_openai( + *, + endpoint: str | None = None, + api_version: str | None = None, + api_key: str | None = None, + auth_mode: str | None = None, + ad_scope: str | None = None, + managed_identity_client_id: str | None = None, + optimizer_endpoint: str | None = None, + optimizer_api_version: str | None = None, + optimizer_api_key: str | None = None, + optimizer_auth_mode: str | None = None, + optimizer_ad_scope: str | None = None, + optimizer_managed_identity_client_id: str | None = None, + target_endpoint: str | None = None, + target_api_version: str | None = None, + target_api_key: str | None = None, + target_auth_mode: str | None = None, + target_ad_scope: str | None = None, + target_managed_identity_client_id: str | None = None, +) -> None: + global ENDPOINT, API_VERSION, API_KEY, AUTH_MODE, AD_SCOPE, MANAGED_IDENTITY_CLIENT_ID + global OPTIMIZER_ENDPOINT, OPTIMIZER_API_VERSION, OPTIMIZER_API_KEY, OPTIMIZER_AUTH_MODE + global OPTIMIZER_AD_SCOPE, OPTIMIZER_MANAGED_IDENTITY_CLIENT_ID + global TARGET_ENDPOINT, TARGET_API_VERSION, TARGET_API_KEY, TARGET_AUTH_MODE + global TARGET_AD_SCOPE, TARGET_MANAGED_IDENTITY_CLIENT_ID + global _optimizer_client, _target_client + + def _clean(value: str | None, *, lower: bool = False) -> str | None: + if value is None: + return None + str_value = str(value).strip() + if not str_value: + return None + if lower: + str_value = str_value.lower() + return str_value + + def _set(global_name: str, value: str | None, env_key: str) -> None: + if value is None: + return + globals()[global_name] = value + os.environ[env_key] = value + + shared_endpoint = _clean(endpoint) + shared_api_version = _clean(api_version) + shared_api_key = _clean(api_key) + shared_auth_mode = _clean(auth_mode, lower=True) + shared_ad_scope = _clean(ad_scope) + shared_managed_identity_client_id = _clean(managed_identity_client_id) + + # Auto-configure for openai_compatible mode + if shared_auth_mode in {"openai_compatible", "compat", "openai"}: + if shared_api_version is None: + shared_api_version = _OPENAI_COMPATIBLE_API_VERSION + + _set("ENDPOINT", shared_endpoint, "AZURE_OPENAI_ENDPOINT") + _set("API_VERSION", shared_api_version, "AZURE_OPENAI_API_VERSION") + _set("API_KEY", shared_api_key, "AZURE_OPENAI_API_KEY") + _set("AUTH_MODE", shared_auth_mode, "AZURE_OPENAI_AUTH_MODE") + _set("AD_SCOPE", shared_ad_scope, "AZURE_OPENAI_AD_SCOPE") + _set( + "MANAGED_IDENTITY_CLIENT_ID", + shared_managed_identity_client_id, + "AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID", + ) + + resolved_optimizer_endpoint = _clean(optimizer_endpoint) or shared_endpoint + resolved_optimizer_api_version = _clean(optimizer_api_version) or shared_api_version + resolved_optimizer_api_key = _clean(optimizer_api_key) or shared_api_key + resolved_optimizer_auth_mode = _clean(optimizer_auth_mode, lower=True) or shared_auth_mode + resolved_optimizer_ad_scope = _clean(optimizer_ad_scope) or shared_ad_scope + resolved_optimizer_mi = ( + _clean(optimizer_managed_identity_client_id) + or shared_managed_identity_client_id + ) + + # Auto-configure for openai_compatible mode + if resolved_optimizer_auth_mode in {"openai_compatible", "compat", "openai"}: + if resolved_optimizer_api_version is None: + resolved_optimizer_api_version = _OPENAI_COMPATIBLE_API_VERSION + + resolved_target_endpoint = _clean(target_endpoint) or shared_endpoint + resolved_target_api_version = _clean(target_api_version) or shared_api_version + resolved_target_api_key = _clean(target_api_key) or shared_api_key + resolved_target_auth_mode = _clean(target_auth_mode, lower=True) or shared_auth_mode + resolved_target_ad_scope = _clean(target_ad_scope) or shared_ad_scope + resolved_target_mi = ( + _clean(target_managed_identity_client_id) + or shared_managed_identity_client_id + ) + + # Auto-configure for openai_compatible mode + if resolved_target_auth_mode in {"openai_compatible", "compat", "openai"}: + if resolved_target_api_version is None: + resolved_target_api_version = _OPENAI_COMPATIBLE_API_VERSION + + _set("OPTIMIZER_ENDPOINT", resolved_optimizer_endpoint, "OPTIMIZER_AZURE_OPENAI_ENDPOINT") + _set( + "OPTIMIZER_API_VERSION", + resolved_optimizer_api_version, + "OPTIMIZER_AZURE_OPENAI_API_VERSION", + ) + _set("OPTIMIZER_API_KEY", resolved_optimizer_api_key, "OPTIMIZER_AZURE_OPENAI_API_KEY") + _set("OPTIMIZER_AUTH_MODE", resolved_optimizer_auth_mode, "OPTIMIZER_AZURE_OPENAI_AUTH_MODE") + _set("OPTIMIZER_AD_SCOPE", resolved_optimizer_ad_scope, "OPTIMIZER_AZURE_OPENAI_AD_SCOPE") + _set( + "OPTIMIZER_MANAGED_IDENTITY_CLIENT_ID", + resolved_optimizer_mi, + "OPTIMIZER_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID", + ) + _set("TARGET_ENDPOINT", resolved_target_endpoint, "TARGET_AZURE_OPENAI_ENDPOINT") + _set( + "TARGET_API_VERSION", + resolved_target_api_version, + "TARGET_AZURE_OPENAI_API_VERSION", + ) + _set("TARGET_API_KEY", resolved_target_api_key, "TARGET_AZURE_OPENAI_API_KEY") + _set("TARGET_AUTH_MODE", resolved_target_auth_mode, "TARGET_AZURE_OPENAI_AUTH_MODE") + _set("TARGET_AD_SCOPE", resolved_target_ad_scope, "TARGET_AZURE_OPENAI_AD_SCOPE") + _set( + "TARGET_MANAGED_IDENTITY_CLIENT_ID", + resolved_target_mi, + "TARGET_AZURE_OPENAI_MANAGED_IDENTITY_CLIENT_ID", + ) + + with _optimizer_lock: + _optimizer_client = None + with _target_lock: + _target_client = None + + +def chat_optimizer( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + reasoning_effort: str | None = None, + timeout: int | None = None, +) -> tuple[str, dict]: + """Call the optimizer model. Returns (response_text, usage_dict).""" + return _chat_impl( + get_optimizer_client(), OPTIMIZER_DEPLOYMENT, + system, user, max_completion_tokens, retries, stage, reasoning_effort, timeout, + ) + + +def chat_with_deployment( + deployment: str, + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "custom", + reasoning_effort: str | None = None, + timeout: int | None = None, +) -> tuple[str, dict]: + """Call an arbitrary deployment using the shared Azure client.""" + return _chat_impl( + get_optimizer_client(), + deployment, + system, + user, + max_completion_tokens, + retries, + stage, + reasoning_effort, + timeout, + ) + + +def chat_target( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + reasoning_effort: str | None = None, + timeout: int | None = None, +) -> tuple[str, dict]: + """Call the target model. Returns (response_text, usage_dict).""" + return _chat_impl( + get_target_client(), TARGET_DEPLOYMENT, + system, user, max_completion_tokens, retries, stage, reasoning_effort, timeout, + ) + + +def chat_optimizer_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict]: + """Call the optimizer model with a pre-built chat message list.""" + return _chat_messages_impl( + get_optimizer_client(), + OPTIMIZER_DEPLOYMENT, + messages, + max_completion_tokens, + retries, + stage, + reasoning_effort, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_messages_with_deployment( + deployment: str, + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "custom", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict]: + """Call an arbitrary deployment with a pre-built chat message list.""" + return _chat_messages_impl( + get_optimizer_client(), + deployment, + messages, + max_completion_tokens, + retries, + stage, + reasoning_effort, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_target_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict]: + """Call the target model with a pre-built chat message list.""" + return _chat_messages_impl( + get_target_client(), + TARGET_DEPLOYMENT, + messages, + max_completion_tokens, + retries, + stage, + reasoning_effort, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def get_token_summary() -> dict: + """Return per-stage and total token usage.""" + return tracker.summary() + + +def reset_token_tracker() -> None: + tracker.reset() + + +def set_target_deployment(deployment: str) -> None: + """Change target deployment at runtime.""" + global _target_client, TARGET_DEPLOYMENT + TARGET_DEPLOYMENT = deployment + os.environ["TARGET_DEPLOYMENT"] = deployment + os.environ["AZURE_OPENAI_DEPLOYMENT"] = deployment + with _target_lock: + _target_client = None + try: + import llm_client as _legacy + _legacy.DEPLOYMENT = deployment + _legacy._client = None + except Exception: + pass + + +def set_reasoning_effort(effort: str | None) -> None: + """Set reasoning effort for all LLM calls. None = off.""" + global REASONING_EFFORT + REASONING_EFFORT = effort if effort else None + + +def get_reasoning_effort() -> str | None: + """Return the process-wide reasoning effort for direct Azure client users.""" + return REASONING_EFFORT + + +def set_optimizer_deployment(deployment: str) -> None: + """Change optimizer deployment at runtime.""" + global _optimizer_client, OPTIMIZER_DEPLOYMENT + OPTIMIZER_DEPLOYMENT = deployment + os.environ["OPTIMIZER_DEPLOYMENT"] = deployment + with _optimizer_lock: + _optimizer_client = None diff --git a/skillopt/model/backend_config.py b/skillopt/model/backend_config.py new file mode 100644 index 0000000..f23725c --- /dev/null +++ b/skillopt/model/backend_config.py @@ -0,0 +1,185 @@ +"""Runtime backend configuration for optimizer/target model calls.""" +from __future__ import annotations + +import os + +from skillopt.model.common import default_model_for_backend, normalize_backend_name + + +def _parse_bool(value: str | None, default: bool) -> bool: + if value is None: + return default + return str(value).strip().lower() in {"1", "true", "yes", "on"} + + +OPTIMIZER_BACKEND = normalize_backend_name(os.environ.get("OPTIMIZER_BACKEND", "openai_chat")) +TARGET_BACKEND = normalize_backend_name(os.environ.get("TARGET_BACKEND", "openai_chat")) + +CODEX_EXEC_PATH = os.environ.get("CODEX_EXEC_PATH", "codex") +CODEX_EXEC_SANDBOX = os.environ.get("CODEX_EXEC_SANDBOX", "workspace-write") +CODEX_EXEC_PROFILE = os.environ.get("CODEX_EXEC_PROFILE", "") +CODEX_EXEC_FULL_AUTO = _parse_bool(os.environ.get("CODEX_EXEC_FULL_AUTO"), True) +CODEX_EXEC_REASONING_EFFORT = os.environ.get("CODEX_EXEC_REASONING_EFFORT", "none") +CODEX_EXEC_USE_SDK = os.environ.get("CODEX_EXEC_USE_SDK", "auto") +CODEX_EXEC_NETWORK_ACCESS = _parse_bool(os.environ.get("CODEX_EXEC_NETWORK_ACCESS"), False) +CODEX_EXEC_WEB_SEARCH = _parse_bool(os.environ.get("CODEX_EXEC_WEB_SEARCH"), False) +CODEX_EXEC_APPROVAL_POLICY = os.environ.get("CODEX_EXEC_APPROVAL_POLICY", "never") +CLAUDE_CODE_EXEC_PATH = os.environ.get("CLAUDE_CODE_EXEC_PATH", "claude") +CLAUDE_CODE_EXEC_PROFILE = os.environ.get("CLAUDE_CODE_EXEC_PROFILE", "") +CLAUDE_CODE_EXEC_USE_SDK = os.environ.get("CLAUDE_CODE_EXEC_USE_SDK", "auto") +CLAUDE_CODE_EXEC_EFFORT = os.environ.get("CLAUDE_CODE_EXEC_EFFORT", "medium") + + +def _parse_int(value: str | None, default: int) -> int: + if value is None: + return default + try: + return int(str(value).strip()) + except ValueError: + return default + + +EXEC_EMPTY_RESPONSE_RETRIES = max(0, _parse_int(os.environ.get("EXEC_EMPTY_RESPONSE_RETRIES"), 1)) +CLAUDE_CODE_EXEC_MAX_THINKING_TOKENS = max( + 0, + _parse_int(os.environ.get("CLAUDE_CODE_EXEC_MAX_THINKING_TOKENS"), 16384), +) + + +def set_optimizer_backend(backend: str) -> None: + global OPTIMIZER_BACKEND + OPTIMIZER_BACKEND = normalize_backend_name(backend or "openai_chat") + if OPTIMIZER_BACKEND not in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat"}: + raise ValueError( + f"Unsupported optimizer backend: {OPTIMIZER_BACKEND!r}. " + "Supported values are 'openai_chat', 'claude_chat', 'qwen_chat', and 'minimax_chat'." + ) + os.environ["OPTIMIZER_BACKEND"] = OPTIMIZER_BACKEND + + +def get_optimizer_backend() -> str: + return OPTIMIZER_BACKEND + + +def set_target_backend(backend: str) -> None: + global TARGET_BACKEND + TARGET_BACKEND = normalize_backend_name(backend or "openai_chat") + if TARGET_BACKEND not in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat", "codex_exec", "claude_code_exec"}: + raise ValueError( + f"Unsupported target backend: {TARGET_BACKEND!r}. " + "Supported values are 'openai_chat', 'claude_chat', 'qwen_chat', 'minimax_chat', 'codex_exec', and 'claude_code_exec'." + ) + os.environ["TARGET_BACKEND"] = TARGET_BACKEND + + +def get_target_backend() -> str: + return TARGET_BACKEND + + +def is_target_exec_backend() -> bool: + return TARGET_BACKEND in {"codex_exec", "claude_code_exec"} + + +def is_optimizer_chat_backend() -> bool: + return OPTIMIZER_BACKEND in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat"} + + +def is_target_chat_backend() -> bool: + return TARGET_BACKEND in {"openai_chat", "claude_chat", "qwen_chat", "minimax_chat"} + + +def configure_codex_exec( + *, + path: str | None = None, + sandbox: str | None = None, + profile: str | None = None, + full_auto: bool | None = None, + reasoning_effort: str | None = None, + use_sdk: str | None = None, + network_access: bool | None = None, + web_search: bool | None = None, + approval_policy: str | None = None, +) -> None: + global CODEX_EXEC_PATH, CODEX_EXEC_SANDBOX, CODEX_EXEC_PROFILE, CODEX_EXEC_FULL_AUTO, CODEX_EXEC_REASONING_EFFORT, CODEX_EXEC_USE_SDK, CODEX_EXEC_NETWORK_ACCESS, CODEX_EXEC_WEB_SEARCH, CODEX_EXEC_APPROVAL_POLICY + if path is not None: + CODEX_EXEC_PATH = str(path).strip() or "codex" + os.environ["CODEX_EXEC_PATH"] = CODEX_EXEC_PATH + if sandbox is not None: + CODEX_EXEC_SANDBOX = str(sandbox).strip() or "workspace-write" + os.environ["CODEX_EXEC_SANDBOX"] = CODEX_EXEC_SANDBOX + if profile is not None: + CODEX_EXEC_PROFILE = str(profile).strip() + os.environ["CODEX_EXEC_PROFILE"] = CODEX_EXEC_PROFILE + if full_auto is not None: + CODEX_EXEC_FULL_AUTO = bool(full_auto) + os.environ["CODEX_EXEC_FULL_AUTO"] = "true" if CODEX_EXEC_FULL_AUTO else "false" + if reasoning_effort is not None: + CODEX_EXEC_REASONING_EFFORT = str(reasoning_effort).strip() or "none" + os.environ["CODEX_EXEC_REASONING_EFFORT"] = CODEX_EXEC_REASONING_EFFORT + if use_sdk is not None: + CODEX_EXEC_USE_SDK = str(use_sdk).strip().lower() or "auto" + os.environ["CODEX_EXEC_USE_SDK"] = CODEX_EXEC_USE_SDK + if network_access is not None: + CODEX_EXEC_NETWORK_ACCESS = bool(network_access) + os.environ["CODEX_EXEC_NETWORK_ACCESS"] = "true" if CODEX_EXEC_NETWORK_ACCESS else "false" + if web_search is not None: + CODEX_EXEC_WEB_SEARCH = bool(web_search) + os.environ["CODEX_EXEC_WEB_SEARCH"] = "true" if CODEX_EXEC_WEB_SEARCH else "false" + if approval_policy is not None: + CODEX_EXEC_APPROVAL_POLICY = str(approval_policy).strip() or "never" + os.environ["CODEX_EXEC_APPROVAL_POLICY"] = CODEX_EXEC_APPROVAL_POLICY + + +def get_codex_exec_config() -> dict[str, str | bool | int]: + return { + "path": CODEX_EXEC_PATH, + "sandbox": CODEX_EXEC_SANDBOX, + "profile": CODEX_EXEC_PROFILE, + "full_auto": CODEX_EXEC_FULL_AUTO, + "reasoning_effort": CODEX_EXEC_REASONING_EFFORT, + "use_sdk": CODEX_EXEC_USE_SDK, + "network_access": CODEX_EXEC_NETWORK_ACCESS, + "web_search": CODEX_EXEC_WEB_SEARCH, + "approval_policy": CODEX_EXEC_APPROVAL_POLICY, + "empty_response_retries": EXEC_EMPTY_RESPONSE_RETRIES, + } + + +def configure_claude_code_exec( + *, + path: str | None = None, + profile: str | None = None, + use_sdk: str | None = None, + effort: str | None = None, + max_thinking_tokens: int | str | None = None, +) -> None: + global CLAUDE_CODE_EXEC_PATH, CLAUDE_CODE_EXEC_PROFILE, CLAUDE_CODE_EXEC_USE_SDK, CLAUDE_CODE_EXEC_EFFORT, CLAUDE_CODE_EXEC_MAX_THINKING_TOKENS + if path is not None: + CLAUDE_CODE_EXEC_PATH = str(path).strip() or "claude" + os.environ["CLAUDE_CODE_EXEC_PATH"] = CLAUDE_CODE_EXEC_PATH + if profile is not None: + CLAUDE_CODE_EXEC_PROFILE = str(profile).strip() + os.environ["CLAUDE_CODE_EXEC_PROFILE"] = CLAUDE_CODE_EXEC_PROFILE + if use_sdk is not None: + CLAUDE_CODE_EXEC_USE_SDK = str(use_sdk).strip().lower() or "auto" + os.environ["CLAUDE_CODE_EXEC_USE_SDK"] = CLAUDE_CODE_EXEC_USE_SDK + if effort is not None: + CLAUDE_CODE_EXEC_EFFORT = str(effort).strip().lower() or "medium" + os.environ["CLAUDE_CODE_EXEC_EFFORT"] = CLAUDE_CODE_EXEC_EFFORT + if max_thinking_tokens is not None: + CLAUDE_CODE_EXEC_MAX_THINKING_TOKENS = max( + 0, + _parse_int(str(max_thinking_tokens), 16384), + ) + os.environ["CLAUDE_CODE_EXEC_MAX_THINKING_TOKENS"] = str(CLAUDE_CODE_EXEC_MAX_THINKING_TOKENS) + + +def get_claude_code_exec_config() -> dict[str, str | int]: + return { + "path": CLAUDE_CODE_EXEC_PATH, + "profile": CLAUDE_CODE_EXEC_PROFILE, + "use_sdk": CLAUDE_CODE_EXEC_USE_SDK, + "effort": CLAUDE_CODE_EXEC_EFFORT, + "max_thinking_tokens": CLAUDE_CODE_EXEC_MAX_THINKING_TOKENS, + "empty_response_retries": EXEC_EMPTY_RESPONSE_RETRIES, + } diff --git a/skillopt/model/claude_backend.py b/skillopt/model/claude_backend.py new file mode 100644 index 0000000..b2d0e94 --- /dev/null +++ b/skillopt/model/claude_backend.py @@ -0,0 +1,371 @@ +"""Claude CLI chat backend for ReflACT.""" +from __future__ import annotations + +import base64 +import json +import mimetypes +import os +import shutil +import subprocess +import tempfile +import time +from typing import Any +from urllib.parse import unquote, urlparse + +from skillopt.model.common import CompatAssistantMessage, CompatToolCall, CompatToolFunction, default_model_for_backend, tracker + +CLAUDE_BIN = os.environ.get("CLAUDE_CLI_BIN", "claude") +CLAUDE_PERMISSION_MODE = os.environ.get("CLAUDE_PERMISSION_MODE", "dontAsk") +CLAUDE_SETTING_SOURCES = os.environ.get("CLAUDE_SETTING_SOURCES", "user,project") +CLAUDE_ALLOW_ATTACHMENT_READ = os.environ.get("CLAUDE_ALLOW_ATTACHMENT_READ", "1").strip().lower() not in {"0", "false", "no"} + +OPTIMIZER_DEPLOYMENT = os.environ.get("OPTIMIZER_DEPLOYMENT", "claude-sonnet-4-6") +TARGET_DEPLOYMENT = os.environ.get("TARGET_DEPLOYMENT", "claude-sonnet-4-6") +REASONING_EFFORT: str | None = None +_VALID_EFFORTS = {"low", "medium", "high", "xhigh", "max"} + + +def _parse_data_uri(url: str) -> tuple[bytes, str]: + header, data = url.split(",", 1) + mime = header[5:].split(";", 1)[0] or "image/png" + return base64.b64decode(data), mime + + +def _content_to_text(content: Any, attachments: list[dict[str, Any]], *, image_counter: int) -> tuple[str, int]: + if isinstance(content, str): + return content, image_counter + if not isinstance(content, list): + return str(content), image_counter + parts: list[str] = [] + for item in content: + if not isinstance(item, dict): + continue + item_type = item.get("type") + if item_type == "text": + parts.append(str(item.get("text", ""))) + continue + if item_type != "image_url": + continue + image_counter += 1 + label = f"[Attached image {image_counter}]" + parts.append(label) + image_url = item.get("image_url", {}) or {} + url = str(image_url.get("url", "") or "") + if not url: + continue + if url.startswith("data:") and ";base64," in url: + data, mime = _parse_data_uri(url) + attachments.append({"bytes": data, "mime": mime, "label": label}) + continue + if url.startswith("file://"): + parsed = urlparse(url) + path = unquote(parsed.path) + if path: + attachments.append({"path": path, "label": label}) + continue + if os.path.exists(url): + attachments.append({"path": url, "label": label}) + return "".join(parts), image_counter + + +def _simplify_tool_schemas(tools: list[dict[str, Any]] | None) -> list[dict[str, Any]]: + simplified: list[dict[str, Any]] = [] + for tool in tools or []: + function = tool.get("function", tool) + simplified.append({ + "name": function.get("name", ""), + "description": function.get("description", ""), + "parameters": function.get("parameters", {}), + }) + return simplified + + +def _build_prompt_from_messages(messages: list[dict[str, Any]], *, tools: list[dict[str, Any]] | None = None, tool_choice: str | dict[str, Any] | None = None, structured_output: bool = False) -> tuple[str, str, list[dict[str, Any]]]: + system_parts: list[str] = [] + history_parts: list[str] = [] + attachments: list[dict[str, Any]] = [] + image_counter = 0 + + def _history_line(label: str, body: str) -> str: + stripped = body.strip() + if not stripped: + return f"- {label}:" + indented = stripped.replace("\n", "\n ") + return f"- {label}: {indented}" + + for message in messages: + role = str(message.get("role", "user")) + text, image_counter = _content_to_text(message.get("content", ""), attachments, image_counter=image_counter) + if role == "system": + if text.strip(): + system_parts.append(text.strip()) + continue + if role == "assistant": + block = _history_line("Assistant", text) + tool_calls = message.get("tool_calls") or [] + if tool_calls: + simplified_calls = [] + for tool_call in tool_calls: + function = tool_call.get("function", {}) or {} + simplified_calls.append({ + "name": function.get("name", ""), + "arguments": function.get("arguments", "{}"), + }) + block += "\n Compatibility tool requests:\n" + json.dumps(simplified_calls, ensure_ascii=False, indent=2) + history_parts.append(block) + continue + if role == "tool": + tool_call_id = str(message.get("tool_call_id", "") or "") + history_parts.append(_history_line(f"Tool result (tool_call_id={tool_call_id})", text)) + continue + history_parts.append(_history_line(role.capitalize(), text)) + + prompt_parts: list[str] = [] + if tools: + simplified_tools = _simplify_tool_schemas(tools) + prompt_parts.append("Available compatibility tools:\n" + json.dumps(simplified_tools, ensure_ascii=False, indent=2)) + prompt_parts.append("Do not execute these compatibility tools yourself. If you need one, request it in `tool_calls`. Each `arguments` field must be a JSON string.") + if tool_choice == "required": + prompt_parts.append("Tool choice policy: you must request at least one compatibility tool.") + elif isinstance(tool_choice, dict) and tool_choice.get("type") == "function": + function = tool_choice.get("function", {}) or {} + prompt_parts.append(f"Tool choice policy: you must request the compatibility tool `{function.get('name', '')}`.") + history_text = "\n".join(part for part in history_parts if part).strip() + if history_text: + prompt_parts.append("History:\n" + history_text) + if structured_output: + prompt_parts.append("Return only JSON matching the provided schema.") + if tools: + prompt_parts.append("Set `content` to the assistant-visible reply. Set `tool_calls` to an empty array when no compatibility tool is needed.") + else: + prompt_parts.append("Answer the latest user request.") + return "\n\n".join(part for part in system_parts if part).strip(), "\n\n".join(prompt_parts), attachments + + +def _copy_attachments_to_temp(attachments: list[dict[str, Any]], temp_dir: str) -> list[dict[str, str]]: + copied: list[dict[str, str]] = [] + for index, attachment in enumerate(attachments, 1): + source_path = attachment.get("path") + if source_path: + source_path = str(source_path) + source_suffix = os.path.splitext(source_path)[1] + target_path = os.path.join(temp_dir, f"image_{index}{source_suffix or '.bin'}") + shutil.copyfile(source_path, target_path) + copied.append({"path": target_path, "label": str(attachment.get("label", ""))}) + continue + mime = str(attachment.get("mime", "image/png")) + suffix = mimetypes.guess_extension(mime) or ".png" + target_path = os.path.join(temp_dir, f"image_{index}{suffix}") + with open(target_path, "wb") as f: + f.write(attachment.get("bytes", b"") or b"") + copied.append({"path": target_path, "label": str(attachment.get("label", ""))}) + return copied + + +def _append_attachment_instructions(prompt: str, copied_attachments: list[dict[str, str]]) -> str: + if not copied_attachments or not CLAUDE_ALLOW_ATTACHMENT_READ: + return prompt + lines = [ + "Attached image files:", + *[f"- {item['label'] or f'Attached image {index}'}: {item['path']}" for index, item in enumerate(copied_attachments, 1)], + "If you need to inspect an attached image, you may use the built-in `Read` tool on those listed paths only. Do not use built-in tools for any other purpose.", + ] + return prompt.rstrip() + "\n\n" + "\n".join(lines) + + +def _usage_from_result(result_event: dict[str, Any] | None) -> dict[str, int]: + usage = (result_event or {}).get("usage", {}) or {} + input_tokens = int(usage.get("input_tokens", 0) or 0) + output_tokens = int(usage.get("output_tokens", 0) or 0) + return { + "prompt_tokens": input_tokens, + "completion_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens, + } + + +def _extract_result(event_stream: list[dict[str, Any]]) -> tuple[str, dict[str, Any] | None]: + result_event = None + for event in reversed(event_stream): + if event.get("type") == "result": + result_event = event + break + if result_event is None: + raise RuntimeError("Claude backend did not return a result event.") + content = result_event.get("result") or result_event.get("content") or "" + return str(content), result_event + + +def _check_claude_error(stderr_text: str, model: str) -> None: + lowered = stderr_text.lower() + if "invalid api key" in lowered or "authentication" in lowered or "login" in lowered: + raise RuntimeError("Claude CLI is not logged in. Run `claude auth login` (or start `claude` and use `/login`) first.") + if "unknown model" in lowered or "not available" in lowered or "invalid model" in lowered: + default_model = default_model_for_backend("claude") + raise RuntimeError(f"Claude backend tried to use model {model!r}, but your current Claude CLI/account rejected it. Try an available Claude model such as {default_model!r}.") + + +def _normalize_reasoning_effort(effort: str | None) -> str | None: + normalized = str(effort or "").strip().lower() + if not normalized or normalized == "off": + return None + if normalized in _VALID_EFFORTS: + return normalized + return None + + +def _assistant_message_schema() -> dict[str, Any]: + return { + "type": "object", + "properties": { + "content": {"type": "string"}, + "tool_calls": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "arguments": {"type": "string"}, + }, + "required": ["name", "arguments"], + "additionalProperties": False, + }, + }, + }, + "required": ["content", "tool_calls"], + "additionalProperties": False, + } + + +def _assistant_message_schema_wrapper() -> str: + return json.dumps(_assistant_message_schema(), ensure_ascii=False) + + +def _run_claude_print(*, system: str, prompt: str, model: str, tools: list[dict[str, Any]] | None, tool_choice: str | dict[str, Any] | None, return_message: bool, timeout: int | None, attachments: list[dict[str, Any]] | None = None) -> tuple[str, dict[str, Any], dict[str, int]]: + effort = _normalize_reasoning_effort(REASONING_EFFORT) + with tempfile.TemporaryDirectory(prefix="skillopt_claude_") as temp_dir: + copied_attachments = _copy_attachments_to_temp(attachments or [], temp_dir) + prompt_for_cli = _append_attachment_instructions(prompt, copied_attachments) + cmd = [CLAUDE_BIN, "-p", "--output-format", "json", "--permission-mode", CLAUDE_PERMISSION_MODE, "--add-dir", temp_dir] + if model: + cmd.extend(["--model", model]) + if CLAUDE_SETTING_SOURCES: + cmd.extend(["--setting-sources", CLAUDE_SETTING_SOURCES]) + if system: + # Write the system prompt to a file, not argv: here the skill being + # optimized IS the system prompt, and SkillOpt grows it over training, + # so past ~30 KB it would re-hit the Windows argv cap (WinError 206). + # The CLI reads it via --append-system-prompt-file. + system_path = os.path.join(temp_dir, "system_prompt.txt") + with open(system_path, "w", encoding="utf-8") as system_fh: + system_fh.write(system) + cmd.extend(["--append-system-prompt-file", system_path]) + if effort: + cmd.extend(["--effort", effort]) + structured_output = bool(return_message) + if structured_output: + cmd.extend(["--schema", _assistant_message_schema_wrapper()]) + # Feed the prompt via stdin (and the system prompt via a file, above), not + # argv: on Windows the whole command line is capped at ~32 KB and large + # optimizer prompts / grown skills overflow it → [WinError 206]. Pin UTF-8 + # so a zh-CN default codepage (cp936) can't raise UnicodeEncodeError on + # emoji / non-GBK glyphs before the CLI even starts. + proc = subprocess.run(cmd, input=prompt_for_cli, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout or 300, cwd=temp_dir) + stderr_text = (proc.stderr or "").strip() + if proc.returncode != 0: + _check_claude_error(stderr_text, model) + raise RuntimeError(stderr_text or f"Claude CLI exited with code {proc.returncode}") + stream = [] + for raw_line in (proc.stdout or "").splitlines(): + raw_line = raw_line.strip() + if not raw_line: + continue + try: + stream.append(json.loads(raw_line)) + except json.JSONDecodeError: + continue + raw_text, result_event = _extract_result(stream) + usage_info = _usage_from_result(result_event) + return raw_text, result_event or {}, usage_info + + +def _compat_message_from_payload(payload: Any) -> CompatAssistantMessage: + if not isinstance(payload, dict): + return CompatAssistantMessage(content=str(payload or ""), tool_calls=[]) + content = str(payload.get("content", "") or "") + tool_calls: list[CompatToolCall] = [] + for index, tool_call in enumerate(payload.get("tool_calls", []) or [], start=1): + name = str(tool_call.get("name", "") or "") + arguments = str(tool_call.get("arguments", "{}") or "{}") + tool_calls.append(CompatToolCall(id=f"claude_tool_{index}", function=CompatToolFunction(name=name, arguments=arguments))) + return CompatAssistantMessage(content=content, tool_calls=tool_calls) + + +def _call_messages(messages: list[dict[str, Any]], max_completion_tokens: int, retries: int, stage: str, *, tools: list[dict[str, Any]] | None = None, tool_choice: str | dict[str, Any] | None = None, return_message: bool = False, deployment: str | None = None, timeout: int | None = None) -> tuple[Any, dict[str, int]]: + del max_completion_tokens + system, prompt, attachments = _build_prompt_from_messages(messages, tools=tools, tool_choice=tool_choice, structured_output=return_message) + model = deployment or TARGET_DEPLOYMENT + last_err = None + for attempt in range(retries): + try: + raw_text, payload, usage_info = _run_claude_print(system=system, prompt=prompt, model=model, tools=tools, tool_choice=tool_choice, return_message=return_message, timeout=timeout, attachments=attachments) + tracker.record(stage, usage_info["prompt_tokens"], usage_info["completion_tokens"]) + if return_message: + return _compat_message_from_payload(payload.get("result", payload)), usage_info + return raw_text, usage_info + except Exception as e: # noqa: BLE001 + last_err = e + time.sleep(min(2 ** attempt, 15)) + raise RuntimeError(f"Claude backend failed after {retries} retries: {last_err}") + + +def chat_optimizer(system: str, user: str, max_completion_tokens: int = 16384, retries: int = 5, stage: str = "optimizer", timeout: int | None = None) -> tuple[str, dict[str, int]]: + messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] + return _call_messages(messages, max_completion_tokens, retries, stage, deployment=OPTIMIZER_DEPLOYMENT, timeout=timeout) + + +def chat_target(system: str, user: str, max_completion_tokens: int = 16384, retries: int = 5, stage: str = "target", timeout: int | None = None) -> tuple[str, dict[str, int]]: + messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] + return _call_messages(messages, max_completion_tokens, retries, stage, deployment=TARGET_DEPLOYMENT, timeout=timeout) + + +def chat_with_deployment(deployment: str, system: str, user: str, max_completion_tokens: int = 16384, retries: int = 5, stage: str = "custom", timeout: int | None = None) -> tuple[str, dict[str, int]]: + messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] + return _call_messages(messages, max_completion_tokens, retries, stage, deployment=deployment, timeout=timeout) + + +def chat_optimizer_messages(messages: list[dict[str, Any]], max_completion_tokens: int = 16384, retries: int = 5, stage: str = "optimizer", *, tools: list[dict[str, Any]] | None = None, tool_choice: str | dict[str, Any] | None = None, return_message: bool = False, timeout: int | None = None) -> tuple[Any, dict[str, int]]: + return _call_messages(messages, max_completion_tokens, retries, stage, tools=tools, tool_choice=tool_choice, return_message=return_message, deployment=OPTIMIZER_DEPLOYMENT, timeout=timeout) + + +def chat_target_messages(messages: list[dict[str, Any]], max_completion_tokens: int = 16384, retries: int = 5, stage: str = "target", *, tools: list[dict[str, Any]] | None = None, tool_choice: str | dict[str, Any] | None = None, return_message: bool = False, timeout: int | None = None) -> tuple[Any, dict[str, int]]: + return _call_messages(messages, max_completion_tokens, retries, stage, tools=tools, tool_choice=tool_choice, return_message=return_message, deployment=TARGET_DEPLOYMENT, timeout=timeout) + + +def chat_messages_with_deployment(deployment: str, messages: list[dict[str, Any]], max_completion_tokens: int = 16384, retries: int = 5, stage: str = "custom", *, tools: list[dict[str, Any]] | None = None, tool_choice: str | dict[str, Any] | None = None, return_message: bool = False, timeout: int | None = None) -> tuple[Any, dict[str, int]]: + return _call_messages(messages, max_completion_tokens, retries, stage, tools=tools, tool_choice=tool_choice, return_message=return_message, deployment=deployment, timeout=timeout) + + +def get_token_summary() -> dict[str, dict[str, int]]: + return tracker.summary() + + +def reset_token_tracker() -> None: + tracker.reset() + + +def set_reasoning_effort(effort: str | None) -> None: + global REASONING_EFFORT + REASONING_EFFORT = effort if effort else None + + +def set_target_deployment(deployment: str) -> None: + global TARGET_DEPLOYMENT + TARGET_DEPLOYMENT = deployment or default_model_for_backend("claude") + os.environ["TARGET_DEPLOYMENT"] = TARGET_DEPLOYMENT + + +def set_optimizer_deployment(deployment: str) -> None: + global OPTIMIZER_DEPLOYMENT + OPTIMIZER_DEPLOYMENT = deployment or default_model_for_backend("claude") + os.environ["OPTIMIZER_DEPLOYMENT"] = OPTIMIZER_DEPLOYMENT diff --git a/skillopt/model/codex_backend.py b/skillopt/model/codex_backend.py new file mode 100644 index 0000000..64b6f35 --- /dev/null +++ b/skillopt/model/codex_backend.py @@ -0,0 +1,666 @@ +"""Codex CLI backend for ReflACT.""" +from __future__ import annotations + +import base64 +import json +import mimetypes +import os +import subprocess +import tempfile +import time +import uuid +from typing import Any +from urllib.parse import unquote, urlparse + +from skillopt.model.common import ( + CompatAssistantMessage, + CompatToolCall, + CompatToolFunction, + tracker, +) + + +CODEX_BIN = os.environ.get("CODEX_CLI_BIN", "codex") +CODEX_PROFILE = os.environ.get("CODEX_PROFILE", "review") +CODEX_SANDBOX_MODE = os.environ.get("CODEX_SANDBOX_MODE", "read-only") + +OPTIMIZER_DEPLOYMENT = os.environ.get("OPTIMIZER_DEPLOYMENT", "gpt-4o") +TARGET_DEPLOYMENT = os.environ.get("TARGET_DEPLOYMENT", "gpt-4o") + +REASONING_EFFORT: str | None = None + + +def _default_working_directory() -> str: + return os.environ.get("CODEX_WORKING_DIRECTORY", os.getcwd()) + + +def _parse_data_uri(url: str) -> tuple[bytes, str]: + header, data = url.split(",", 1) + mime = header[5:].split(";", 1)[0] or "image/png" + return base64.b64decode(data), mime + + +def _content_to_text( + content: Any, + attachments: list[dict[str, Any]], + *, + image_counter: int, +) -> tuple[str, int]: + if isinstance(content, str): + return content, image_counter + + if not isinstance(content, list): + return str(content), image_counter + + parts: list[str] = [] + for item in content: + if not isinstance(item, dict): + continue + item_type = item.get("type") + if item_type == "text": + parts.append(str(item.get("text", ""))) + continue + if item_type != "image_url": + continue + + image_counter += 1 + label = f"[Attached image {image_counter}]" + parts.append(label) + + image_url = item.get("image_url", {}) or {} + url = str(image_url.get("url", "") or "") + if not url: + continue + if url.startswith("data:") and ";base64," in url: + data, mime = _parse_data_uri(url) + attachments.append({"bytes": data, "mime": mime}) + continue + if url.startswith("file://"): + parsed = urlparse(url) + path = unquote(parsed.path) + if path: + attachments.append({"path": path}) + continue + if os.path.exists(url): + attachments.append({"path": url}) + + return "".join(parts), image_counter + + +def _simplify_tool_schemas(tools: list[dict[str, Any]] | None) -> list[dict[str, Any]]: + simplified: list[dict[str, Any]] = [] + for tool in tools or []: + function = tool.get("function", tool) + simplified.append( + { + "name": function.get("name", ""), + "description": function.get("description", ""), + "parameters": function.get("parameters", {}), + } + ) + return simplified + + +def _build_prompt_from_messages( + messages: list[dict[str, Any]], + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + structured_output: bool = False, +) -> tuple[str, list[dict[str, Any]]]: + system_parts: list[str] = [] + history_parts: list[str] = [] + attachments: list[dict[str, Any]] = [] + image_counter = 0 + + def _history_line(label: str, body: str) -> str: + stripped = body.strip() + if not stripped: + return f"- {label}:" + indented = stripped.replace("\n", "\n ") + return f"- {label}: {indented}" + + for message in messages: + role = str(message.get("role", "user")) + text, image_counter = _content_to_text( + message.get("content", ""), + attachments, + image_counter=image_counter, + ) + + if role == "system": + if text.strip(): + system_parts.append(text.strip()) + continue + + if role == "assistant": + block = _history_line("Assistant", text) + tool_calls = message.get("tool_calls") or [] + if tool_calls: + simplified_calls = [] + for tool_call in tool_calls: + function = tool_call.get("function", {}) or {} + simplified_calls.append( + { + "name": function.get("name", ""), + "arguments": function.get("arguments", "{}"), + } + ) + block += ( + "\n Compatibility tool requests:\n" + + json.dumps(simplified_calls, ensure_ascii=False, indent=2) + ) + history_parts.append(block) + continue + + if role == "tool": + tool_call_id = str(message.get("tool_call_id", "") or "") + label = f"Tool result (tool_call_id={tool_call_id})" + history_parts.append(_history_line(label, text)) + continue + + history_parts.append(_history_line(role.capitalize(), text)) + + prompt_parts: list[str] = [] + + system_text = "\n\n".join(part for part in system_parts if part).strip() + if system_text: + prompt_parts.append(system_text) + + if tools: + simplified_tools = _simplify_tool_schemas(tools) + prompt_parts.append( + "Available compatibility tools:\n" + + json.dumps(simplified_tools, ensure_ascii=False, indent=2) + ) + prompt_parts.append( + "Do not execute these tools yourself. If you need one, request it in " + "`tool_calls`. Each `arguments` field must be a JSON string." + ) + + if tool_choice == "required": + prompt_parts.append( + "Tool choice policy: you must request at least one compatibility tool." + ) + elif isinstance(tool_choice, dict) and tool_choice.get("type") == "function": + function = tool_choice.get("function", {}) or {} + prompt_parts.append( + "Tool choice policy: you must request the compatibility tool " + f"`{function.get('name', '')}`." + ) + + history_text = "\n".join(part for part in history_parts if part).strip() + if history_text: + prompt_parts.append("History:\n" + history_text) + + if structured_output: + prompt_parts.append("Return only JSON matching the provided schema.") + if tools: + prompt_parts.append( + "Set `content` to the assistant-visible reply. Set `tool_calls` to " + "an empty array when no tool is needed." + ) + else: + prompt_parts.append("Answer the latest user request.") + + return "\n\n".join(prompt_parts), attachments + + +def _assistant_message_schema() -> dict[str, Any]: + return { + "type": "object", + "properties": { + "content": {"type": "string"}, + "tool_calls": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "arguments": {"type": "string"}, + }, + "required": ["name", "arguments"], + "additionalProperties": False, + }, + }, + }, + "required": ["content", "tool_calls"], + "additionalProperties": False, + } + + +def _materialize_attachments( + attachments: list[dict[str, Any]], + temp_dir: str, +) -> list[str]: + image_paths: list[str] = [] + for index, attachment in enumerate(attachments, 1): + path = attachment.get("path") + if path: + image_paths.append(str(path)) + continue + + mime = str(attachment.get("mime", "image/png")) + suffix = mimetypes.guess_extension(mime) or ".png" + image_path = os.path.join(temp_dir, f"image_{index}{suffix}") + with open(image_path, "wb") as f: + f.write(attachment.get("bytes", b"")) + image_paths.append(image_path) + return image_paths + + +def _usage_from_event(usage: dict[str, Any] | None) -> dict[str, int]: + usage = usage or {} + prompt_tokens = int(usage.get("input_tokens", 0) or 0) + completion_tokens = int(usage.get("output_tokens", 0) or 0) + return { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + } + + +def _extract_error(stdout: str, stderr: str) -> str: + for raw_line in reversed(stdout.splitlines()): + line = raw_line.strip() + if not line: + continue + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if payload.get("type") == "turn.failed": + error = payload.get("error", {}) or {} + return str(error.get("message", "") or "Codex turn failed") + if payload.get("type") == "error": + return str(payload.get("message", "") or "Codex execution failed") + return stderr.strip() or stdout.strip() or "Codex execution failed" + + +def _run_codex_exec( + *, + model: str, + prompt: str, + attachments: list[dict[str, Any]], + output_schema: dict[str, Any] | None, + timeout: int | None, +) -> tuple[str, dict[str, int]]: + with tempfile.TemporaryDirectory(prefix="skillopt_codex_") as temp_dir: + output_path = os.path.join(temp_dir, "last_message.txt") + image_paths = _materialize_attachments(attachments, temp_dir) + + command = [ + CODEX_BIN, + "exec", + "--json", + "--ephemeral", + "--profile", + CODEX_PROFILE, + "-c", + "approval_policy=\"never\"", + "--sandbox", + CODEX_SANDBOX_MODE, + "--skip-git-repo-check", + "--cd", + _default_working_directory(), + "--model", + model, + "--output-last-message", + output_path, + ] + + if REASONING_EFFORT: + command.extend(["-c", f"model_reasoning_effort={json.dumps(REASONING_EFFORT)}"]) + + schema_path = None + if output_schema is not None: + schema_path = os.path.join(temp_dir, "schema.json") + with open(schema_path, "w", encoding="utf-8") as f: + json.dump(output_schema, f, ensure_ascii=False) + command.extend(["--output-schema", schema_path]) + + for image_path in image_paths: + command.extend(["--image", image_path]) + + command.append("-") + + proc = subprocess.run( + command, + input=prompt, + text=True, + encoding="utf-8", + errors="replace", + capture_output=True, + timeout=timeout, + check=False, + ) + + usage_info = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + fallback_text = "" + for raw_line in proc.stdout.splitlines(): + line = raw_line.strip() + if not line: + continue + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if payload.get("type") == "item.completed": + item = payload.get("item", {}) or {} + if item.get("type") == "agent_message": + fallback_text = str(item.get("text", "") or fallback_text) + if payload.get("type") == "turn.completed": + usage_info = _usage_from_event(payload.get("usage")) + + last_message = "" + if os.path.exists(output_path): + with open(output_path, encoding="utf-8") as f: + last_message = f.read().strip() + if not last_message: + last_message = fallback_text.strip() + + if proc.returncode != 0: + raise RuntimeError(_extract_error(proc.stdout, proc.stderr)) + if not last_message: + raise RuntimeError("Codex returned an empty final message") + return last_message, usage_info + + +def _tool_name_from_choice(tool_choice: str | dict[str, Any] | None) -> str | None: + if not isinstance(tool_choice, dict): + return None + if tool_choice.get("type") != "function": + return None + function = tool_choice.get("function", {}) or {} + return str(function.get("name", "") or "") or None + + +def _compat_message_from_payload( + payload: dict[str, Any], + *, + tool_choice: str | dict[str, Any] | None = None, +) -> CompatAssistantMessage: + content = str(payload.get("content", "") or "") + tool_calls: list[CompatToolCall] = [] + for index, raw_tool_call in enumerate(payload.get("tool_calls", []) or [], 1): + if not isinstance(raw_tool_call, dict): + continue + name = str(raw_tool_call.get("name", "") or "") + arguments = raw_tool_call.get("arguments", "{}") + if not isinstance(arguments, str): + arguments = json.dumps(arguments, ensure_ascii=False) + tool_calls.append( + CompatToolCall( + id=f"tool_{index}_{uuid.uuid4().hex[:12]}", + function=CompatToolFunction(name=name, arguments=arguments), + ) + ) + + if tool_choice == "required" and not tool_calls: + raise RuntimeError("Codex response did not request a tool under tool_choice='required'") + + required_name = _tool_name_from_choice(tool_choice) + if required_name and all( + tool_call.function.name != required_name for tool_call in tool_calls + ): + raise RuntimeError( + f"Codex response did not request the required tool {required_name!r}" + ) + + return CompatAssistantMessage(content=content, tool_calls=tool_calls) + + +def _chat_messages_impl( + model: str, + messages: list[dict[str, Any]], + max_completion_tokens: int, + retries: int, + stage: str, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict[str, int]]: + del max_completion_tokens + last_err = None + structured_output = bool(tools) or return_message + + for attempt in range(retries): + try: + prompt, attachments = _build_prompt_from_messages( + messages, + tools=tools, + tool_choice=tool_choice, + structured_output=structured_output, + ) + raw_text, usage_info = _run_codex_exec( + model=model, + prompt=prompt, + attachments=attachments, + output_schema=_assistant_message_schema() if structured_output else None, + timeout=timeout, + ) + tracker.record( + stage, + usage_info["prompt_tokens"], + usage_info["completion_tokens"], + ) + + if not structured_output: + return raw_text, usage_info + + payload = json.loads(raw_text) + compat = _compat_message_from_payload(payload, tool_choice=tool_choice) + return (compat if return_message else compat.content), usage_info + except subprocess.TimeoutExpired as exc: + last_err = RuntimeError(f"Codex CLI timed out after {timeout}s") if timeout else exc + except Exception as exc: # noqa: BLE001 + last_err = exc + time.sleep(min(2 ** attempt, 30)) + + raise RuntimeError(f"Codex call failed after {retries} retries: {last_err}") + + +def chat_with_model( + model: str, + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "custom", + timeout: int | None = None, +) -> tuple[str, dict[str, int]]: + messages = [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + return _chat_messages_impl( + model, + messages, + max_completion_tokens, + retries, + stage, + timeout=timeout, + ) + + +def chat_messages_with_model( + model: str, + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "custom", + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict[str, int]]: + return _chat_messages_impl( + model, + messages, + max_completion_tokens, + retries, + stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_optimizer( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + timeout: int | None = None, +) -> tuple[str, dict[str, int]]: + return chat_with_model( + model=OPTIMIZER_DEPLOYMENT, + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + timeout=timeout, + ) + + +def chat_with_deployment( + deployment: str, + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "custom", + timeout: int | None = None, +) -> tuple[str, dict[str, int]]: + return chat_with_model( + model=deployment, + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + timeout=timeout, + ) + + +def chat_target( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + timeout: int | None = None, +) -> tuple[str, dict[str, int]]: + return chat_with_model( + model=TARGET_DEPLOYMENT, + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + timeout=timeout, + ) + + +def chat_optimizer_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict[str, int]]: + return _chat_messages_impl( + OPTIMIZER_DEPLOYMENT, + messages, + max_completion_tokens, + retries, + stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_messages_with_deployment( + deployment: str, + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "custom", + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict[str, int]]: + return _chat_messages_impl( + deployment, + messages, + max_completion_tokens, + retries, + stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_target_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict[str, int]]: + return _chat_messages_impl( + TARGET_DEPLOYMENT, + messages, + max_completion_tokens, + retries, + stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def get_token_summary() -> dict[str, dict[str, int]]: + return tracker.summary() + + +def reset_token_tracker() -> None: + tracker.reset() + + +def set_target_deployment(deployment: str) -> None: + global TARGET_DEPLOYMENT + TARGET_DEPLOYMENT = deployment + os.environ["TARGET_DEPLOYMENT"] = deployment + + +def set_reasoning_effort(effort: str | None) -> None: + global REASONING_EFFORT + REASONING_EFFORT = effort if effort else None + + +def set_optimizer_deployment(deployment: str) -> None: + global OPTIMIZER_DEPLOYMENT + OPTIMIZER_DEPLOYMENT = deployment + os.environ["OPTIMIZER_DEPLOYMENT"] = deployment diff --git a/skillopt/model/codex_harness.py b/skillopt/model/codex_harness.py new file mode 100644 index 0000000..a8513ea --- /dev/null +++ b/skillopt/model/codex_harness.py @@ -0,0 +1,1057 @@ +"""Helpers for running exec backends as the target harness.""" +from __future__ import annotations + +import asyncio +import json +import os +import re +import shutil +import subprocess +import threading +import traceback +from typing import Any + +from skillopt.model.backend_config import ( + get_claude_code_exec_config, + get_codex_exec_config, + get_target_backend, +) + + +ANSWER_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "final_response": { + "type": "string", + "description": "The exact final answer text to return, preserving required ... tags.", + }, + "final_answer": { + "type": "string", + "description": "The concise answer value without explanation, if separable.", + }, + }, + "required": ["final_response", "final_answer"], + "additionalProperties": False, +} + + +def render_skill_md( + skill_content: str, + *, + name: str = "skillopt-target", + description: str = "Dynamic ReflACT skill for the current benchmark task.", + preamble: str = "", +) -> str: + body = skill_content.strip() or "No additional dynamic guidance was provided for this task." + chunks = [ + "---", + f'name: "{name}"', + f'description: "{description}"', + "---", + "", + "# ReflACT Target Skill", + "", + ] + if preamble.strip(): + chunks.append(preamble.strip()) + chunks.append("") + chunks.extend([ + "## Dynamic Guidance", + "", + body, + "", + ]) + return "\n".join(chunks) + + +def prepare_workspace( + *, + work_dir: str, + skill_md: str, + task_text: str = "", + task_filename: str = "task.md", + images: list[str] | None = None, + extra_files: dict[str, str] | None = None, + copy_files: list[tuple[str, str]] | None = None, + link_dirs: list[tuple[str, str]] | None = None, +) -> tuple[str, str]: + if os.path.exists(work_dir): + shutil.rmtree(work_dir) + os.makedirs(os.path.join(work_dir, ".agents", "skills", "skillopt-target"), exist_ok=True) + + skill_path = os.path.join(work_dir, ".agents", "skills", "skillopt-target", "SKILL.md") + with open(skill_path, "w", encoding="utf-8") as f: + f.write(skill_md) + + task_path = os.path.join(work_dir, task_filename) + if task_text: + with open(task_path, "w", encoding="utf-8") as f: + f.write(task_text) + + if extra_files: + for rel_path, content in extra_files.items(): + full_path = os.path.join(work_dir, rel_path) + parent = os.path.dirname(full_path) + if parent: + os.makedirs(parent, exist_ok=True) + with open(full_path, "w", encoding="utf-8") as f: + f.write(content) + + if copy_files: + for src, rel_dst in copy_files: + dst = os.path.join(work_dir, rel_dst) + parent = os.path.dirname(dst) + if parent: + os.makedirs(parent, exist_ok=True) + shutil.copy2(src, dst) + + if link_dirs: + for src, rel_dst in link_dirs: + dst = os.path.join(work_dir, rel_dst) + parent = os.path.dirname(dst) + if parent: + os.makedirs(parent, exist_ok=True) + os.symlink(os.path.abspath(src), dst) + + attachment_lines: list[str] = [] + if images: + attachments_dir = os.path.join(work_dir, "attachments") + os.makedirs(attachments_dir, exist_ok=True) + for index, image in enumerate(images, 1): + if not os.path.exists(image): + raise FileNotFoundError(image) + src = os.path.abspath(image) + base = os.path.basename(src) or f"image_{index}" + dst_name = f"{index:02d}_{base}" + dst = os.path.join(attachments_dir, dst_name) + if os.path.abspath(src) != os.path.abspath(dst): + shutil.copy2(src, dst) + rel_dst = os.path.relpath(dst, work_dir) + attachment_lines.append(f"- `{rel_dst}` (source: `{src}`)") + + if attachment_lines: + with open(os.path.join(work_dir, "ATTACHMENTS.md"), "w", encoding="utf-8") as f: + f.write( + "# Attachments\n\n" + "Use these local files when the task refers to attached images or documents.\n\n" + + "\n".join(attachment_lines) + + "\n" + ) + + return skill_path, task_path + + +def _build_codex_trace_summary(raw: str, response: str) -> str: + lines = [ln.rstrip() for ln in (raw or "").splitlines()] + + def _find(prefix: str) -> str: + for ln in lines: + if ln.startswith(prefix): + return ln[len(prefix):].strip() + return "" + + sandbox = _find("sandbox: ") + reasoning = _find("reasoning effort: ") + task_read = "unknown" + skill_read = "unknown" + exec_errors: list[str] = [] + tokens_used = "" + + for idx, ln in enumerate(lines): + if ln.startswith("exec"): + cmd = lines[idx + 1] if idx + 1 < len(lines) else "" + outcome = lines[idx + 2] if idx + 2 < len(lines) else "" + joined = f"{cmd}\n{outcome}" + if "task.md" in joined: + if "succeeded" in outcome: + task_read = "success" + elif "failed" in outcome or "ERROR" in outcome: + task_read = "failed" + if "SKILL.md" in joined: + if "succeeded" in outcome: + skill_read = "success" + elif "failed" in outcome or "ERROR" in outcome: + skill_read = "failed" + if ln.startswith("ERROR:"): + exec_errors.append(ln[len("ERROR:"):].strip()) + if ln == "tokens used" and idx + 1 < len(lines): + tokens_used = lines[idx + 1].strip() + + match = re.search(r"\s*([A-E])\s*", response or "", re.IGNORECASE) + if match: + answer_format = "well_formed" + answer_label = match.group(1).upper() + elif "" in (response or "").lower(): + answer_format = "tagged_nonlabel" + answer_label = "" + elif (response or "").strip(): + answer_format = "plain_text" + answer_label = "" + else: + answer_format = "missing" + answer_label = "" + + parts = ["Codex Trace Summary"] + if sandbox: + parts.append(f"- sandbox: {sandbox}") + if reasoning: + parts.append(f"- reasoning: {reasoning}") + parts.append(f"- read task.md: {task_read}") + parts.append(f"- read SKILL.md: {skill_read}") + if exec_errors: + parts.append(f"- shell/tool errors: {' | '.join(exec_errors[:3])}") + else: + parts.append("- shell/tool errors: none") + parts.append(f"- final answer format: {answer_format}") + parts.append(f"- final answer label: {answer_label or '(none)'}") + if tokens_used: + parts.append(f"- tokens used: {tokens_used}") + return "\n".join(parts) + + +def _build_claude_trace_summary(raw: str, response: str) -> str: + answer_format = "missing" + if "" in (response or "").lower(): + answer_format = "tagged" + elif (response or "").strip(): + answer_format = "plain_text" + errors: list[str] = [] + for ln in (raw or "").splitlines(): + if "error" in ln.lower() or "traceback" in ln.lower(): + errors.append(ln.strip()) + if len(errors) >= 3: + break + parts = ["Claude Code Trace Summary", f"- final answer format: {answer_format}"] + parts.append(f"- final response chars: {len(response or '')}") + parts.append(f"- errors: {' | '.join(errors) if errors else 'none'}") + return "\n".join(parts) + + +def _persist_artifacts( + *, + work_dir: str, + raw: str, + response: str, + prefix: str, + summary_builder, +) -> None: + pred_dir = os.path.dirname(work_dir.rstrip(os.sep)) + raw_path = os.path.join(pred_dir, f"{prefix}_raw.txt") + summary_path = os.path.join(pred_dir, f"{prefix}_trace_summary.txt") + + combined_raw = raw + if os.path.exists(raw_path): + with open(raw_path, encoding="utf-8") as f: + prev = f.read() + combined_raw = f"{prev}\n\n===== TURN BREAK =====\n\n{raw}" if prev.strip() else raw + + with open(raw_path, "w", encoding="utf-8") as f: + f.write(combined_raw) + with open(summary_path, "w", encoding="utf-8") as f: + f.write(summary_builder(combined_raw, response)) + + +def _persist_codex_artifacts(work_dir: str, raw: str, response: str) -> None: + _persist_artifacts( + work_dir=work_dir, + raw=raw, + response=response, + prefix="codex", + summary_builder=_build_codex_trace_summary, + ) + + +def _persist_claude_artifacts(work_dir: str, raw: str, response: str) -> None: + _persist_artifacts( + work_dir=work_dir, + raw=raw, + response=response, + prefix="claude", + summary_builder=_build_claude_trace_summary, + ) + + +def parse_codex_raw(raw: str) -> dict: + """Parse raw Codex CLI output into step sections. + + Returns a dict with: + - ``steps``: ordered sections beginning at the first ``user/codex/exec`` marker + - ``trace_body``: raw trace starting at the first marker + """ + lines = (raw or "").splitlines() + markers = {"user", "codex", "exec"} + first_step_line: int | None = None + for idx, line in enumerate(lines): + if line in markers: + first_step_line = idx + break + if first_step_line is None: + return {"steps": [], "trace_body": ""} + + steps: list[dict] = [] + current: dict | None = None + for idx in range(first_step_line, len(lines)): + line = lines[idx] + if line in markers: + if current is not None: + current["end_line"] = idx + current["content"] = "\n".join(current["content_lines"]).strip() + current.pop("content_lines", None) + steps.append(current) + current = { + "index": len(steps) + 1, + "type": line, + "start_line": idx, + "content_lines": [], + } + continue + if current is not None: + current["content_lines"].append(line) + if current is not None: + current["end_line"] = len(lines) + current["content"] = "\n".join(current["content_lines"]).strip() + current.pop("content_lines", None) + steps.append(current) + + trace_body = "\n".join(lines[first_step_line:]).strip() + return {"steps": steps, "trace_body": trace_body} + + +def format_codex_trace_steps(raw: str, *, max_chars: int = 4000) -> str: + """Render parsed Codex trace into numbered compact steps for optimizer prompts.""" + parsed = parse_codex_raw(raw) + steps = parsed["steps"] + if not steps: + return "" + + rendered: list[str] = [] + for step in steps: + summary = "" + content = str(step.get("content") or "").strip() + if step["type"] == "exec": + body_lines = [ln.strip() for ln in content.splitlines() if ln.strip()] + cmd = body_lines[0] if body_lines else "" + status = "" + for ln in body_lines[1:]: + low = ln.lower() + if "succeeded in" in low or "failed in" in low or "timed out" in low or low.startswith("error"): + status = ln + break + summary = cmd + if status: + summary = f"{summary} | {status}" if summary else status + else: + summary = " ".join(content.splitlines()) + summary = summary[:500] if summary else "(empty)" + rendered.append(f"[{step['index']}] {step['type']}: {summary}") + + text = "\n".join(rendered) + if len(text) > max_chars: + text = text[:max_chars] + "\n...[trace steps truncated]..." + return text + + +def extract_codex_trace_prefix(raw: str, *, after_step: int) -> str: + """Return raw trace body up to and including ``after_step``. + + ``after_step <= 0`` yields an empty string. + """ + if after_step <= 0: + return "" + parsed = parse_codex_raw(raw) + steps = parsed["steps"] + if not steps: + return "" + clamped = min(after_step, len(steps)) + lines = parsed["trace_body"].splitlines() + end_line = int(steps[clamped - 1]["end_line"]) - int(steps[0]["start_line"]) + return "\n".join(lines[:end_line]).strip() + + +_DENIED_DATA_DIR_NAMES = {"officeqa_split", "sealqa_split"} + + +def _normalize_tools(allowed_tools: list[str] | str | None) -> str: + if allowed_tools is None: + return "" + if isinstance(allowed_tools, str): + return ",".join(part.strip() for part in allowed_tools.split(",") if part.strip()) + return ",".join(str(tool).strip() for tool in allowed_tools if str(tool).strip()) + + +def _tools_list(allowed_tools: list[str] | str | None) -> list[str]: + tools = _normalize_tools(allowed_tools) + return [part.strip() for part in tools.split(",") if part.strip()] + + +def _validate_exec_path(path: str) -> str: + resolved = os.path.realpath(os.path.abspath(path)) + parts = set(resolved.split(os.sep)) + denied = parts & _DENIED_DATA_DIR_NAMES + if denied: + raise ValueError(f"Refusing to expose denied data directory to exec backend: {', '.join(sorted(denied))}") + return resolved + + +def _validated_add_dirs(work_dir: str, data_dirs: list[str] | None, images: list[str] | None) -> list[str]: + add_dirs = [_validate_exec_path(work_dir)] + for data_dir in data_dirs or []: + add_dirs.append(_validate_exec_path(data_dir)) + for image in images or []: + add_dirs.append(_validate_exec_path(os.path.dirname(image) or work_dir)) + deduped: list[str] = [] + for path in add_dirs: + if path not in deduped: + deduped.append(path) + return deduped + + +def _sdk_mode(value: Any) -> str: + mode = str(value or "auto").strip().lower() + if mode in {"1", "true", "yes", "on", "sdk"}: + return "sdk" + if mode in {"0", "false", "no", "off", "cli"}: + return "cli" + return "auto" + + +def _claude_effort(value: Any) -> str: + effort = str(value or "medium").strip().lower() + if effort in {"", "none", "off"}: + return "" + if effort == "xhigh": + return "max" + if effort not in {"low", "medium", "high", "max"}: + return "medium" + return effort + + +def _json_default(obj: Any) -> Any: + if isinstance(obj, (str, int, float, bool)) or obj is None: + return obj + if isinstance(obj, (list, tuple)): + return list(obj) + if isinstance(obj, dict): + return obj + if hasattr(obj, "model_dump"): + return obj.model_dump(mode="json") + if hasattr(obj, "__dict__"): + return {k: v for k, v in vars(obj).items() if not k.startswith("_")} + return str(obj) + + +def _json_dumps(data: Any) -> str: + return json.dumps(data, ensure_ascii=False, indent=2, default=_json_default) + + +def _run_async(coro): + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + + box: dict[str, Any] = {} + + def _target() -> None: + try: + box["result"] = asyncio.run(coro) + except BaseException as exc: # noqa: BLE001 + box["exception"] = exc + + thread = threading.Thread(target=_target, daemon=True) + thread.start() + thread.join() + if "exception" in box: + raise box["exception"] + return box.get("result") + + +def _exec_prompt(prompt: str, *, allow_file_edits: bool = False) -> str: + edit_instruction = ( + "You may modify files in the workspace when the task asks you to create an artifact. " + if allow_file_edits + else "Do not modify files. " + ) + return ( + "Use the workspace files to solve the task. Read task.md and the skill at " + ".agents/skills/skillopt-target/SKILL.md before answering. " + "If ATTACHMENTS.md exists, read it and inspect the listed local files. " + "Do not call a Skill tool; the ReflACT guidance is a local markdown file. " + f"Do not ask for permission. {edit_instruction}" + "Return only the final answer text, keeping any required ... tags exactly.\n\n" + f"{_normalize_target_exec_prompt(prompt)}" + ) + + +def _retry_prompt(prompt: str, attempt: int) -> str: + if attempt <= 0: + return prompt + return ( + f"{prompt}\n\n" + "Previous execution returned an empty final response. Re-read task.md and " + ".agents/skills/skillopt-target/SKILL.md. If ATTACHMENTS.md exists, use the listed files. " + "Then produce the final answer inside ...." + ) + + +def _normalize_target_exec_prompt(prompt: str) -> str: + """Avoid wording that makes Claude Code call an unregistered Skill tool.""" + text = prompt or "" + replacements = { + "Use the `skillopt-target` skill available in this workspace.": ( + "Read `.agents/skills/skillopt-target/SKILL.md` directly; do not call a Skill tool." + ), + "- Use the local `skillopt-target` skill before writing code.": ( + "- Read `.agents/skills/skillopt-target/SKILL.md` before writing code; do not call a Skill tool." + ), + } + for old, new in replacements.items(): + text = text.replace(old, new) + return text + + +def _strict_schema(schema: dict[str, Any]) -> dict[str, Any]: + strict = json.loads(json.dumps(schema)) + strict["additionalProperties"] = False + properties = strict.get("properties") or {} + strict["required"] = list(properties.keys()) + return strict + + +def _structured_response(data: Any) -> tuple[str, str]: + if not isinstance(data, dict): + return "", f"Structured output was not an object: {type(data).__name__}" + final_response = str(data.get("final_response") or "").strip() + final_answer = str(data.get("final_answer") or "").strip() + if final_response: + return final_response, "" + if final_answer: + if "" in final_answer.lower(): + return final_answer, "" + return f"{final_answer}", "" + return "", "Structured output did not contain a final response." + + +def _extract_claude_structured_output(messages: list[Any]) -> Any: + """Claude Code SDK can finish with error_during_execution after StructuredOutput.""" + for msg in reversed(messages): + structured = getattr(msg, "structured_output", None) + if isinstance(structured, dict): + return structured + + content = getattr(msg, "content", None) + if content is None and isinstance(msg, dict): + content = msg.get("content") + if not isinstance(content, list): + continue + + for item in reversed(content): + name = getattr(item, "name", None) + payload = getattr(item, "input", None) + if isinstance(item, dict): + name = item.get("name", name) + payload = item.get("input", payload) + if name == "StructuredOutput" and isinstance(payload, dict): + return payload + return None + + +def _raw_exception(label: str, exc: BaseException) -> str: + return _json_dumps({ + "backend": label, + "is_error": True, + "error_type": type(exc).__name__, + "error": str(exc), + "traceback": traceback.format_exc(), + }) + + +def _run_claude_code_sdk_exec( + *, + work_dir: str, + prompt: str, + model: str, + timeout: int, + images: list[str] | None = None, + data_dirs: list[str] | None = None, + allowed_tools: list[str] | str | None = None, + permission_mode: str | None = None, + allow_file_edits: bool = False, +) -> tuple[str, str]: + from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient + + async def _query() -> tuple[str, str]: + system_prompt: dict[str, Any] = { + "type": "preset", + "preset": "claude_code", + "append": ( + "Use the workspace files to solve the task. Read task.md and the skill at " + ".agents/skills/skillopt-target/SKILL.md before answering. " + "If ATTACHMENTS.md exists, read it and inspect the listed local files. " + "Do not call a Skill tool; the ReflACT guidance is a local markdown file. " + + ( + "You may modify files in the workspace when the task asks you to create an artifact. " + if allow_file_edits + else "Do not modify files. " + ) + + "Return structured output whose final_response preserves required ... tags." + ), + } + kwargs: dict[str, Any] = { + "system_prompt": system_prompt, + "output_format": {"type": "json_schema", "schema": ANSWER_SCHEMA}, + "allowed_tools": _tools_list(allowed_tools) or ["Read", "Bash"], + "cwd": str(work_dir), + "permission_mode": permission_mode or "bypassPermissions", + "add_dirs": _validated_add_dirs(work_dir, data_dirs, images), + "max_buffer_size": 8 * 1024 * 1024, + } + config = get_claude_code_exec_config() + effort = _claude_effort(config.get("effort")) + if effort: + kwargs["effort"] = effort + max_thinking_tokens = int(config.get("max_thinking_tokens", 0) or 0) + if max_thinking_tokens > 0: + kwargs["max_thinking_tokens"] = max_thinking_tokens + options = ClaudeAgentOptions(**kwargs) + if model: + options.model = model.split("/", 1)[1] if model.startswith("anthropic/") else model + + messages = [] + async with ClaudeSDKClient(options) as client: + await client.query(_normalize_target_exec_prompt(prompt)) + messages = [msg async for msg in client.receive_response()] + last = messages[-1] if messages else None + raw_structured_output = _extract_claude_structured_output(messages) + response, parse_error = _structured_response(raw_structured_output) + first = messages[0] if messages else None + first_data = getattr(first, "data", {}) if first is not None else {} + terminal_is_error = bool(getattr(last, "is_error", False)) if last is not None else False + raw = _json_dumps({ + "backend": "claude_code_sdk", + "uuid": first_data.get("uuid", "") if isinstance(first_data, dict) else "", + "session_id": getattr(last, "session_id", "") if last is not None else "", + "model": first_data.get("model", model) if isinstance(first_data, dict) else model, + "tools": first_data.get("tools", _tools_list(allowed_tools)) if isinstance(first_data, dict) else _tools_list(allowed_tools), + "duration_ms": getattr(last, "duration_ms", 0) if last is not None else 0, + "total_cost_usd": getattr(last, "total_cost_usd", 0.0) if last is not None else 0.0, + "num_turns": getattr(last, "num_turns", 0) if last is not None else 0, + "usage": getattr(last, "usage", {}) if last is not None else {}, + "result": getattr(last, "result", "") if last is not None else "", + "is_error": bool(parse_error) or (terminal_is_error and not response.strip()), + "terminal_is_error": terminal_is_error, + "parse_error": parse_error, + "raw_structured_output": raw_structured_output, + "messages": messages, + }) + return response, raw + + return _run_async(asyncio.wait_for(_query(), timeout=timeout)) + + +def _run_claude_code_cli_exec( + *, + work_dir: str, + prompt: str, + model: str, + timeout: int, + images: list[str] | None = None, + data_dirs: list[str] | None = None, + allowed_tools: list[str] | str | None = None, + permission_mode: str | None = None, + allow_file_edits: bool = False, +) -> tuple[str, str]: + config = get_claude_code_exec_config() + tools = "Read,Bash" if allowed_tools is None else _normalize_tools(allowed_tools) + cmd = [ + str(config["path"]), + "-p", + "--output-format", + "text", + "--permission-mode", + permission_mode or "bypassPermissions", + "--add-dir", + work_dir, + "--tools", + tools, + "--allowedTools", + tools, + ] + if config.get("profile"): + cmd.extend(["--settings", '{"env":{"CLAUDE_CODE_USE_BEDROCK":"0"}}']) + cmd.extend(["--append-system-prompt", f"Profile: {config['profile']}"]) + if model: + cmd.extend(["--model", model]) + effort = _claude_effort(config.get("effort")) + if effort: + cmd.extend(["--effort", effort]) + max_thinking_tokens = int(config.get("max_thinking_tokens", 0) or 0) + if max_thinking_tokens > 0: + cmd.extend(["--max-thinking-tokens", str(max_thinking_tokens)]) + for data_dir in data_dirs or []: + cmd.extend(["--add-dir", _validate_exec_path(data_dir)]) + if images: + for image in images: + cmd.extend(["--add-dir", _validate_exec_path(os.path.dirname(image) or work_dir)]) + cmd.extend(["--", _exec_prompt(prompt, allow_file_edits=allow_file_edits)]) + + try: + proc = subprocess.run( + cmd, + cwd=work_dir, + capture_output=True, + text=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired as exc: + stdout = exc.stdout or "" + stderr = exc.stderr or "" + raw = stdout + if stderr: + raw = f"{raw}\n[stderr]\n{stderr}" if raw else stderr + return "", raw + + stdout = proc.stdout or "" + stderr = proc.stderr or "" + raw = stdout + if stderr: + raw = f"{raw}\n[stderr]\n{stderr}" if raw else stderr + response = stdout.strip() + if proc.returncode != 0 and not response: + return "", raw + return response, raw + + +def run_claude_code_exec( + *, + work_dir: str, + prompt: str, + model: str, + timeout: int, + images: list[str] | None = None, + data_dirs: list[str] | None = None, + allowed_tools: list[str] | str | None = None, + permission_mode: str | None = None, + allow_file_edits: bool = False, +) -> tuple[str, str]: + config = get_claude_code_exec_config() + mode = _sdk_mode(config.get("use_sdk")) + retries = int(config.get("empty_response_retries", 0) or 0) + last_response = "" + all_raw: list[str] = [] + + for attempt in range(retries + 1): + attempt_prompt = _retry_prompt(prompt, attempt) + if mode != "cli": + try: + response, raw = _run_claude_code_sdk_exec( + work_dir=work_dir, + prompt=attempt_prompt, + model=model, + timeout=timeout, + images=images, + data_dirs=data_dirs, + allowed_tools=allowed_tools, + permission_mode=permission_mode, + allow_file_edits=allow_file_edits, + ) + all_raw.append(f"===== CLAUDE SDK ATTEMPT {attempt + 1} =====\n{raw}") + if response.strip(): + combined = "\n\n".join(all_raw) + _persist_claude_artifacts(work_dir, combined, response) + return response, combined + except (ImportError, ModuleNotFoundError) as exc: + raw = _raw_exception("claude_code_sdk", exc) + all_raw.append(f"===== CLAUDE SDK ATTEMPT {attempt + 1} =====\n{raw}") + if mode == "sdk": + _persist_claude_artifacts(work_dir, "\n\n".join(all_raw), "") + raise + except Exception as exc: # noqa: BLE001 + raw = _raw_exception("claude_code_sdk", exc) + all_raw.append(f"===== CLAUDE SDK ATTEMPT {attempt + 1} =====\n{raw}") + if mode == "sdk" and attempt >= retries: + _persist_claude_artifacts(work_dir, "\n\n".join(all_raw), "") + raise + if mode != "sdk": + response, raw = _run_claude_code_cli_exec( + work_dir=work_dir, + prompt=attempt_prompt, + model=model, + timeout=timeout, + images=images, + data_dirs=data_dirs, + allowed_tools=allowed_tools, + permission_mode=permission_mode, + allow_file_edits=allow_file_edits, + ) + all_raw.append(f"===== CLAUDE CLI ATTEMPT {attempt + 1} =====\n{raw}") + last_response = response + if response.strip(): + combined = "\n\n".join(all_raw) + _persist_claude_artifacts(work_dir, combined, response) + return response, combined + + combined = "\n\n".join(all_raw) + _persist_claude_artifacts(work_dir, combined, last_response) + return last_response, combined + + +def _run_codex_sdk_exec( + *, + work_dir: str, + prompt: str, + model: str, + timeout: int, + images: list[str] | None = None, + data_dirs: list[str] | None = None, +) -> tuple[str, str]: + from openai_codex_sdk import Codex + + for data_dir in data_dirs or []: + _validate_exec_path(data_dir) + for image in images or []: + _validate_exec_path(os.path.dirname(image) or work_dir) + + async def _query() -> tuple[str, str]: + config = get_codex_exec_config() + reasoning_effort = str(config.get("reasoning_effort", "") or "").strip() + thread_options: dict[str, Any] = { + "working_directory": work_dir, + "skip_git_repo_check": True, + "sandbox_mode": str(config.get("sandbox") or "workspace-write"), + "network_access_enabled": bool(config.get("network_access", False)), + "web_search_enabled": bool(config.get("web_search", False)), + "approval_policy": str(config.get("approval_policy") or "never"), + } + if model: + thread_options["model"] = model + if data_dirs: + thread_options["additional_directories"] = data_dirs + if reasoning_effort and reasoning_effort != "none": + thread_options["model_reasoning_effort"] = reasoning_effort + + codex_options: dict[str, Any] = {"env": os.environ.copy()} + codex_path = str(config.get("path") or "").strip() + if codex_path: + codex_options["codexPathOverride"] = codex_path + codex = Codex(codex_options) + thread = codex.start_thread(thread_options) + turn = await thread.run(prompt, {"output_schema": _strict_schema(ANSWER_SCHEMA)}) + result_text = str(getattr(turn, "final_response", "") or "") + parsed: Any = None + parse_error = "" + response = "" + if result_text.strip(): + try: + parsed = json.loads(result_text) + response, parse_error = _structured_response(parsed) + except Exception as exc: # noqa: BLE001 + parse_error = f"{type(exc).__name__}: {exc}" + else: + parse_error = "No response from Codex SDK (final_response is empty)." + raw = _json_dumps({ + "backend": "codex_sdk", + "id": getattr(turn, "id", ""), + "thread_id": getattr(turn, "thread_id", ""), + "model": model, + "thread_options": thread_options, + "final_response": result_text, + "raw_structured_output": parsed, + "parse_error": parse_error, + "is_error": bool(parse_error), + "items": getattr(turn, "items", []), + }) + return response, raw + + return _run_async(asyncio.wait_for(_query(), timeout=timeout)) + + +def _run_codex_cli_exec( + *, + work_dir: str, + prompt: str, + model: str, + timeout: int, + images: list[str] | None = None, + data_dirs: list[str] | None = None, + sandbox: str | None = None, + full_auto: bool | None = None, +) -> tuple[str, str]: + config = get_codex_exec_config() + last_message_path = os.path.join(work_dir, "codex_last_message.txt") + cmd = [ + str(config["path"]), + "exec", + "--skip-git-repo-check", + "--color", + "never", + "-C", + work_dir, + ] + if config.get("profile"): + cmd.extend(["-p", str(config["profile"])]) + reasoning_effort = str(config.get("reasoning_effort", "")).strip() + if reasoning_effort: + cmd.extend(["-c", f'model_reasoning_effort="{reasoning_effort}"']) + actual_full_auto = bool(config.get("full_auto", True)) if full_auto is None else bool(full_auto) + actual_sandbox = str(sandbox or config["sandbox"]) + if actual_full_auto: + cmd.append("--full-auto") + else: + cmd.extend(["--sandbox", actual_sandbox]) + if model: + cmd.extend(["-m", model]) + for data_dir in data_dirs or []: + _validate_exec_path(data_dir) + for image in images or []: + _validate_exec_path(os.path.dirname(image) or work_dir) + cmd.extend(["-i", image]) + cmd.extend(["--output-last-message", last_message_path, prompt]) + + try: + proc = subprocess.run( + cmd, + cwd=work_dir, + capture_output=True, + text=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired as exc: + stdout = exc.stdout or "" + stderr = exc.stderr or "" + raw = stdout + if stderr: + raw = f"{raw}\n[stderr]\n{stderr}" if raw else stderr + _persist_codex_artifacts(work_dir, raw, "") + raise + try: + from skillopt.model import azure_openai as _openai + _openai.tracker.record("rollout", 0, 0) + except Exception: + pass + stdout = proc.stdout or "" + stderr = proc.stderr or "" + last_message = "" + if os.path.exists(last_message_path): + with open(last_message_path, encoding="utf-8") as f: + last_message = f.read() + raw = stdout + if stderr: + raw = f"{raw}\n[stderr]\n{stderr}" if raw else stderr + if proc.returncode != 0: + _persist_codex_artifacts(work_dir, raw, last_message) + detail = (stderr or stdout).strip() + raise RuntimeError( + f"codex exec failed with exit code {proc.returncode}: {detail[:4000]}" + ) + return last_message, raw + + +def run_codex_exec( + *, + work_dir: str, + prompt: str, + model: str, + timeout: int, + images: list[str] | None = None, + data_dirs: list[str] | None = None, + sandbox: str | None = None, + full_auto: bool | None = None, +) -> tuple[str, str]: + config = get_codex_exec_config() + mode = _sdk_mode(config.get("use_sdk")) + retries = int(config.get("empty_response_retries", 0) or 0) + last_response = "" + all_raw: list[str] = [] + + for attempt in range(retries + 1): + attempt_prompt = _retry_prompt(prompt, attempt) + if mode != "cli": + try: + response, raw = _run_codex_sdk_exec( + work_dir=work_dir, + prompt=attempt_prompt, + model=model, + timeout=timeout, + images=images, + data_dirs=data_dirs, + ) + all_raw.append(f"===== CODEX SDK ATTEMPT {attempt + 1} =====\n{raw}") + if response.strip(): + combined = "\n\n".join(all_raw) + _persist_codex_artifacts(work_dir, combined, response) + return response, combined + except (ImportError, ModuleNotFoundError) as exc: + raw = _raw_exception("codex_sdk", exc) + all_raw.append(f"===== CODEX SDK ATTEMPT {attempt + 1} =====\n{raw}") + if mode == "sdk": + _persist_codex_artifacts(work_dir, "\n\n".join(all_raw), "") + raise + except Exception as exc: # noqa: BLE001 + raw = _raw_exception("codex_sdk", exc) + all_raw.append(f"===== CODEX SDK ATTEMPT {attempt + 1} =====\n{raw}") + if mode == "sdk" and attempt >= retries: + _persist_codex_artifacts(work_dir, "\n\n".join(all_raw), "") + raise + if mode != "sdk": + response, raw = _run_codex_cli_exec( + work_dir=work_dir, + prompt=attempt_prompt, + model=model, + timeout=timeout, + images=images, + data_dirs=data_dirs, + sandbox=sandbox, + full_auto=full_auto, + ) + all_raw.append(f"===== CODEX CLI ATTEMPT {attempt + 1} =====\n{raw}") + last_response = response + if response.strip(): + combined = "\n\n".join(all_raw) + _persist_codex_artifacts(work_dir, combined, response) + return response, combined + + combined = "\n\n".join(all_raw) + _persist_codex_artifacts(work_dir, combined, last_response) + return last_response, combined + + +def run_target_exec( + *, + work_dir: str, + prompt: str, + model: str, + timeout: int, + images: list[str] | None = None, + data_dirs: list[str] | None = None, + allowed_tools: list[str] | str | None = None, + permission_mode: str | None = None, + sandbox: str | None = None, + full_auto: bool | None = None, + allow_file_edits: bool = False, +) -> tuple[str, str]: + backend = get_target_backend() + if backend == "codex_exec": + return run_codex_exec( + work_dir=work_dir, + prompt=prompt, + model=model, + timeout=timeout, + images=images, + data_dirs=data_dirs, + sandbox=sandbox, + full_auto=full_auto, + ) + if backend == "claude_code_exec": + return run_claude_code_exec( + work_dir=work_dir, + prompt=prompt, + model=model, + timeout=timeout, + images=images, + data_dirs=data_dirs, + allowed_tools=allowed_tools, + permission_mode=permission_mode, + allow_file_edits=allow_file_edits, + ) + raise ValueError(f"Unsupported exec backend: {backend}") diff --git a/skillopt/model/common.py b/skillopt/model/common.py new file mode 100644 index 0000000..80983b5 --- /dev/null +++ b/skillopt/model/common.py @@ -0,0 +1,229 @@ +"""Shared model utilities for ReflACT backends.""" +from __future__ import annotations + +import json +import threading +from dataclasses import dataclass, field +from typing import Any + + +_RESPONSES_API_MODELS = { + "gpt-5.3-codex", + "gpt-5.1-codex", + "gpt-5.2-codex", + "gpt-5-codex", + "codex-mini", + "gpt-5.4-pro", +} + +_BACKEND_DEFAULT_MODELS = { + "azure_openai": "gpt-4o", + "openai_chat": "gpt-4o", + "codex": "gpt-4o", + "codex_exec": "gpt-4o", + "claude": "claude-sonnet-4-6", + "claude_chat": "claude-sonnet-4-6", + "claude_code_exec": "claude-sonnet-4-6", + "qwen_chat": "Qwen/Qwen3.5-4B", + "minimax_chat": "MiniMax-M2.7", +} + +_BACKEND_ALIASES = { + "azure": "azure_openai", + "azure_openai": "azure_openai", + "azure-openai": "azure_openai", + "openai_chat": "openai_chat", + "openai": "codex", + "codex": "codex", + "codex_exec": "codex_exec", + "claude": "claude_chat", + "claude_chat": "claude_chat", + "claude_code_exec": "claude_code_exec", + "anthropic": "claude_chat", + "qwen": "qwen_chat", + "qwen_chat": "qwen_chat", + "minimax": "minimax_chat", + "minimax_chat": "minimax_chat", +} + + +def normalize_backend_name(name: str | None) -> str: + normalized = str(name or "").strip().lower() + return _BACKEND_ALIASES.get(normalized, normalized or "azure_openai") + + +def default_model_for_backend(backend: str | None) -> str: + return _BACKEND_DEFAULT_MODELS.get( + normalize_backend_name(backend), + _BACKEND_DEFAULT_MODELS["azure_openai"], + ) + + +def needs_responses_api(model: str) -> bool: + normalized = str(model or "").strip().lower() + return any( + normalized == prefix or normalized.startswith(prefix + "-") + for prefix in _RESPONSES_API_MODELS + ) + + +class TokenTracker: + def __init__(self) -> None: + self._lock = threading.Lock() + self._data: dict[str, dict[str, int]] = {} + + def record(self, stage: str, prompt_tokens: int, completion_tokens: int) -> None: + with self._lock: + if stage not in self._data: + self._data[stage] = { + "calls": 0, + "prompt_tokens": 0, + "completion_tokens": 0, + } + entry = self._data[stage] + entry["calls"] += 1 + entry["prompt_tokens"] += prompt_tokens + entry["completion_tokens"] += completion_tokens + + def summary(self) -> dict[str, dict[str, int]]: + with self._lock: + out: dict[str, dict[str, int]] = {} + total_prompt = total_completion = total_calls = 0 + for stage, entry in sorted(self._data.items()): + prompt_tokens = entry["prompt_tokens"] + completion_tokens = entry["completion_tokens"] + out[stage] = { + "calls": entry["calls"], + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + } + total_prompt += prompt_tokens + total_completion += completion_tokens + total_calls += entry["calls"] + out["_total"] = { + "calls": total_calls, + "prompt_tokens": total_prompt, + "completion_tokens": total_completion, + "total_tokens": total_prompt + total_completion, + } + return out + + def reset(self) -> None: + with self._lock: + self._data.clear() + + +tracker = TokenTracker() + + +@dataclass +class CompatToolFunction: + name: str + arguments: str + + def model_dump(self, mode: str = "json") -> dict[str, str]: + del mode + return { + "name": self.name, + "arguments": self.arguments, + } + + +@dataclass +class CompatToolCall: + id: str + function: CompatToolFunction + type: str = "function" + + def model_dump(self, mode: str = "json") -> dict[str, Any]: + del mode + return { + "id": self.id, + "type": self.type, + "function": self.function.model_dump(), + } + + +@dataclass +class CompatAssistantMessage: + content: str + tool_calls: list[CompatToolCall] = field(default_factory=list) + metadata: dict[str, Any] = field(default_factory=dict) + + def model_dump(self, mode: str = "json") -> dict[str, Any]: + del mode + data: dict[str, Any] = {"role": "assistant", "content": self.content} + if self.tool_calls: + data["tool_calls"] = [tool_call.model_dump() for tool_call in self.tool_calls] + return data + + +def usage_from_openai_usage(usage: Any) -> dict[str, int]: + if not usage: + return {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + prompt_tokens = getattr(usage, "prompt_tokens", 0) or 0 + completion_tokens = getattr(usage, "completion_tokens", 0) or 0 + total_tokens = getattr(usage, "total_tokens", 0) or (prompt_tokens + completion_tokens) + return { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": total_tokens, + } + + +def usage_from_responses_usage(usage: Any) -> dict[str, int]: + if not usage: + return {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + prompt_tokens = getattr(usage, "input_tokens", 0) or 0 + completion_tokens = getattr(usage, "output_tokens", 0) or 0 + return { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + } + + +def compat_message_from_chat_message(message: Any) -> CompatAssistantMessage: + content = getattr(message, "content", "") or "" + tool_calls = [] + for tool_call in getattr(message, "tool_calls", None) or []: + function = getattr(tool_call, "function", None) + tool_calls.append( + CompatToolCall( + id=getattr(tool_call, "id", "") or "", + function=CompatToolFunction( + name=getattr(function, "name", "") or "", + arguments=getattr(function, "arguments", "") or "{}", + ), + ) + ) + return CompatAssistantMessage(content=content, tool_calls=tool_calls) + + +def compat_message_from_responses_output(output: list[Any]) -> CompatAssistantMessage: + text_parts: list[str] = [] + tool_calls: list[CompatToolCall] = [] + for item in output: + item_type = getattr(item, "type", "") or "" + if item_type == "function_call": + raw_arguments = getattr(item, "arguments", None) + if raw_arguments is None: + raw_arguments = json.dumps(getattr(item, "input", {}) or {}) + tool_calls.append( + CompatToolCall( + id=getattr(item, "call_id", "") or getattr(item, "id", "") or "", + function=CompatToolFunction( + name=getattr(item, "name", "") or "", + arguments=str(raw_arguments or "{}"), + ), + ) + ) + continue + if item_type != "message": + continue + for part in getattr(item, "content", []) or []: + part_type = getattr(part, "type", "") or "" + if part_type in {"output_text", "text"}: + text_parts.append(getattr(part, "text", "") or "") + return CompatAssistantMessage(content="".join(text_parts), tool_calls=tool_calls) diff --git a/skillopt/model/minimax_backend.py b/skillopt/model/minimax_backend.py new file mode 100644 index 0000000..7d9a42c --- /dev/null +++ b/skillopt/model/minimax_backend.py @@ -0,0 +1,303 @@ +"""OpenAI-compatible MiniMax chat backend for the target path.""" +from __future__ import annotations + +import json +import os +import threading +import time +import urllib.error +import urllib.request +from typing import Any + +from skillopt.model.common import ( + CompatAssistantMessage, + CompatToolCall, + CompatToolFunction, + TokenTracker, + default_model_for_backend, +) + +BASE_URL = os.environ.get("MINIMAX_BASE_URL", "https://api.minimax.io/v1") +API_KEY = os.environ.get("MINIMAX_API_KEY", "") +TIMEOUT_SECONDS = float(os.environ.get("MINIMAX_TIMEOUT_SECONDS", "300") or 300) +MAX_TOKENS = int(os.environ.get("MINIMAX_MAX_TOKENS", "8000") or 8000) +TEMPERATURE: float | None = None +_raw_temperature = os.environ.get("MINIMAX_TEMPERATURE", "0.7").strip() +if _raw_temperature: + TEMPERATURE = float(_raw_temperature) +ENABLE_THINKING = os.environ.get("MINIMAX_ENABLE_THINKING", "false").strip().lower() in { + "1", + "true", + "yes", + "on", +} + +TARGET_DEPLOYMENT = os.environ.get( + "TARGET_DEPLOYMENT", + default_model_for_backend("minimax_chat"), +) + +_config_lock = threading.Lock() +tracker = TokenTracker() + + +def _chat_url() -> str: + base = BASE_URL.rstrip("/") + if base.endswith("/chat/completions"): + return base + return f"{base}/chat/completions" + + +def _json_safe(value: Any) -> Any: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, list): + return [_json_safe(item) for item in value] + if isinstance(value, dict): + return {str(key): _json_safe(val) for key, val in value.items()} + model_dump = getattr(value, "model_dump", None) + if callable(model_dump): + try: + return model_dump(mode="json") + except TypeError: + return model_dump() + return str(value) + + +def _usage_from_payload(payload: dict[str, Any]) -> dict[str, int]: + usage = payload.get("usage") or {} + prompt_tokens = int(usage.get("prompt_tokens") or usage.get("input_tokens") or 0) + completion_tokens = int(usage.get("completion_tokens") or usage.get("output_tokens") or 0) + total_tokens = int(usage.get("total_tokens") or (prompt_tokens + completion_tokens)) + return { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": total_tokens, + } + + +def _compat_message_from_payload(message: dict[str, Any], choice: dict[str, Any]) -> CompatAssistantMessage: + content = message.get("content") or "" + if not isinstance(content, str): + content = json.dumps(content, ensure_ascii=False) + tool_calls: list[CompatToolCall] = [] + for index, tool_call in enumerate(message.get("tool_calls") or [], start=1): + function = tool_call.get("function") or {} + tool_calls.append( + CompatToolCall( + id=str(tool_call.get("id") or f"minimax_tool_{index}"), + type=str(tool_call.get("type") or "function"), + function=CompatToolFunction( + name=str(function.get("name") or ""), + arguments=str(function.get("arguments") or "{}"), + ), + ) + ) + return CompatAssistantMessage( + content=content, + tool_calls=tool_calls, + metadata={ + "finish_reason": choice.get("finish_reason"), + "choice0": _json_safe(choice), + }, + ) + + +def _post_chat_completion(payload: dict[str, Any], timeout: float | None) -> dict[str, Any]: + headers = {"Content-Type": "application/json"} + if API_KEY: + headers["Authorization"] = f"Bearer {API_KEY}" + req = urllib.request.Request( + _chat_url(), + data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), + headers=headers, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=timeout or TIMEOUT_SECONDS) as resp: + raw = resp.read().decode("utf-8") + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", errors="replace") + raise RuntimeError(f"MiniMax chat API returned HTTP {e.code}: {body}") from e + except urllib.error.URLError as e: + raise RuntimeError(f"MiniMax chat API request failed: {e}") from e + try: + return json.loads(raw) + except json.JSONDecodeError as e: + raise RuntimeError(f"MiniMax chat API returned non-JSON response: {raw[:1000]}") from e + + +def _chat_messages_impl( + messages: list[dict[str, Any]], + max_completion_tokens: int, + retries: int, + stage: str, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + deployment: str | None = None, + timeout: float | None = None, +) -> tuple[Any, dict[str, int]]: + payload: dict[str, Any] = { + "model": deployment or TARGET_DEPLOYMENT, + "messages": _json_safe(messages), + "max_tokens": min(max_completion_tokens, MAX_TOKENS), + } + payload["chat_template_kwargs"] = {"enable_thinking": ENABLE_THINKING} + if TEMPERATURE is not None: + payload["temperature"] = TEMPERATURE + if tools: + payload["tools"] = _json_safe(tools) + if tool_choice is not None: + payload["tool_choice"] = _json_safe(tool_choice) + + last_err: Exception | None = None + for attempt in range(retries): + try: + data = _post_chat_completion(payload, timeout) + choices = data.get("choices") or [] + if not choices: + raise RuntimeError(f"MiniMax chat API returned no choices: {data}") + choice0 = choices[0] + message = choice0.get("message") or {} + text = message.get("content") or "" + if not isinstance(text, str): + text = json.dumps(text, ensure_ascii=False) + usage_info = _usage_from_payload(data) + tracker.record(stage, usage_info["prompt_tokens"], usage_info["completion_tokens"]) + if return_message: + return _compat_message_from_payload(message, choice0), usage_info + return text, usage_info + except Exception as e: # noqa: BLE001 + last_err = e + time.sleep(min(2 ** attempt, 30)) + raise RuntimeError(f"MiniMax chat call failed after {retries} retries: {last_err}") + + +def configure_minimax_chat( + *, + base_url: str | None = None, + api_key: str | None = None, + temperature: float | str | None = None, + timeout_seconds: float | str | None = None, + max_tokens: int | str | None = None, + enable_thinking: bool | str | None = None, +) -> None: + global BASE_URL, API_KEY, TEMPERATURE, TIMEOUT_SECONDS, MAX_TOKENS, ENABLE_THINKING + with _config_lock: + if base_url is not None: + BASE_URL = str(base_url).strip() or BASE_URL + os.environ["MINIMAX_BASE_URL"] = BASE_URL + if api_key is not None: + API_KEY = str(api_key).strip() + os.environ["MINIMAX_API_KEY"] = API_KEY + if temperature is not None: + raw = str(temperature).strip() + TEMPERATURE = float(raw) if raw else None + os.environ["MINIMAX_TEMPERATURE"] = raw + if timeout_seconds is not None: + TIMEOUT_SECONDS = float(timeout_seconds) + os.environ["MINIMAX_TIMEOUT_SECONDS"] = str(timeout_seconds) + if max_tokens is not None: + MAX_TOKENS = int(max_tokens) + os.environ["MINIMAX_MAX_TOKENS"] = str(max_tokens) + if enable_thinking is not None: + if isinstance(enable_thinking, str): + ENABLE_THINKING = enable_thinking.strip().lower() in {"1", "true", "yes", "on"} + else: + ENABLE_THINKING = bool(enable_thinking) + os.environ["MINIMAX_ENABLE_THINKING"] = "true" if ENABLE_THINKING else "false" + + +def get_max_tokens() -> int: + return MAX_TOKENS + + +def chat_target( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + reasoning_effort: str | None = None, + timeout: float | None = None, +) -> tuple[str, dict[int]]: + del reasoning_effort + messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] + return _chat_messages_impl( + messages, + max_completion_tokens, + retries, + stage, + timeout=timeout, + ) + + +def chat_optimizer( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + reasoning_effort: str | None = None, + timeout: float | None = None, +) -> tuple[str, dict[int]]: + """Optimizer chat call. Backend stores the trained skill; uses the same + MiniMax-proxied OpenAI-compat endpoint as `chat_target`. Added in the + parallel-training fix; previously missing in skillopt 0.2.0's + miniamax backend, which forced the dispatcher into _openai.chat_optimizer + (Azure) and produced "[skip] no usable patches" for any user running + optimizer+target on `minimax_chat`. + """ + messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] + return _chat_messages_impl( + messages, + max_completion_tokens, + retries, + stage, + timeout=timeout, + ) + + +def chat_target_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: float | None = None, +) -> tuple[Any, dict[str, int]]: + del reasoning_effort + return _chat_messages_impl( + messages, + max_completion_tokens, + retries, + stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def get_token_summary() -> dict[str, dict[str, int]]: + return tracker.summary() + + +def reset_token_tracker() -> None: + tracker.reset() + + +def set_reasoning_effort(effort: str | None) -> None: + del effort + + +def set_target_deployment(deployment: str) -> None: + global TARGET_DEPLOYMENT + TARGET_DEPLOYMENT = deployment or default_model_for_backend("minimax_chat") + os.environ["TARGET_DEPLOYMENT"] = TARGET_DEPLOYMENT \ No newline at end of file diff --git a/skillopt/model/qwen_backend.py b/skillopt/model/qwen_backend.py new file mode 100644 index 0000000..f05bad3 --- /dev/null +++ b/skillopt/model/qwen_backend.py @@ -0,0 +1,484 @@ +"""OpenAI-compatible Qwen chat backend for optimizer and target paths.""" + +from __future__ import annotations + +import json +import os +import threading +import time +import urllib.error +import urllib.request +from dataclasses import dataclass +from typing import Any + +from skillopt.model.common import ( + CompatAssistantMessage, + CompatToolCall, + CompatToolFunction, + TokenTracker, + default_model_for_backend, +) + + +@dataclass +class QwenChatConfig: + base_url: str + api_key: str + timeout_seconds: float + max_tokens: int + temperature: float | None + enable_thinking: bool + deployment: str + use_max_completion_tokens: bool = False + + +def _parse_bool(value: Any, default: bool = False) -> bool: + if value is None: + return default + return str(value).strip().lower() in {"1", "true", "yes", "on"} + + +def _parse_optional_float(value: Any) -> float | None: + if value is None: + return None + raw = str(value).strip() + return float(raw) if raw else None + + +def _parse_int(value: Any, default: int) -> int: + if value is None: + return default + raw = str(value).strip() + return int(raw) if raw else default + + +def _role_env(role: str, key: str, default: str) -> str: + role_key = f"{role.upper()}_QWEN_CHAT_{key}" + generic_key = f"QWEN_CHAT_{key}" + return os.environ.get(role_key) or os.environ.get(generic_key) or default + + +# Sentinels that mean "omit this optional parameter from the request payload". +# Reasoning models (e.g. GPT-5.x, Claude Opus 4.8) reject an explicit +# `temperature`, so allow it to be turned off via an empty string / none / off. +_OMIT_SENTINELS = {"", "none", "off", "null"} + + +def _resolve_temperature(role: str) -> float | None: + """Return the temperature, or None to omit it entirely. + + Unlike ``_role_env`` an *explicitly set* empty (or ``none``/``off``) value is + honored as "omit" instead of collapsing to the default. Precedence: + role-specific env -> generic env -> 0.7 default. + """ + for key in (f"{role.upper()}_QWEN_CHAT_TEMPERATURE", "QWEN_CHAT_TEMPERATURE"): + if key in os.environ: + raw = os.environ[key].strip() + if raw.lower() in _OMIT_SENTINELS: + return None + return float(raw) + return 0.7 + + +def _initial_config(role: str) -> QwenChatConfig: + role_upper = role.upper() + deployment_env = "OPTIMIZER_DEPLOYMENT" if role == "optimizer" else "TARGET_DEPLOYMENT" + return QwenChatConfig( + base_url=_role_env(role, "BASE_URL", "http://localhost:8000/v1"), + api_key=_role_env(role, "API_KEY", ""), + timeout_seconds=float(_role_env(role, "TIMEOUT_SECONDS", "300") or 300), + max_tokens=_parse_int(_role_env(role, "MAX_TOKENS", "8000"), 8000), + temperature=_resolve_temperature(role), + enable_thinking=_parse_bool(_role_env(role, "ENABLE_THINKING", "false")), + use_max_completion_tokens=_parse_bool(_role_env(role, "USE_MAX_COMPLETION_TOKENS", "false")), + deployment=( + os.environ.get(f"{role_upper}_QWEN_CHAT_MODEL") + or os.environ.get("QWEN_CHAT_MODEL") + or os.environ.get(deployment_env) + or default_model_for_backend("qwen_chat") + ), + ) + + +OPTIMIZER_CONFIG = _initial_config("optimizer") +TARGET_CONFIG = _initial_config("target") + +_config_lock = threading.Lock() +tracker = TokenTracker() + + +def _chat_url(config: QwenChatConfig) -> str: + base = config.base_url.rstrip("/") + if base.endswith("/chat/completions"): + return base + return f"{base}/chat/completions" + + +def _json_safe(value: Any) -> Any: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, list): + return [_json_safe(item) for item in value] + if isinstance(value, dict): + return {str(key): _json_safe(val) for key, val in value.items()} + model_dump = getattr(value, "model_dump", None) + if callable(model_dump): + try: + return model_dump(mode="json") + except TypeError: + return model_dump() + return str(value) + + +def _usage_from_payload(payload: dict[str, Any]) -> dict[str, int]: + usage = payload.get("usage") or {} + prompt_tokens = int(usage.get("prompt_tokens") or usage.get("input_tokens") or 0) + completion_tokens = int(usage.get("completion_tokens") or usage.get("output_tokens") or 0) + total_tokens = int(usage.get("total_tokens") or (prompt_tokens + completion_tokens)) + return { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": total_tokens, + } + + +def _compat_message_from_payload(message: dict[str, Any], choice: dict[str, Any]) -> CompatAssistantMessage: + content = message.get("content") or "" + if not isinstance(content, str): + content = json.dumps(content, ensure_ascii=False) + tool_calls: list[CompatToolCall] = [] + for index, tool_call in enumerate(message.get("tool_calls") or [], start=1): + function = tool_call.get("function") or {} + tool_calls.append( + CompatToolCall( + id=str(tool_call.get("id") or f"qwen_tool_{index}"), + type=str(tool_call.get("type") or "function"), + function=CompatToolFunction( + name=str(function.get("name") or ""), + arguments=str(function.get("arguments") or "{}"), + ), + ) + ) + return CompatAssistantMessage( + content=content, + tool_calls=tool_calls, + metadata={ + "finish_reason": choice.get("finish_reason"), + "choice0": _json_safe(choice), + }, + ) + + +def _post_chat_completion( + payload: dict[str, Any], + timeout: float | None, + config: QwenChatConfig, +) -> dict[str, Any]: + headers = {"Content-Type": "application/json"} + if config.api_key: + headers["Authorization"] = f"Bearer {config.api_key}" + req = urllib.request.Request( + _chat_url(config), + data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), + headers=headers, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=timeout or config.timeout_seconds) as resp: + raw = resp.read().decode("utf-8") + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", errors="replace") + raise RuntimeError(f"Qwen chat API returned HTTP {e.code}: {body}") from e + except urllib.error.URLError as e: + raise RuntimeError(f"Qwen chat API request failed: {e}") from e + try: + return json.loads(raw) + except json.JSONDecodeError as e: + raise RuntimeError(f"Qwen chat API returned non-JSON response: {raw[:1000]}") from e + + +def _chat_messages_impl( + messages: list[dict[str, Any]], + max_completion_tokens: int, + retries: int, + stage: str, + *, + role: str, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + deployment: str | None = None, + timeout: float | None = None, +) -> tuple[Any, dict[str, int]]: + config = OPTIMIZER_CONFIG if role == "optimizer" else TARGET_CONFIG + token_limit = min(max_completion_tokens, config.max_tokens) + # Reasoning models on some gateways (GPT-5.x, o-series) require + # `max_completion_tokens` and reject the legacy `max_tokens`. + token_key = "max_completion_tokens" if config.use_max_completion_tokens else "max_tokens" + payload: dict[str, Any] = { + "model": deployment or config.deployment, + "messages": _json_safe(messages), + token_key: token_limit, + } + if config.enable_thinking: + payload["chat_template_kwargs"] = {"enable_thinking": True} + if config.temperature is not None: + payload["temperature"] = config.temperature + if tools: + payload["tools"] = _json_safe(tools) + if tool_choice is not None: + payload["tool_choice"] = _json_safe(tool_choice) + + last_err: Exception | None = None + for attempt in range(retries): + try: + data = _post_chat_completion(payload, timeout, config) + choices = data.get("choices") or [] + if not choices: + raise RuntimeError(f"Qwen chat API returned no choices: {data}") + choice0 = choices[0] + message = choice0.get("message") or {} + text = message.get("content") or "" + if not isinstance(text, str): + text = json.dumps(text, ensure_ascii=False) + usage_info = _usage_from_payload(data) + tracker.record(stage, usage_info["prompt_tokens"], usage_info["completion_tokens"]) + if return_message: + return _compat_message_from_payload(message, choice0), usage_info + return text, usage_info + except Exception as e: # noqa: BLE001 + last_err = e + time.sleep(min(2**attempt, 30)) + raise RuntimeError(f"Qwen chat call failed after {retries} retries: {last_err}") + + +def configure_qwen_chat( + *, + base_url: str | None = None, + api_key: str | None = None, + temperature: float | str | None = None, + timeout_seconds: float | str | None = None, + max_tokens: int | str | None = None, + enable_thinking: bool | str | None = None, + use_max_completion_tokens: bool | str | None = None, + optimizer_base_url: str | None = None, + optimizer_api_key: str | None = None, + optimizer_temperature: float | str | None = None, + optimizer_timeout_seconds: float | str | None = None, + optimizer_max_tokens: int | str | None = None, + optimizer_enable_thinking: bool | str | None = None, + optimizer_use_max_completion_tokens: bool | str | None = None, + target_base_url: str | None = None, + target_api_key: str | None = None, + target_temperature: float | str | None = None, + target_timeout_seconds: float | str | None = None, + target_max_tokens: int | str | None = None, + target_enable_thinking: bool | str | None = None, + target_use_max_completion_tokens: bool | str | None = None, +) -> None: + with _config_lock: + if base_url is not None: + os.environ["QWEN_CHAT_BASE_URL"] = str(base_url).strip() + if api_key is not None: + os.environ["QWEN_CHAT_API_KEY"] = str(api_key).strip() + if temperature is not None: + os.environ["QWEN_CHAT_TEMPERATURE"] = str(temperature).strip() + if timeout_seconds is not None: + os.environ["QWEN_CHAT_TIMEOUT_SECONDS"] = str(timeout_seconds) + if max_tokens is not None: + os.environ["QWEN_CHAT_MAX_TOKENS"] = str(max_tokens) + if enable_thinking is not None: + os.environ["QWEN_CHAT_ENABLE_THINKING"] = "true" if _parse_bool(enable_thinking) else "false" + if use_max_completion_tokens is not None: + os.environ["QWEN_CHAT_USE_MAX_COMPLETION_TOKENS"] = ( + "true" if _parse_bool(use_max_completion_tokens) else "false" + ) + _update_config( + OPTIMIZER_CONFIG, + "optimizer", + base_url=optimizer_base_url if optimizer_base_url is not None else base_url, + api_key=optimizer_api_key if optimizer_api_key is not None else api_key, + temperature=(optimizer_temperature if optimizer_temperature is not None else temperature), + timeout_seconds=(optimizer_timeout_seconds if optimizer_timeout_seconds is not None else timeout_seconds), + max_tokens=optimizer_max_tokens if optimizer_max_tokens is not None else max_tokens, + enable_thinking=(optimizer_enable_thinking if optimizer_enable_thinking is not None else enable_thinking), + use_max_completion_tokens=( + optimizer_use_max_completion_tokens + if optimizer_use_max_completion_tokens is not None + else use_max_completion_tokens + ), + ) + _update_config( + TARGET_CONFIG, + "target", + base_url=target_base_url if target_base_url is not None else base_url, + api_key=target_api_key if target_api_key is not None else api_key, + temperature=target_temperature if target_temperature is not None else temperature, + timeout_seconds=(target_timeout_seconds if target_timeout_seconds is not None else timeout_seconds), + max_tokens=target_max_tokens if target_max_tokens is not None else max_tokens, + enable_thinking=(target_enable_thinking if target_enable_thinking is not None else enable_thinking), + use_max_completion_tokens=( + target_use_max_completion_tokens + if target_use_max_completion_tokens is not None + else use_max_completion_tokens + ), + ) + + +def _update_config( + config: QwenChatConfig, + role: str, + *, + base_url: str | None = None, + api_key: str | None = None, + temperature: float | str | None = None, + timeout_seconds: float | str | None = None, + max_tokens: int | str | None = None, + enable_thinking: bool | str | None = None, + use_max_completion_tokens: bool | str | None = None, +) -> None: + env_prefix = role.upper() + if base_url is not None: + config.base_url = str(base_url).strip() or config.base_url + os.environ[f"{env_prefix}_QWEN_CHAT_BASE_URL"] = config.base_url + if api_key is not None: + config.api_key = str(api_key).strip() + os.environ[f"{env_prefix}_QWEN_CHAT_API_KEY"] = config.api_key + if temperature is not None: + raw = str(temperature).strip() + config.temperature = None if raw.lower() in _OMIT_SENTINELS else float(raw) + os.environ[f"{env_prefix}_QWEN_CHAT_TEMPERATURE"] = raw + if timeout_seconds is not None: + config.timeout_seconds = float(timeout_seconds) + os.environ[f"{env_prefix}_QWEN_CHAT_TIMEOUT_SECONDS"] = str(timeout_seconds) + if max_tokens is not None: + config.max_tokens = int(max_tokens) + os.environ[f"{env_prefix}_QWEN_CHAT_MAX_TOKENS"] = str(max_tokens) + if enable_thinking is not None: + config.enable_thinking = _parse_bool(enable_thinking) + os.environ[f"{env_prefix}_QWEN_CHAT_ENABLE_THINKING"] = "true" if config.enable_thinking else "false" + if use_max_completion_tokens is not None: + config.use_max_completion_tokens = _parse_bool(use_max_completion_tokens) + os.environ[f"{env_prefix}_QWEN_CHAT_USE_MAX_COMPLETION_TOKENS"] = ( + "true" if config.use_max_completion_tokens else "false" + ) + + +def get_max_tokens() -> int: + return TARGET_CONFIG.max_tokens + + +def chat_optimizer( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + reasoning_effort: str | None = None, + timeout: float | None = None, +) -> tuple[str, dict[str, int]]: + del reasoning_effort + messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] + return _chat_messages_impl( + messages, + max_completion_tokens, + retries, + stage, + role="optimizer", + timeout=timeout, + ) + + +def chat_target( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + reasoning_effort: str | None = None, + timeout: float | None = None, +) -> tuple[str, dict[str, int]]: + del reasoning_effort + messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] + return _chat_messages_impl( + messages, + max_completion_tokens, + retries, + stage, + role="target", + timeout=timeout, + ) + + +def chat_optimizer_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: float | None = None, +) -> tuple[Any, dict[str, int]]: + del reasoning_effort + return _chat_messages_impl( + messages, + max_completion_tokens, + retries, + stage, + role="optimizer", + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_target_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + reasoning_effort: str | None = None, + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: float | None = None, +) -> tuple[Any, dict[str, int]]: + del reasoning_effort + return _chat_messages_impl( + messages, + max_completion_tokens, + retries, + stage, + role="target", + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def get_token_summary() -> dict[str, dict[str, int]]: + return tracker.summary() + + +def reset_token_tracker() -> None: + tracker.reset() + + +def set_reasoning_effort(effort: str | None) -> None: + del effort + + +def set_target_deployment(deployment: str) -> None: + TARGET_CONFIG.deployment = deployment or default_model_for_backend("qwen_chat") + os.environ["TARGET_DEPLOYMENT"] = TARGET_CONFIG.deployment + + +def set_optimizer_deployment(deployment: str) -> None: + OPTIMIZER_CONFIG.deployment = deployment or default_model_for_backend("qwen_chat") + os.environ["OPTIMIZER_DEPLOYMENT"] = OPTIMIZER_CONFIG.deployment diff --git a/skillopt/model/router.py b/skillopt/model/router.py new file mode 100644 index 0000000..0863761 --- /dev/null +++ b/skillopt/model/router.py @@ -0,0 +1,236 @@ +"""Runtime backend router for ReflACT model calls.""" +from __future__ import annotations + +import os +from typing import Any + +from . import azure_openai, claude_backend, codex_backend +from .common import normalize_backend_name + + +_ACTIVE_BACKEND = normalize_backend_name( + os.environ.get("REFLACT_MODEL_BACKEND", "azure_openai") +) + + +def _backend_module(name: str): + if name == "azure_openai": + return azure_openai + if name == "codex": + return codex_backend + if name == "claude": + return claude_backend + raise ValueError(f"Unknown backend: {name!r}") + + +def _all_backend_modules() -> list[Any]: + return [azure_openai, codex_backend, claude_backend] + + +def set_backend(name: str | None) -> str: + """Select the active model backend for subsequent calls.""" + global _ACTIVE_BACKEND + normalized = normalize_backend_name(name) + if normalized not in {"azure_openai", "codex", "claude"}: + valid = ", ".join(sorted({"azure_openai", "codex", "claude"})) + raise ValueError(f"Unknown backend {name!r}. Expected one of: {valid}") + _ACTIVE_BACKEND = normalized + os.environ["REFLACT_MODEL_BACKEND"] = normalized + return _ACTIVE_BACKEND + + +def get_backend_name() -> str: + return _ACTIVE_BACKEND + + +def chat_optimizer( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + timeout: int | None = None, +) -> tuple[str, dict[str, int]]: + return _backend_module(_ACTIVE_BACKEND).chat_optimizer( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + timeout=timeout, + ) + + +def chat_target( + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + timeout: int | None = None, +) -> tuple[str, dict[str, int]]: + return _backend_module(_ACTIVE_BACKEND).chat_target( + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + timeout=timeout, + ) + + +def chat_with_deployment( + deployment: str, + system: str, + user: str, + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "custom", + timeout: int | None = None, +) -> tuple[str, dict[str, int]]: + return _backend_module(_ACTIVE_BACKEND).chat_with_deployment( + deployment=deployment, + system=system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + timeout=timeout, + ) + + +def chat_optimizer_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "optimizer", + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict[str, int]]: + return _backend_module(_ACTIVE_BACKEND).chat_optimizer_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_target_messages( + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "target", + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict[str, int]]: + return _backend_module(_ACTIVE_BACKEND).chat_target_messages( + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def chat_messages_with_deployment( + deployment: str, + messages: list[dict[str, Any]], + max_completion_tokens: int = 16384, + retries: int = 5, + stage: str = "custom", + *, + tools: list[dict[str, Any]] | None = None, + tool_choice: str | dict[str, Any] | None = None, + return_message: bool = False, + timeout: int | None = None, +) -> tuple[Any, dict[str, int]]: + return _backend_module(_ACTIVE_BACKEND).chat_messages_with_deployment( + deployment=deployment, + messages=messages, + max_completion_tokens=max_completion_tokens, + retries=retries, + stage=stage, + tools=tools, + tool_choice=tool_choice, + return_message=return_message, + timeout=timeout, + ) + + +def get_token_summary() -> dict[str, dict[str, int]]: + return _backend_module(_ACTIVE_BACKEND).get_token_summary() + + +def reset_token_tracker() -> None: + _backend_module(_ACTIVE_BACKEND).reset_token_tracker() + + +def set_reasoning_effort(effort: str | None) -> None: + for module in _all_backend_modules(): + module.set_reasoning_effort(effort) + + +def set_target_deployment(deployment: str) -> None: + for module in _all_backend_modules(): + module.set_target_deployment(deployment) + + +def set_optimizer_deployment(deployment: str) -> None: + for module in _all_backend_modules(): + module.set_optimizer_deployment(deployment) + + +def configure_azure_openai( + *, + endpoint: str | None = None, + api_version: str | None = None, + api_key: str | None = None, + auth_mode: str | None = None, + ad_scope: str | None = None, + managed_identity_client_id: str | None = None, + optimizer_endpoint: str | None = None, + optimizer_api_version: str | None = None, + optimizer_api_key: str | None = None, + optimizer_auth_mode: str | None = None, + optimizer_ad_scope: str | None = None, + optimizer_managed_identity_client_id: str | None = None, + target_endpoint: str | None = None, + target_api_version: str | None = None, + target_api_key: str | None = None, + target_auth_mode: str | None = None, + target_ad_scope: str | None = None, + target_managed_identity_client_id: str | None = None, +) -> None: + azure_openai.configure_azure_openai( + endpoint=endpoint, + api_version=api_version, + api_key=api_key, + auth_mode=auth_mode, + ad_scope=ad_scope, + managed_identity_client_id=managed_identity_client_id, + optimizer_endpoint=optimizer_endpoint, + optimizer_api_version=optimizer_api_version, + optimizer_api_key=optimizer_api_key, + optimizer_auth_mode=optimizer_auth_mode, + optimizer_ad_scope=optimizer_ad_scope, + optimizer_managed_identity_client_id=optimizer_managed_identity_client_id, + target_endpoint=target_endpoint, + target_api_version=target_api_version, + target_api_key=target_api_key, + target_auth_mode=target_auth_mode, + target_ad_scope=target_ad_scope, + target_managed_identity_client_id=target_managed_identity_client_id, + ) diff --git a/skillopt/optimizer/__init__.py b/skillopt/optimizer/__init__.py new file mode 100644 index 0000000..c9e690b --- /dev/null +++ b/skillopt/optimizer/__init__.py @@ -0,0 +1,15 @@ +"""SkillOpt Optimizer -- skill update operations. + +Analogous to the optimizer in neural network training: applies the computed +"gradient" (patches) to the current skill document to produce an updated +candidate skill. + +Modules +------- +- skill: edit application (optimizer.step() / parameter update) +- clip: edit ranking and selection (gradient clipping) +- slow_update: longitudinal comparison and guidance (EMA / regularization) +- meta_skill: cross-epoch memory for optimizer context +""" +from skillopt.optimizer.skill import apply_edit, apply_patch # noqa: F401 +from skillopt.optimizer.clip import rank_and_select # noqa: F401 diff --git a/skillopt/optimizer/appendix.py b/skillopt/optimizer/appendix.py new file mode 100644 index 0000000..2509260 --- /dev/null +++ b/skillopt/optimizer/appendix.py @@ -0,0 +1,156 @@ +"""Skill-Aware Reflection — protected appendix field (EmbodiSkill S_app). + +EmbodiSkill (paper 2605.10332v1) splits a skill into ``S = (S_body, S_app)``: +the body holds the main prescriptive rules; the appendix only *emphasizes* +existing valid rules that the executor failed to follow (EXECUTION_LAPSE), and +**never introduces new rules**. + +This module owns the appendix region of the skill document. It mirrors the +protected-field pattern of :mod:`skillopt.optimizer.slow_update`, with two +differences: + +1. **Append semantics** (not replace): execution-lapse reminders accumulate + across steps within a run, so new notes are merged into the existing + appendix rather than overwriting it. +2. **Lightweight dedup**: near-duplicate reminders are collapsed (inspired by + GMemory's ``_dedupe_preserve_order``) so the appendix stays compact. + +The appendix lives **inside** the skill markdown, between dedicated markers, so +it is persisted by the normal ``_save_skill`` path and is resume-safe. Step-level +analyst edits cannot modify it (enforced by the shared protected-region check in +:mod:`skillopt.optimizer.skill`). + +Public API +---------- +- :func:`has_appendix_field` — check if markers are present +- :func:`inject_empty_appendix_field` — add empty placeholder (skill init) +- :func:`extract_appendix_notes` — read current notes as a list +- :func:`append_to_appendix_field` — merge new notes (dedup) into the region +""" +from __future__ import annotations + +import re + +# ── Protected field markers ───────────────────────────────────────────────── + +APPENDIX_START = "" +APPENDIX_END = "" + +# Heading shown inside the rendered appendix block (human-readable only). +APPENDIX_HEADING = "## Execution Notes Appendix" + +# Each note is rendered as a markdown bullet so the target model reads it as +# ordinary guidance. +_NOTE_BULLET_PREFIX = "- " + + +# ── Dedup helpers ─────────────────────────────────────────────────────────── + + +def _canonicalize(text: str) -> str: + """Normalize a note for duplicate detection (whitespace/punct/case-insensitive).""" + normalized = re.sub(r"\s+", " ", str(text or "").strip()) + normalized = normalized.rstrip(" .;:,_-") + return normalized.casefold() + + +def _dedupe_preserve_order(notes: list[str]) -> list[str]: + """Drop blanks and near-duplicates, preserving first-seen order.""" + seen: set[str] = set() + deduped: list[str] = [] + for note in notes: + text = re.sub(r"\s+", " ", str(note).strip()) + if not text: + continue + key = _canonicalize(text) + if not key or key in seen: + continue + seen.add(key) + deduped.append(text) + return deduped + + +# ── Field manipulation ────────────────────────────────────────────────────── + + +def has_appendix_field(skill: str) -> bool: + return APPENDIX_START in skill and APPENDIX_END in skill + + +def _render_block(notes: list[str]) -> str: + """Render the full marker-delimited appendix block for *notes*.""" + lines = [APPENDIX_START, APPENDIX_HEADING] + for note in notes: + lines.append(f"{_NOTE_BULLET_PREFIX}{note}") + lines.append(APPENDIX_END) + return "\n".join(lines) + + +def inject_empty_appendix_field(skill: str) -> str: + """Add an empty appendix placeholder at the end of *skill* (idempotent). + + Mirrors ``inject_empty_slow_update_field``: called once at skill init so the + protected region exists before any note is written. + """ + if has_appendix_field(skill): + return skill + block = f"\n\n{APPENDIX_START}\n{APPENDIX_HEADING}\n{APPENDIX_END}\n" + return skill.rstrip() + block + + +def extract_appendix_notes(skill: str) -> list[str]: + """Return the current appendix notes as a list of strings (no markers/heading).""" + start = skill.find(APPENDIX_START) + end = skill.find(APPENDIX_END) + if start == -1 or end == -1: + return [] + inner = skill[start + len(APPENDIX_START):end].strip() + notes: list[str] = [] + for raw_line in inner.splitlines(): + line = raw_line.strip() + if not line: + continue + if line == APPENDIX_HEADING or line.lstrip("#").strip() == APPENDIX_HEADING.lstrip("#").strip(): + continue + if line.startswith(_NOTE_BULLET_PREFIX): + line = line[len(_NOTE_BULLET_PREFIX):].strip() + elif line.startswith("-") or line.startswith("*"): + line = line[1:].strip() + if line: + notes.append(line) + return notes + + +def _strip_all_appendix_fields(skill: str) -> str: + """Remove every appendix marker pair (and content between) from *skill*.""" + while True: + start = skill.find(APPENDIX_START) + if start == -1: + break + end = skill.find(APPENDIX_END, start) + if end == -1: + skill = skill[:start] + skill[start + len(APPENDIX_START):] + break + skill = skill[:end + len(APPENDIX_END)].rsplit(APPENDIX_START, 1)[0] + skill[end + len(APPENDIX_END):] + skill = skill.replace(APPENDIX_END, "") + while "\n\n\n" in skill: + skill = skill.replace("\n\n\n", "\n\n") + return skill.rstrip() + + +def append_to_appendix_field(skill: str, new_notes: list[str]) -> str: + """Merge *new_notes* into the appendix region (dedup), returning updated skill. + + - If no appendix region exists yet, one is created. + - Existing notes are preserved; new ones are appended after dedup against the + combined set, so order is stable and duplicates are dropped. + - Empty / whitespace-only notes are ignored. If the merged set is empty, an + empty placeholder region is still ensured. + """ + incoming = _dedupe_preserve_order(list(new_notes or [])) + existing = extract_appendix_notes(skill) + merged = _dedupe_preserve_order(existing + incoming) + + base = _strip_all_appendix_fields(skill) + block = _render_block(merged) + return f"{base}\n\n{block}\n" diff --git a/skillopt/optimizer/clip.py b/skillopt/optimizer/clip.py new file mode 100644 index 0000000..a2ed965 --- /dev/null +++ b/skillopt/optimizer/clip.py @@ -0,0 +1,109 @@ +"""ReflACT gradient clipping — LLM-driven edit ranking and selection. + +Analogous to gradient clipping in neural network training: ranks candidate +edits by importance and selects the top-L to apply, controlling the +effective step size. Previously core/select.py. +""" +from __future__ import annotations + +from skillopt.model import chat_optimizer +from skillopt.optimizer.meta_skill import format_meta_skill_context +from skillopt.optimizer.update_modes import ( + describe_item, + get_payload_items, + is_rewrite_mode, + normalize_update_mode, + payload_key, + payload_label, +) +from skillopt.prompts import load_prompt +from skillopt.utils import extract_json + + +# ── Public API ──────────────────────────────────────────────────────────────── + +def rank_and_select( + skill_content: str, + patch: dict, + max_edits: int, + meta_skill_context: str = "", + update_mode: str = "patch", +) -> dict: + """Use a optimizer LLM to rank edits by importance, then keep top-L. + + If the edit pool is within budget, returns the patch unchanged. + Otherwise, calls the optimizer to rank and select the most impactful edits. + + Parameters + ---------- + skill_content : str + Current skill document. + patch : dict + Merged :class:`~skillopt.types.Patch` dict with ``edits`` list. + max_edits : int + Maximum number of edits to keep (the "edit budget"). + + Returns + ------- + dict + :class:`~skillopt.types.Patch` dict with selected edits and + optional ``ranking_details``. + """ + update_mode = normalize_update_mode(update_mode) + edits = get_payload_items(patch, update_mode) + if len(edits) <= max_edits: + return patch + + # Build the edit pool description for the optimizer + edits_desc = [] + for i, edit in enumerate(edits): + edits_desc.append(f"[{i}] {describe_item(edit, update_mode)}") + + user = ( + f"## Current Skill\n{skill_content}\n\n" + f"## {payload_label(update_mode, title=True)} Pool ({len(edits)} {payload_label(update_mode)}, budget={max_edits})\n" + + "\n".join(edits_desc) + + f"\n\nSelect the {max_edits} most important {payload_label(update_mode)}. " + f"Return their 0-based indices in priority order." + ) + optimizer_ctx = format_meta_skill_context(meta_skill_context) + if optimizer_ctx: + user = f"{optimizer_ctx}\n\n{user}" + prompt_name = "ranking_rewrite" if is_rewrite_mode(update_mode) else "ranking" + + try: + response, _ = chat_optimizer( + system=load_prompt(prompt_name), user=user, + max_completion_tokens=16384, retries=3, stage="ranking", + ) + result = extract_json(response) + if result and "selected_indices" in result: + indices = result["selected_indices"] + selected = [] + seen: set[int] = set() + for idx in indices: + if ( + isinstance(idx, int) + and 0 <= idx < len(edits) + and idx not in seen + ): + selected.append(edits[idx]) + seen.add(idx) + if len(selected) >= max_edits: + break + if selected: + return { + "reasoning": patch.get("reasoning", "") + + f" [optimizer-ranked: selected {len(selected)}/{len(edits)} {payload_label(update_mode)}]", + payload_key(update_mode): selected, + "ranking_details": result, + } + except Exception: # noqa: BLE001 + pass + + # Fallback: simple truncation + return { + "reasoning": patch.get("reasoning", "") + + f" [fallback truncated {len(edits)}->{max_edits} {payload_label(update_mode)}]", + payload_key(update_mode): edits[:max_edits], + } diff --git a/skillopt/optimizer/lr_autonomous.py b/skillopt/optimizer/lr_autonomous.py new file mode 100644 index 0000000..ceb66e5 --- /dev/null +++ b/skillopt/optimizer/lr_autonomous.py @@ -0,0 +1,108 @@ +"""Optimizer-driven autonomous update-size decisions.""" +from __future__ import annotations + +import json +import re +from typing import Any + +from skillopt.model import chat_optimizer +from skillopt.optimizer.meta_skill import format_meta_skill_context +from skillopt.optimizer.update_modes import describe_item, get_payload_items, payload_label +from skillopt.prompts import load_prompt +from skillopt.utils import extract_json + + +def _coerce_nonnegative_int(value: Any) -> int | None: + if isinstance(value, bool): + return None + if isinstance(value, int): + return max(0, value) + if isinstance(value, float) and value.is_integer(): + return max(0, int(value)) + text = str(value or "").strip() + if not text: + return None + match = re.search(r"-?\d+", text) + if not match: + return None + return max(0, int(match.group(0))) + + +def decide_autonomous_learning_rate( + *, + skill_content: str, + merged_patch: dict, + update_mode: str, + rollout_hard: float, + rollout_soft: float, + rollout_n: int, + step_buffer_context: str = "", + meta_skill_context: str = "", +) -> dict: + """Ask the optimizer to choose the number of update items for this step. + + The prompt intentionally avoids default budgets, candidate budget lists, or + scheduler history. The only hard post-processing is validity: the returned + integer is clamped to the available item count. + """ + items = get_payload_items(merged_patch, update_mode) + available = len(items) + item_lines = [ + f"[{idx}] {describe_item(item, update_mode)}" + for idx, item in enumerate(items) + ] + user = ( + f"## Current Skill\n{skill_content}\n\n" + f"## Current Step Evidence\n" + f"rollout_n={rollout_n}\n" + f"rollout_hard={rollout_hard:.6f}\n" + f"rollout_soft={rollout_soft:.6f}\n" + f"proposed_update_items={available}\n" + f"update_item_type={payload_label(update_mode)}\n\n" + f"## Proposed Update Items\n" + + "\n".join(item_lines) + + "\n\nDecide how many proposed update items should be applied now." + ) + if step_buffer_context.strip(): + user += f"\n\n## Previous Steps in This Epoch\n{step_buffer_context}" + optimizer_ctx = format_meta_skill_context(meta_skill_context) + if optimizer_ctx: + user = f"{optimizer_ctx}\n\n{user}" + + response = "" + parsed: dict | None = None + decision: int | None = None + try: + response, _ = chat_optimizer( + system=load_prompt("lr_autonomous"), + user=user, + max_completion_tokens=16384, + retries=3, + stage="lr_autonomous", + ) + parsed = extract_json(response) + if parsed: + decision = _coerce_nonnegative_int(parsed.get("learning_rate")) + except Exception as exc: # noqa: BLE001 + parsed = {"error": str(exc)} + + fallback = False + if decision is None: + decision = 0 + fallback = True + + chosen = min(decision, available) + record = { + "learning_rate": chosen, + "raw_learning_rate": decision, + "available_update_items": available, + "clamped": chosen != decision, + "fallback": fallback, + "reasoning": (parsed or {}).get("reasoning", ""), + "confidence": (parsed or {}).get("confidence", ""), + "risk_notes": (parsed or {}).get("risk_notes", []), + "raw_response": response, + } + if parsed and "error" in parsed: + record["error"] = parsed["error"] + return record diff --git a/skillopt/optimizer/meta_skill.py b/skillopt/optimizer/meta_skill.py new file mode 100644 index 0000000..6e34ff1 --- /dev/null +++ b/skillopt/optimizer/meta_skill.py @@ -0,0 +1,79 @@ +"""Optimizer-side meta skill memory for cross-epoch optimization guidance. + +This module maintains a compact optimizer-facing memory distilled from +adjacent-epoch skill comparisons. Unlike ``slow_update``, it does not +modify the target skill document. Instead, it produces guidance meant to +improve future optimizer behavior when proposing, merging, and ranking edits. +""" +from __future__ import annotations + +import traceback + +from skillopt.model import chat_optimizer +from skillopt.optimizer.slow_update import format_comparison_text +from skillopt.prompts import load_prompt +from skillopt.utils import extract_json + + +def format_meta_skill_context(meta_skill_content: str) -> str: + """Render optimizer memory into a prompt-ready context block.""" + content = (meta_skill_content or "").strip() + if not content: + return "" + return ( + "## Optimizer Meta Skill\n" + "This is optimizer-side memory distilled from prior epoch transitions in " + "this environment. Use it to improve how you propose, merge, and rank " + "skill edits. Prefer it when the current evidence is ambiguous, but do " + "not force it if the current trajectories clearly contradict it.\n\n" + f"{content}" + ) + + +def run_meta_skill( + prev_skill: str, + curr_skill: str, + comparison_pairs: list[dict], + *, + prev_meta_skill_content: str = "", + system_prompt: str | None = None, +) -> dict | None: + """Produce updated optimizer-side meta skill from adjacent epochs.""" + actual_system = system_prompt if system_prompt is not None else load_prompt("meta_skill") + + prev_meta_section = ( + prev_meta_skill_content.strip() + if prev_meta_skill_content and prev_meta_skill_content.strip() + else "(No previous optimizer meta skill — this is the first update.)" + ) + + comparison_text = format_comparison_text(comparison_pairs) + user = ( + f"## Previous Epoch Last-Step Skill\n{prev_skill}\n\n" + f"## Current Epoch Last-Step Skill\n{curr_skill}\n\n" + f"## Previous Optimizer Meta Skill\n" + f"The following optimizer memory was available during the current epoch. " + f"Reflect on whether it improved or harmed the quality of edits.\n\n" + f"{prev_meta_section}\n\n" + f"## Longitudinal Comparison (same tasks, two last-step skills)\n" + f"{comparison_text}" + ) + + try: + response, _ = chat_optimizer( + system=actual_system, + user=user, + max_completion_tokens=16384, + retries=3, + stage="meta_skill", + ) + result = extract_json(response) + if result and result.get("meta_skill_content"): + return { + "reasoning": str(result.get("reasoning", "")).strip(), + "meta_skill_content": str(result["meta_skill_content"]).strip(), + } + except Exception: # noqa: BLE001 + traceback.print_exc() + + return None diff --git a/skillopt/optimizer/rewrite.py b/skillopt/optimizer/rewrite.py new file mode 100644 index 0000000..f8b062b --- /dev/null +++ b/skillopt/optimizer/rewrite.py @@ -0,0 +1,59 @@ +"""Optimizer-driven full skill rewrite from selected revise_suggestions.""" +from __future__ import annotations + +import json + +from skillopt.model import chat_optimizer +from skillopt.prompts import load_prompt +from skillopt.optimizer.update_modes import get_payload_items +from skillopt.utils import extract_json + + +def rewrite_skill_from_suggestions( + skill_content: str, + patch: dict, + *, + system_prompt: str | None = None, + step_buffer_context: str = "", + env: str | None = None, + reasoning_effort: str | None = "high", + max_completion_tokens: int = 64000, +) -> dict | None: + suggestions = get_payload_items(patch, "rewrite_from_suggestions") + if not suggestions: + return None + + user = ( + f"## Current Skill\n{skill_content}\n\n" + f"## Selected Revise Suggestions ({len(suggestions)} total)\n" + f"{json.dumps(suggestions, ensure_ascii=False, indent=2)}\n\n" + ) + if step_buffer_context.strip(): + user += f"## Previous Steps in This Epoch\n{step_buffer_context}\n\n" + user += ( + "Rewrite the full skill document so it integrates the selected suggestions. " + "Return the complete new skill in `new_skill`." + ) + + actual_system = system_prompt if system_prompt is not None else load_prompt( + "rewrite_skill", env=env, + ) + + try: + response, _ = chat_optimizer( + system=actual_system, + user=user, + max_completion_tokens=max_completion_tokens, + retries=3, + stage="rewrite", + reasoning_effort=reasoning_effort, + ) + result = extract_json(response) + if result and str(result.get("new_skill", "")).strip(): + result["new_skill"] = str(result["new_skill"]).rstrip() + "\n" + if "change_summary" not in result or not isinstance(result["change_summary"], list): + result["change_summary"] = [] + return result + except Exception: # noqa: BLE001 + return None + return None diff --git a/skillopt/optimizer/scheduler.py b/skillopt/optimizer/scheduler.py new file mode 100644 index 0000000..63b944e --- /dev/null +++ b/skillopt/optimizer/scheduler.py @@ -0,0 +1,127 @@ +"""Learning-rate (edit budget) schedulers for ReflACT. + +The "learning rate" in ReflACT is the maximum number of skill edits allowed +per optimization step. A scheduler controls how this budget changes over +the course of training. + +Supported modes +--------------- +- ``constant`` : Fixed budget throughout training. +- ``linear`` : Linear decay from ``max_lr`` to ``min_lr``. +- ``cosine`` : Cosine annealing from ``max_lr`` to ``min_lr``. +- ``autonomous`` : No limit — the model decides how many edits to make. + +Usage:: + + scheduler = build_scheduler(cfg) + for step in range(1, total_steps + 1): + lr = scheduler.step() # returns edit budget for this step + # ... use lr as max_edits ... +""" +from __future__ import annotations + +import math +from abc import ABC, abstractmethod + + +class LRScheduler(ABC): + """Base class for edit-budget schedulers.""" + + def __init__(self, max_lr: int, min_lr: int, total_steps: int) -> None: + self.max_lr = max_lr + self.min_lr = min_lr + self.total_steps = total_steps + self._current_step = 0 + + @abstractmethod + def _compute_lr(self, step: int) -> int: + """Return the edit budget for the given 1-indexed step.""" + + def step(self) -> int: + """Advance one step and return the edit budget.""" + self._current_step += 1 + return self._compute_lr(self._current_step) + + def get_lr(self, step: int) -> int: + """Return the edit budget for an arbitrary step (1-indexed).""" + return self._compute_lr(step) + + def state_dict(self) -> dict: + return {"current_step": self._current_step} + + def load_state_dict(self, state: dict) -> None: + self._current_step = state.get("current_step", 0) + + +class ConstantScheduler(LRScheduler): + """Fixed edit budget throughout training.""" + + def _compute_lr(self, step: int) -> int: + return self.max_lr + + +class LinearScheduler(LRScheduler): + """Linear decay from ``max_lr`` to ``min_lr`` over ``total_steps``.""" + + def _compute_lr(self, step: int) -> int: + if self.total_steps <= 1: + return self.max_lr + t = min(step, self.total_steps) / self.total_steps + lr = self.max_lr + (self.min_lr - self.max_lr) * t + return max(self.min_lr, round(lr)) + + +class CosineScheduler(LRScheduler): + """Cosine annealing from ``max_lr`` to ``min_lr`` over ``total_steps``.""" + + def _compute_lr(self, step: int) -> int: + if self.total_steps <= 1: + return self.max_lr + t = min(step, self.total_steps) / self.total_steps + lr = self.min_lr + 0.5 * (self.max_lr - self.min_lr) * (1 + math.cos(math.pi * t)) + return max(self.min_lr, round(lr)) + + +class AutonomousScheduler(LRScheduler): + """No edit limit — the model decides freely.""" + + NO_LIMIT = 999 + + def _compute_lr(self, step: int) -> int: + return self.NO_LIMIT + + +# ── Factory ────────────────────────────────────────────────────────────── + +_REGISTRY: dict[str, type[LRScheduler]] = { + "constant": ConstantScheduler, + "linear": LinearScheduler, + "cosine": CosineScheduler, + "autonomous": AutonomousScheduler, +} + + +def build_scheduler( + mode: str = "constant", + max_lr: int = 8, + min_lr: int = 2, + total_steps: int = 8, +) -> LRScheduler: + """Build a scheduler from config parameters. + + Parameters + ---------- + mode : str + One of ``constant``, ``linear``, ``cosine``, ``autonomous``. + max_lr : int + Initial / maximum edit budget. + min_lr : int + Minimum edit budget (for decay modes). + total_steps : int + Total number of optimization steps in training. + """ + if mode not in _REGISTRY: + raise ValueError( + f"Unknown scheduler mode '{mode}'. Available: {list(_REGISTRY.keys())}" + ) + return _REGISTRY[mode](max_lr=max_lr, min_lr=min_lr, total_steps=total_steps) diff --git a/skillopt/optimizer/select.py b/skillopt/optimizer/select.py new file mode 100644 index 0000000..fc49eeb --- /dev/null +++ b/skillopt/optimizer/select.py @@ -0,0 +1,4 @@ +"""Backward-compat stub — moved to skillopt.optimizer.clip.""" +from skillopt.optimizer.clip import rank_and_select # noqa: F401 + +__all__ = ["rank_and_select"] diff --git a/skillopt/optimizer/skill.py b/skillopt/optimizer/skill.py new file mode 100644 index 0000000..65d5741 --- /dev/null +++ b/skillopt/optimizer/skill.py @@ -0,0 +1,201 @@ +"""ReflACT skill operations — edit application and patch processing. + +The Update stage (⑤) of the ReflACT pipeline: apply a ranked set of +edits to the current skill document, producing an updated candidate. +Analogous to optimizer.step() in neural network training. +""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from skillopt.types import Edit as EditType, Patch as PatchType + +SLOW_UPDATE_START = "" +SLOW_UPDATE_END = "" + +# Skill-aware reflection (EmbodiSkill S_app) appendix region. Like the slow +# update region, it is protected: step-level analyst edits must not modify it. +APPENDIX_START = "" +APPENDIX_END = "" + +# All protected (start, end) marker pairs. Step-level edits cannot target text +# inside any of these regions, and `append` / `insert_after`-fallback ops are +# inserted before the earliest-occurring region so protected blocks stay at the +# document tail. With only the slow-update region present, every helper reduces +# to the original slow-update-only behavior (byte-identical skill output). +_PROTECTED_REGIONS: tuple[tuple[str, str], ...] = ( + (SLOW_UPDATE_START, SLOW_UPDATE_END), + (APPENDIX_START, APPENDIX_END), +) + + +def _earliest_protected_start(skill: str) -> int: + """Index of the earliest protected-region start marker, or -1 if none.""" + positions = [ + idx + for idx in (skill.find(start) for start, _ in _PROTECTED_REGIONS) + if idx != -1 + ] + return min(positions) if positions else -1 + + +def _is_in_protected_region(skill: str, target: str) -> bool: + """Check if *target* text falls within any protected region.""" + if not target: + return False + target_idx = skill.find(target) + if target_idx == -1: + return False + for start_marker, end_marker in _PROTECTED_REGIONS: + start_idx = skill.find(start_marker) + end_idx = skill.find(end_marker) + if start_idx == -1 or end_idx == -1: + continue + region_end = end_idx + len(end_marker) + if start_idx <= target_idx < region_end: + return True + return False + + +def _is_in_slow_update_region(skill: str, target: str) -> bool: + """Backward-compatible alias kept for any external callers/tests.""" + return _is_in_protected_region(skill, target) + + +def _strip_slow_update_markers(text: str) -> str: + """Remove any protected-region markers from edit content to prevent duplication.""" + return ( + text.replace(SLOW_UPDATE_START, "") + .replace(SLOW_UPDATE_END, "") + .replace(APPENDIX_START, "") + .replace(APPENDIX_END, "") + ) + + +def _edit_fields(edit: EditType | dict) -> tuple[str, str, str]: + op = edit.op if hasattr(edit, "op") else edit.get("op", "") + content = _strip_slow_update_markers( + (edit.content if hasattr(edit, "content") else edit.get("content", "")).strip() + ) + target = edit.target if hasattr(edit, "target") else edit.get("target", "") + return op, content, target + + +def _apply_edit_with_report(skill: str, edit: EditType | dict) -> tuple[str, dict]: + op, content, target = _edit_fields(edit) + report = { + "op": op, + "target": target[:200], + "content_preview": content[:200], + "status": "unknown", + } + + if target and _is_in_protected_region(skill, target): + report["status"] = "skipped_protected_region" + return skill, report + + if op == "append": + prot_start = _earliest_protected_start(skill) + if prot_start != -1: + before = skill[:prot_start].rstrip() + after = skill[prot_start:] + report["status"] = "applied_append_before_protected_region" + return before + "\n\n" + content + "\n\n" + after, report + report["status"] = "applied_append" + return skill.rstrip() + "\n\n" + content + "\n", report + + if op == "insert_after": + if not target or target not in skill: + prot_start = _earliest_protected_start(skill) + if prot_start != -1: + before = skill[:prot_start].rstrip() + after = skill[prot_start:] + report["status"] = "applied_insert_after_fallback_before_protected_region" + return before + "\n\n" + content + "\n\n" + after, report + report["status"] = "applied_insert_after_fallback_append" + return skill.rstrip() + "\n\n" + content + "\n", report + idx = skill.index(target) + len(target) + newline = skill.find("\n", idx) + insert_at = newline + 1 if newline != -1 else len(skill) + report["status"] = "applied_insert_after" + return skill[:insert_at] + "\n" + content + "\n" + skill[insert_at:], report + + if op == "replace": + if not target: + report["status"] = "skipped_replace_missing_target" + return skill, report + if target not in skill: + report["status"] = "skipped_replace_target_not_found" + return skill, report + report["status"] = "applied_replace" + return skill.replace(target, content, 1), report + + if op == "delete": + if not target: + report["status"] = "skipped_delete_missing_target" + return skill, report + if target not in skill: + report["status"] = "skipped_delete_target_not_found" + return skill, report + report["status"] = "applied_delete" + return skill.replace(target, "", 1), report + + report["status"] = "skipped_unknown_op" + return skill, report + + +def apply_edit(skill: str, edit: EditType | dict) -> str: + """Apply a single edit operation to the skill document. + + Parameters + ---------- + skill : str + Current skill document content. + edit : Edit | dict + An :class:`~skillopt.types.Edit` instance or a plain dict with + keys ``op``, ``content``, ``target``. + + Edits targeting the protected slow-update region are silently skipped. + """ + updated_skill, _ = _apply_edit_with_report(skill, edit) + return updated_skill + + +def apply_patch_with_report( + skill: str, + patch: PatchType | dict, +) -> tuple[str, list[dict]]: + """Apply a patch and return a per-edit report for observability.""" + edits = patch.edits if hasattr(patch, "edits") else patch.get("edits", []) + reports: list[dict] = [] + for idx, edit in enumerate(edits, 1): + try: + skill, report = _apply_edit_with_report(skill, edit) + report["index"] = idx + except Exception as exc: # noqa: BLE001 + report = { + "index": idx, + "op": "", + "target": "", + "content_preview": "", + "status": "error", + "error": str(exc), + } + reports.append(report) + return skill, reports + + +def apply_patch(skill: str, patch: PatchType | dict) -> str: + """Apply a patch (list of edits) to the skill document sequentially. + + Parameters + ---------- + skill : str + Current skill document content. + patch : Patch | dict + A :class:`~skillopt.types.Patch` instance or a plain dict with + key ``edits`` containing a list of edit operations. + """ + updated_skill, _ = apply_patch_with_report(skill, patch) + return updated_skill diff --git a/skillopt/optimizer/skill_aware.py b/skillopt/optimizer/skill_aware.py new file mode 100644 index 0000000..de39427 --- /dev/null +++ b/skillopt/optimizer/skill_aware.py @@ -0,0 +1,206 @@ +"""Skill-Aware Reflection — analyst prompt augmentation (EmbodiSkill). + +When ``use_skill_aware_reflection`` is enabled, the failure/success analysts are +asked to additionally classify each reflection by EmbodiSkill type and to route +**EXECUTION_LAPSE** reflections (the skill rule is correct, the executor just +failed to follow it) into a separate ``appendix_notes`` list instead of the body +patch. This module owns: + +1. the instruction text appended to the resolved analyst system prompt, and +2. extraction of ``appendix_notes`` from the analyst JSON response. + +Design notes +------------ +- The suffix is appended **at runtime, gated by the toggle**, so env-specific and + generic analyst prompts are augmented uniformly and — when the toggle is off — + remain byte-identical to baseline. +- Discrimination follows the paper / GMemory: ``SKILL_DEFECT`` = the skill rule is + wrong / missing / underspecified (→ body edit); ``EXECUTION_LAPSE`` = the rule + is valid but the agent didn't follow it (→ appendix reminder, body untouched). + **When unsure, default to EXECUTION_LAPSE** (protect the body — never delete a + valid rule over a one-off execution slip). +- Success reflections are labeled DISCOVERY / OPTIMIZATION for logging only; their + edit behavior is unchanged. +""" +from __future__ import annotations + + +# ── Runtime switch (config-driven, env-independent) ───────────────────────── +# +# The trainer calls :func:`configure_skill_aware_reflection` once at startup +# from the resolved config. ``run_minibatch_reflect`` then picks these values +# up automatically, so env adapters never need to thread the toggle through — +# the feature is controlled purely by ``optimizer.use_skill_aware_reflection`` +# regardless of benchmark. Mirrors the ``configure_azure_openai`` pattern in +# :mod:`skillopt.model`. Explicit kwargs at a call site still take precedence +# (backward compatible). + +_RUNTIME: dict = {"enabled": False, "appendix_source": "both"} + + +def configure_skill_aware_reflection( + enabled: bool, + appendix_source: str = "both", +) -> None: + """Set the process-wide skill-aware reflection switch from config.""" + _RUNTIME["enabled"] = bool(enabled) + _RUNTIME["appendix_source"] = str(appendix_source or "both") + + +def is_skill_aware_enabled() -> bool: + return bool(_RUNTIME["enabled"]) + + +def get_skill_aware_appendix_source() -> str: + return str(_RUNTIME["appendix_source"]) + + +# ── Prompt suffixes ───────────────────────────────────────────────────────── + +# Appended to the FAILURE analyst system prompt when the toggle is on. +ERROR_SUFFIX = """ + +## Skill-Aware Reflection (EmbodiSkill) + +Before proposing body edits, classify EACH failure pattern as one of: + +- **SKILL_DEFECT**: the current skill is wrong, missing, or underspecified for + this situation — i.e. an agent that *followed the skill* would still fail, or + the skill gives no relevant guidance. These become normal body `edits`. +- **EXECUTION_LAPSE**: the skill ALREADY contains a relevant, correct rule that + would have avoided the failure, but the agent did not follow it (e.g. ignored a + rule, malformed output, copied the feedback text verbatim, emitted a non-action + token like "stop", or otherwise broke execution unrelated to skill content). + +Discrimination test: "Is there a rule in the current skill that, if followed, +prevents this failure?" If yes → EXECUTION_LAPSE. If no (rule absent/wrong) → +SKILL_DEFECT. **When genuinely unsure, choose EXECUTION_LAPSE** — do not edit or +delete a valid rule over a one-off execution slip. + +Routing: +- SKILL_DEFECT → put the fix in `patch.edits` (body), as usual. +- EXECUTION_LAPSE → put a concise reminder in `appendix_notes` (a flat list of + strings). DO NOT add a body edit for it. Each note should re-emphasize the + existing valid rule the agent failed to follow; it must NOT introduce a new + rule. Keep notes short, concrete, and reusable. + +Add `appendix_notes` as a TOP-LEVEL key of your JSON output (a sibling of +`patch`), e.g. `"appendix_notes": ["Follow the existing X rule before Y."]`. +Use `[]` when there is no execution lapse. Body edits and appendix notes are +independent: a batch may yield only edits, only notes, both, or neither. +""" + +# Appended to the SUCCESS analyst system prompt when the toggle is on. +SUCCESS_SUFFIX = """ + +## Skill-Aware Reflection (EmbodiSkill) + +For each proposed edit, optionally label its `reflection_type` for logging: +- **DISCOVERY**: a useful new rule not yet in the skill (typically an `append`). +- **OPTIMIZATION**: a better way to perform an existing rule (typically a + `replace` of that rule). + +This labeling does not change edit behavior. You may also add a top-level +`appendix_notes` list (flat strings) if a successful trajectory reveals an +existing valid rule worth re-emphasizing; otherwise use `[]`. +""" + + +def augment_error_prompt(system_prompt: str) -> str: + """Append the failure-analyst skill-aware instruction.""" + return system_prompt.rstrip() + "\n" + ERROR_SUFFIX + + +def augment_success_prompt(system_prompt: str) -> str: + """Append the success-analyst skill-aware instruction.""" + return system_prompt.rstrip() + "\n" + SUCCESS_SUFFIX + + +# ── Response parsing ──────────────────────────────────────────────────────── + + +def extract_appendix_notes(result: dict | None) -> list[str]: + """Pull a clean list of appendix-note strings from an analyst JSON result. + + Tolerant of shape: accepts a top-level ``appendix_notes`` list, a single + string, or items wrapped in dicts with a ``note``/``content`` field. Returns + ``[]`` for anything missing or malformed (so a non-compliant model degrades + gracefully to baseline body-only behavior). + """ + if not isinstance(result, dict): + return [] + raw = result.get("appendix_notes") + if raw is None: + return [] + if isinstance(raw, str): + raw = [raw] + if not isinstance(raw, list): + return [] + notes: list[str] = [] + for item in raw: + if isinstance(item, str): + text = item.strip() + elif isinstance(item, dict): + text = str(item.get("note") or item.get("content") or "").strip() + else: + text = "" + if text: + notes.append(text) + return notes + + +# ── Appendix consolidation (threshold-gated, paper Eq.11 UpdateSkillAppendix) ── + +_CONSOLIDATE_SYSTEM = ( + "You compact the Execution Notes Appendix of an agent skill. Each note " + "re-emphasizes an existing skill rule the agent failed to follow. Your job " + "is a periodic compaction pass: remove duplicates and redundant overlap, " + "merge near-identical reminders into one, and simplify phrasing while keeping " + "each note concrete and operational. Do not invent new rules. Preserve the " + "distinct actionable content. Return valid JSON only." +) + + +def consolidate_appendix_notes( + notes: list[str], + *, + chat_fn, + max_completion_tokens: int = 4096, +) -> list[str]: + """LLM-consolidate appendix notes: dedupe / merge / compact. + + Mirrors GMemory ``_maybe_refactor_execution_notes`` and paper Eq.11. ``chat_fn`` + is the optimizer chat callable ``(system, user, max_completion_tokens, retries, + stage) -> (text, meta)``. On ANY failure (parse, empty, exception) the original + notes are returned unchanged, so consolidation can never lose the appendix. + """ + from skillopt.utils import extract_json # local import to avoid cycles + + clean = [str(n).strip() for n in (notes or []) if str(n).strip()] + if len(clean) < 2: + return clean + + numbered = "\n".join(f"{i}. {n}" for i, n in enumerate(clean, 1)) + user = ( + f"## Current Execution Notes ({len(clean)} total)\n{numbered}\n\n" + "Compact these into a shorter list without losing distinct actionable " + "information. Merge duplicates and near-duplicates; keep each note short, " + "concrete, and reusable. Return valid JSON only with this schema:\n" + '{ "appendix_notes": ["compacted note 1", "compacted note 2"] }' + ) + try: + response, _ = chat_fn( + system=_CONSOLIDATE_SYSTEM, + user=user, + max_completion_tokens=max_completion_tokens, + retries=2, + stage="appendix_consolidate", + ) + result = extract_json(response) + compacted = extract_appendix_notes(result) + # Guard: only accept a non-empty result that actually shrinks the set. + if compacted and len(compacted) <= len(clean): + return compacted + except Exception: # noqa: BLE001 + pass + return clean diff --git a/skillopt/optimizer/slow_update.py b/skillopt/optimizer/slow_update.py new file mode 100644 index 0000000..a2264ec --- /dev/null +++ b/skillopt/optimizer/slow_update.py @@ -0,0 +1,396 @@ +"""ReflACT Slow Update — epoch-level longitudinal skill refinement. + +At the end of each epoch, the slow update compares rollout performance of the +same sample set under the previous epoch's skill vs. the current epoch's skill +(Markov: only adjacent epochs). A optimizer analyzes regressions, improvements, +and persistent failures, then writes a free-form guidance block into a +**protected** section of the skill document. This section cannot be modified by +step-level analyst edits — only the slow update process overwrites it. + +Public API +---------- +- :func:`inject_empty_slow_update_field` — add empty placeholder (epoch 1) +- :func:`extract_slow_update_field` — read current content +- :func:`replace_slow_update_field` — overwrite content +- :func:`has_slow_update_field` — check if markers are present +- :func:`build_comparison_text` — format side-by-side rollout results +- :func:`run_slow_update` — optimizer call to produce guidance +""" +from __future__ import annotations + +import json +import os +import traceback + +from skillopt.model import chat_optimizer +from skillopt.prompts import load_prompt +from skillopt.utils import extract_json + +# ── Protected field markers ───────────────────────────────────────────────── + +SLOW_UPDATE_START = "" +SLOW_UPDATE_END = "" + +# ── Field manipulation helpers ────────────────────────────────────────────── + + +def has_slow_update_field(skill: str) -> bool: + return SLOW_UPDATE_START in skill and SLOW_UPDATE_END in skill + + +def inject_empty_slow_update_field(skill: str) -> str: + if has_slow_update_field(skill): + return skill + block = ( + f"\n\n{SLOW_UPDATE_START}\n" + f"{SLOW_UPDATE_END}\n" + ) + return skill.rstrip() + block + + +def extract_slow_update_field(skill: str) -> str: + start = skill.find(SLOW_UPDATE_START) + end = skill.find(SLOW_UPDATE_END) + if start == -1 or end == -1: + return "" + inner_start = start + len(SLOW_UPDATE_START) + return skill[inner_start:end].strip() + + +def _strip_all_slow_update_fields(skill: str) -> str: + """Remove every SLOW_UPDATE_START/END pair (and content between) from *skill*.""" + while True: + start = skill.find(SLOW_UPDATE_START) + if start == -1: + break + end = skill.find(SLOW_UPDATE_END, start) + if end == -1: + # Orphan start marker — remove it + skill = skill[:start] + skill[start + len(SLOW_UPDATE_START):] + break + skill = skill[:start] + skill[end + len(SLOW_UPDATE_END):] + # Clean up stray end markers + skill = skill.replace(SLOW_UPDATE_END, "") + # Collapse excess blank lines left behind + while "\n\n\n" in skill: + skill = skill.replace("\n\n\n", "\n\n") + return skill.rstrip() + + +def replace_slow_update_field(skill: str, new_content: str) -> str: + # Remove all existing slow update regions first to guarantee exactly one. + skill = _strip_all_slow_update_fields(skill) + block = ( + f"\n\n{SLOW_UPDATE_START}\n" + f"{new_content.strip()}\n" + f"{SLOW_UPDATE_END}\n" + ) + return skill + block + + +# ── Comparison text builder ───────────────────────────────────────────────── + + +# NOTE: Character-length limits on the comparison samples fed to the slow-update / +# meta-skill optimizer have been REMOVED. Previously a whole-trajectory cap plus +# per-field caps (cmd/obs/reasoning/etc.) and comparison-metadata caps +# (task/answer/fail_reason) trimmed this context to save optimizer tokens and +# speed up the call. They never affected what gets written into the skill — only +# how much longitudinal context the optimizer sees. We now pass everything through +# at full length: the comparison input is as long as the source data is. + + +def _clip_text(value, limit: int | None = None) -> str: + # Truncation disabled: return the full text. The `limit` argument is kept only + # for call-site compatibility and is intentionally ignored (see NOTE above). + if value is None: + return "" + return str(value) + + +def _read_trajectory(rollout_dir: str, task_id: str) -> str: + """Read and format a single trajectory from a rollout directory.""" + conv_path = os.path.join(rollout_dir, "predictions", task_id, "conversation.json") + if not os.path.exists(conv_path): + return "(trajectory not available)" + try: + with open(conv_path) as f: + conversation = json.load(f) + except Exception: + return "(trajectory read error)" + if not conversation: + return "(empty trajectory)" + + lines: list[str] = [] + for entry in conversation: + if not isinstance(entry, dict): + continue + # Per-field truncation removed: feed each step's full cmd/obs/reasoning/ + # action/feedback/content (see NOTE above). + if entry.get("type") == "tool_call": + cmd = _clip_text(entry.get("cmd")) + obs = _clip_text(entry.get("obs")) + lines.append(f"[action] {cmd}") + lines.append(f"[obs] {obs}") + elif "action" in entry and "env_feedback" in entry: + step = entry.get("step", "?") + reasoning = _clip_text(entry.get("reasoning")) + action = _clip_text(entry.get("action")) + feedback = _clip_text(entry.get("env_feedback")) + if reasoning: + lines.append(f"[step {step} think] {reasoning}") + lines.append(f"[step {step} action] {action}") + lines.append(f"[step {step} obs] {feedback}") + elif entry.get("role") == "system": + msg = _clip_text(entry.get("content")) + lines.append(f"[verification] {msg}") + else: + msg = _clip_text(entry.get("content")) + role = entry.get("role", "agent") + lines.append(f"[{role}] {msg}") + + # Whole-trajectory truncation removed: return the full formatted trajectory. + return "\n".join(lines) + + +# ── Structured comparison pairs ───────────────────────────────────────────── + + +def build_comparison_pairs( + results_prev: list[dict], + results_curr: list[dict], + items: list[dict], + prev_rollout_dir: str = "", + curr_rollout_dir: str = "", +) -> list[dict]: + """Build a structured list of per-sample comparison entries. + + Each entry bundles the original item, both rollout results, the change + category, and both trajectories into one dict — the single source of + truth for this sample's longitudinal comparison. + + Returns + ------- + list[dict] + One dict per sample with keys: + ``id, task, category, prev, curr, prev_trajectory, curr_trajectory`` + """ + prev_by_id = {str(r["id"]): r for r in results_prev} + curr_by_id = {str(r["id"]): r for r in results_curr} + + pairs: list[dict] = [] + for item in items: + tid = str(item.get("id", "")) + prev = prev_by_id.get(tid, {}) + curr = curr_by_id.get(tid, {}) + prev_ok = bool(prev.get("hard", 0)) + curr_ok = bool(curr.get("hard", 0)) + + if not prev_ok and curr_ok: + category = "improved" + elif prev_ok and not curr_ok: + category = "regressed" + elif not prev_ok and not curr_ok: + category = "persistent_fail" + else: + category = "stable_success" + + pairs.append({ + "id": tid, + "task": item.get("question", item.get("task_description", item.get("instruction", tid))), + "category": category, + "prev": { + "hard": int(prev_ok), + "soft": float(prev.get("soft", 0.0)), + "predicted_answer": prev.get("predicted_answer", prev.get("answer", "N/A")), + "fail_reason": prev.get("fail_reason", ""), + }, + "curr": { + "hard": int(curr_ok), + "soft": float(curr.get("soft", 0.0)), + "predicted_answer": curr.get("predicted_answer", curr.get("answer", "N/A")), + "fail_reason": curr.get("fail_reason", ""), + }, + "prev_trajectory": ( + _read_trajectory(prev_rollout_dir, tid) if prev_rollout_dir else "" + ), + "curr_trajectory": ( + _read_trajectory(curr_rollout_dir, tid) if curr_rollout_dir else "" + ), + }) + + return pairs + + +def save_comparison_pairs(pairs: list[dict], out_path: str) -> None: + """Persist comparison pairs to JSON (without trajectory text to save space).""" + slim = [] + for p in pairs: + slim.append({ + "id": p["id"], + "task": p["task"], + "category": p["category"], + "prev": p["prev"], + "curr": p["curr"], + }) + with open(out_path, "w") as f: + json.dump(slim, f, ensure_ascii=False, indent=2) + + +def format_comparison_text(pairs: list[dict]) -> str: + """Format structured comparison pairs into optimizer-readable text.""" + by_cat: dict[str, list[dict]] = { + "regressed": [], + "persistent_fail": [], + "improved": [], + "stable_success": [], + } + for p in pairs: + by_cat.setdefault(p["category"], []).append(p) + + total = len(pairs) + parts = [ + f"## Longitudinal Comparison Summary\n" + f"Total samples: {total}\n" + f"- Improved (wrong→right): {len(by_cat['improved'])}\n" + f"- Regressed (right→wrong): {len(by_cat['regressed'])}\n" + f"- Persistent failures (wrong→wrong): {len(by_cat['persistent_fail'])}\n" + f"- Stable successes (right→right): {len(by_cat['stable_success'])}\n" + ] + + categories = [ + ("regressed", "Regressions (right→wrong) — HIGHEST PRIORITY", True), + ("persistent_fail", "Persistent Failures (wrong→wrong)", True), + ("improved", "Improvements (wrong→right)", True), + ("stable_success", "Stable Successes (right→right)", False), + ] + + for cat_key, label, show_traj in categories: + entries = by_cat[cat_key] + if not entries: + parts.append(f"### {label}\n(none)\n") + continue + + lines = [f"### {label}"] + for e in entries: + prev = e["prev"] + curr = e["curr"] + lines.append( + f"\n#### Task {e['id']}: {e['task']}\n" + f"- Prev epoch: {'PASS' if prev['hard'] else 'FAIL'} " + f"(soft={prev['soft']:.2f}) — answer: {str(prev['predicted_answer'])}\n" + f"- Curr epoch: {'PASS' if curr['hard'] else 'FAIL'} " + f"(soft={curr['soft']:.2f}) — answer: {str(curr['predicted_answer'])}" + ) + if curr.get("fail_reason"): + lines.append(f"- Curr fail reason: {curr['fail_reason']}") + if prev.get("fail_reason") and not prev["hard"]: + lines.append(f"- Prev fail reason: {prev['fail_reason']}") + + if show_traj: + if e.get("prev_trajectory"): + lines.append( + f"\n**Previous epoch trajectory:**\n```\n{e['prev_trajectory']}\n```" + ) + if e.get("curr_trajectory"): + lines.append( + f"\n**Current epoch trajectory:**\n```\n{e['curr_trajectory']}\n```" + ) + + parts.append("\n".join(lines)) + + return "\n\n".join(parts) + + + +# ── Optimizer call ──────────────────────────────────────────────────────────── + + +def run_slow_update( + skill_content: str, + results_prev: list[dict], + results_curr: list[dict], + items: list[dict], + *, + prev_skill: str = "", + prev_slow_update_content: str = "", + prev_rollout_dir: str = "", + curr_rollout_dir: str = "", + comparison_pairs: list[dict] | None = None, + system_prompt: str | None = None, +) -> dict | None: + """Run the slow update optimizer call for one epoch boundary. + + Parameters + ---------- + skill_content : str + Current epoch's skill (after fast updates). + results_prev : list[dict] + Rollout results of the 20 samples under previous epoch's skill. + results_curr : list[dict] + Rollout results of the 20 samples under current epoch's skill. + items : list[dict] + The 20 sample items used for comparison. + prev_skill : str + Previous epoch's skill content. + prev_slow_update_content : str + The slow update guidance from the previous epoch (to reflect on). + prev_rollout_dir : str + Path to previous epoch rollout output (contains predictions/). + curr_rollout_dir : str + Path to current epoch rollout output (contains predictions/). + system_prompt : str | None + Custom system prompt override. + + Returns + ------- + dict | None + Conforms to :class:`~skillopt.types.SlowUpdateResult`: + ``{"reasoning": str, "slow_update_content": str}`` or ``None``. + """ + actual_system = system_prompt if system_prompt is not None else load_prompt("slow_update") + + pairs = comparison_pairs + if pairs is None: + pairs = build_comparison_pairs( + results_prev, results_curr, items, + prev_rollout_dir=prev_rollout_dir, + curr_rollout_dir=curr_rollout_dir, + ) + comparison_text = format_comparison_text(pairs) + + prev_guidance_section = ( + prev_slow_update_content.strip() + if prev_slow_update_content and prev_slow_update_content.strip() + else "(No previous guidance — this is the first slow update.)" + ) + + user = ( + f"## Previous Epoch's Skill\n{prev_skill}\n\n" + f"## Current Epoch's Skill\n{skill_content}\n\n" + f"## Previous Slow Update Guidance\n" + f"The following guidance was active during the current epoch. " + f"Reflect on its effectiveness before writing the new version.\n\n" + f"{prev_guidance_section}\n\n" + f"## Longitudinal Comparison (same 20 tasks, two skill versions)\n" + f"{comparison_text}" + ) + + try: + response, _ = chat_optimizer( + system=actual_system, + user=user, + max_completion_tokens=16384, + retries=3, + stage="slow_update", + ) + result = extract_json(response) + if result and result.get("slow_update_content"): + return { + "reasoning": str(result.get("reasoning", "")).strip(), + "slow_update_content": str(result["slow_update_content"]).strip(), + } + except Exception: # noqa: BLE001 + traceback.print_exc() + + return None diff --git a/skillopt/optimizer/update_modes.py b/skillopt/optimizer/update_modes.py new file mode 100644 index 0000000..e2dc22d --- /dev/null +++ b/skillopt/optimizer/update_modes.py @@ -0,0 +1,135 @@ +"""Helpers for switching between patch edits and rewrite-from-suggestions.""" +from __future__ import annotations + +from typing import Any + +PATCH_MODE = "patch" +REWRITE_MODE = "rewrite_from_suggestions" +FULL_REWRITE_MINIBATCH_MODE = "full_rewrite_minibatch" + + +def normalize_update_mode(mode: str | None) -> str: + raw = str(mode or PATCH_MODE).strip().lower() + aliases = { + "patch": PATCH_MODE, + "edits": PATCH_MODE, + "rewrite": REWRITE_MODE, + "rewrite_from_suggestions": REWRITE_MODE, + "suggestions": REWRITE_MODE, + "rewrite_suggestions": REWRITE_MODE, + "full_rewrite": FULL_REWRITE_MINIBATCH_MODE, + "full_rewrite_minibatch": FULL_REWRITE_MINIBATCH_MODE, + "minibatch_full_rewrite": FULL_REWRITE_MINIBATCH_MODE, + "skill_rewrite_minibatch": FULL_REWRITE_MINIBATCH_MODE, + } + return aliases.get(raw, PATCH_MODE) + + +def is_rewrite_mode(mode: str | None) -> bool: + return normalize_update_mode(mode) == REWRITE_MODE + + +def is_full_rewrite_minibatch_mode(mode: str | None) -> bool: + return normalize_update_mode(mode) == FULL_REWRITE_MINIBATCH_MODE + + +def payload_key(mode: str | None) -> str: + if is_full_rewrite_minibatch_mode(mode): + return "skill_candidates" + return "revise_suggestions" if is_rewrite_mode(mode) else "edits" + + +def payload_label(mode: str | None, *, singular: bool = False, title: bool = False) -> str: + if is_full_rewrite_minibatch_mode(mode): + word = "skill candidate" if singular else "skill candidates" + elif is_rewrite_mode(mode): + word = "suggestion" if singular else "suggestions" + else: + word = "edit" if singular else "edits" + return word.title() if title else word + + +def get_payload_items(container: dict | None, mode: str | None) -> list[dict]: + if not isinstance(container, dict): + return [] + items = container.get(payload_key(mode), []) + return items if isinstance(items, list) else [] + + +def set_payload_items(container: dict, items: list[dict], mode: str | None) -> dict: + container[payload_key(mode)] = items + return container + + +def truncate_payload(container: dict, max_items: int, mode: str | None) -> dict: + if max_items < 0: + return container + items = get_payload_items(container, mode) + if len(items) > max_items: + set_payload_items(container, items[:max_items], mode) + return container + + +def describe_item(item: dict, mode: str | None, *, max_chars: int | None = None) -> str: + if not isinstance(item, dict): + return "" + if is_full_rewrite_minibatch_mode(mode): + parts = [ + f"title={item.get('title', '')!r}", + f"change_summary={item.get('change_summary', [])!r}", + ] + if item.get("source_type"): + parts.append(f"source={item.get('source_type')}") + if item.get("support_count") is not None: + parts.append(f"support={item.get('support_count')}") + new_skill = str(item.get("new_skill", "")).strip() + if new_skill: + parts.append(f"new_skill_preview={new_skill!r}") + text = " ".join(parts) + elif is_rewrite_mode(mode): + parts = [ + f"type={item.get('type', '?')}", + f"title={item.get('title', '')!r}", + f"instruction={item.get('instruction', '')!r}", + ] + if item.get("priority_hint"): + parts.append(f"priority={item.get('priority_hint')}") + if item.get("support_count") is not None: + parts.append(f"support={item.get('support_count')}") + text = " ".join(parts) + else: + op = item.get("op", "?") + target = item.get("target", "") + content = item.get("content", "") + parts = [f"op={op}"] + if target: + parts.append(f"target={target!r}") + if content: + parts.append(f"content={content!r}") + if item.get("support_count") is not None: + parts.append(f"support={item.get('support_count')}") + text = " ".join(parts) + # Truncation disabled: the optimizer is given the full item description. + return text + + +def short_item_summary(item: dict, mode: str | None, *, max_chars: int | None = None) -> dict[str, Any]: + if is_full_rewrite_minibatch_mode(mode): + return { + "title": str(item.get("title", "")), + "change_summary": [ + str(x) for x in item.get("change_summary", []) + ] if isinstance(item.get("change_summary"), list) else [], + "source_type": item.get("source_type", ""), + } + if is_rewrite_mode(mode): + return { + "type": item.get("type", "?"), + "title": str(item.get("title", "")), + "instruction": str(item.get("instruction", "")), + } + return { + "op": item.get("op", "?"), + "content": str(item.get("content", "")), + "target": item.get("target", ""), + } diff --git a/skillopt/prompts/__init__.py b/skillopt/prompts/__init__.py new file mode 100644 index 0000000..af1bc58 --- /dev/null +++ b/skillopt/prompts/__init__.py @@ -0,0 +1,63 @@ +"""Prompt loading utilities for ReflACT. + +Prompts are stored as ``.md`` files and loaded at runtime: + +- **Generic** prompts live in ``skillopt/prompts/*.md`` +- **Env-specific** prompts live in ``skillopt/envs//prompts/*.md`` + +``load_prompt(name, env)`` tries the env-specific path first, then falls +back to the generic default. +""" +from __future__ import annotations + +import os + +_PROMPTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_REFLACT_DIR = os.path.dirname(_PROMPTS_DIR) + +_cache: dict[str, str] = {} + + +def _read_file(path: str) -> str | None: + if path in _cache: + return _cache[path] + if not os.path.isfile(path): + return None + with open(path, encoding="utf-8") as f: + content = f.read() + _cache[path] = content + return content + + +def load_prompt(name: str, env: str | None = None) -> str: + """Load a prompt by name with env-specific override and generic fallback. + + Lookup order: + 1. ``skillopt/envs/{env}/prompts/{name}.md`` (if *env* given) + 2. ``skillopt/prompts/{name}.md`` (generic default) + + Raises ``FileNotFoundError`` if neither path exists. + """ + if env is not None: + env_path = os.path.join(_REFLACT_DIR, "envs", env, "prompts", f"{name}.md") + content = _read_file(env_path) + if content is not None: + return content + + generic_path = os.path.join(_PROMPTS_DIR, f"{name}.md") + content = _read_file(generic_path) + if content is not None: + return content + + searched = [] + if env is not None: + searched.append(os.path.join("skillopt/envs", env, "prompts", f"{name}.md")) + searched.append(f"skillopt/prompts/{name}.md") + raise FileNotFoundError( + f"Prompt '{name}' not found. Searched: {', '.join(searched)}" + ) + + +def clear_cache() -> None: + """Clear the prompt file cache (useful for testing).""" + _cache.clear() diff --git a/skillopt/prompts/analyst_error.md b/skillopt/prompts/analyst_error.md new file mode 100644 index 0000000..af1c0c5 --- /dev/null +++ b/skillopt/prompts/analyst_error.md @@ -0,0 +1,41 @@ +You are an expert failure-analysis agent for AI agent tasks. + +You will be given MULTIPLE failed agent trajectories from a single minibatch +and the current skill document. +Your job is to identify the most important COMMON failure patterns across +the batch and propose a concise set of skill edits. + +## Analysis Process +1. Read ALL trajectories in the minibatch. +2. Identify the most prevalent, systematic failure patterns across them. +3. For each pattern, classify its failure type. +4. Propose skill edits that address the COMMON patterns — not individual edge cases. +5. Edits must be generalizable; do not hardcode task-specific values. +6. Only patch gaps in the skill — do not duplicate existing content. + +You will be told the maximum number of edits (the budget L). Produce AT MOST L edits, +focusing on the highest-impact patterns. You may produce fewer if warranted. + +Respond ONLY with a valid JSON object (no markdown fences, no extra text): +{ + "batch_size": , + "failure_summary": [ + {"failure_type": "", "count": , "description": ""} + ], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +Only include edits that are needed. "edits" can be an empty list if no patch is warranted. + +IMPORTANT: The skill document may contain a section between + and markers. +This is a PROTECTED section managed by a separate slow-update process. +Do NOT propose any edits that target, modify, or delete content within +these markers. diff --git a/skillopt/prompts/analyst_error_full_rewrite.md b/skillopt/prompts/analyst_error_full_rewrite.md new file mode 100644 index 0000000..5d7e2c5 --- /dev/null +++ b/skillopt/prompts/analyst_error_full_rewrite.md @@ -0,0 +1,32 @@ +You will be given several failed agent trajectories from one minibatch and the current skill document. + +Summarize the lessons from these trajectories into one complete replacement skill document. + +When rewriting from a minibatch, use the current trajectories as the primary +evidence for updates. Preserve essential task-format instructions, but avoid mechanically carrying over +stale, redundant, or conflicting rules. Prefer a concise, coherent replacement +skill over a long document with weakly supported guidance. + +Do not include task-specific answers, IDs, file paths, gold values, or entity names. +If the skill contains a protected block between and +, keep that block unchanged. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "failure_summary": [ + {"failure_type": "", "count": , "description": ""} + ], + "patch": { + "reasoning": "", + "skill_candidates": [ + { + "title": "", + "change_summary": ["", ""], + "new_skill": "" + } + ] + } +} + +Return exactly one item in "skill_candidates". diff --git a/skillopt/prompts/analyst_error_rewrite.md b/skillopt/prompts/analyst_error_rewrite.md new file mode 100644 index 0000000..1d34d0e --- /dev/null +++ b/skillopt/prompts/analyst_error_rewrite.md @@ -0,0 +1,44 @@ +You are an expert failure-analysis agent for AI agent tasks. + +You will be given MULTIPLE failed agent trajectories from a single minibatch +and the current skill document. +Your job is to identify the most important COMMON failure patterns across +the batch and propose a concise set of skill-revision suggestions. + +## Analysis Process +1. Read ALL trajectories in the minibatch. +2. Identify the most prevalent, systematic failure patterns across them. +3. For each pattern, classify its failure type. +4. Propose revision suggestions that address the COMMON patterns, not individual edge cases. +5. Suggestions must be generalizable and should help a later optimizer rewrite the full skill document. +6. Do not hardcode task-specific values. + +You will be told the maximum number of suggestions (the budget L). Produce AT MOST L suggestions, +focusing on the highest-impact patterns. You may produce fewer if warranted. + +Respond ONLY with a valid JSON object (no markdown fences, no extra text): +{ + "batch_size": , + "failure_summary": [ + {"failure_type": "", "count": , "description": ""} + ], + "patch": { + "reasoning": "", + "revise_suggestions": [ + { + "type": "add_rule|remove_rule|merge_rules|reorganize|compress|clarify", + "title": "", + "motivation": "", + "instruction": "", + "priority_hint": "high|medium|low" + } + ] + } +} +"revise_suggestions" may be an empty list if no revision is warranted. + +IMPORTANT: The skill document may contain a section between + and markers. +This is a PROTECTED section managed by a separate slow-update process. +Do NOT propose suggestions that target, modify, or delete content within +these markers. diff --git a/skillopt/prompts/analyst_success.md b/skillopt/prompts/analyst_success.md new file mode 100644 index 0000000..f79336d --- /dev/null +++ b/skillopt/prompts/analyst_success.md @@ -0,0 +1,36 @@ +You are an expert success-pattern analyst for AI agents. + +You will be given MULTIPLE successful agent trajectories from a single minibatch +and the current skill document. Your job is to identify generalizable behavior +patterns that are COMMON across the batch and worth encoding in the skill. + +## Rules +- Only propose patches for patterns NOT already covered in the skill. +- Focus on patterns that appear across MULTIPLE trajectories in the batch. +- Be concise. Patterns must generalize beyond specific tasks. +- Prefer reinforcing existing sections over adding new top-level sections. + +You will be told the maximum number of edits (the budget L). Produce AT MOST L edits, +focusing on the most broadly applicable patterns. You may produce fewer if warranted. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "success_patterns": ["", ""], + "patch": { + "reasoning": "", + "edits": [ + {"op": "append", "content": ""}, + {"op": "insert_after", "target": "", "content": ""}, + {"op": "replace", "target": "", "content": ""}, + {"op": "delete", "target": ""} + ] + } +} +"edits" may be empty if the skill already covers all observed patterns. + +IMPORTANT: The skill document may contain a section between + and markers. +This is a PROTECTED section managed by a separate slow-update process. +Do NOT propose any edits that target, modify, or delete content within +these markers. diff --git a/skillopt/prompts/analyst_success_full_rewrite.md b/skillopt/prompts/analyst_success_full_rewrite.md new file mode 100644 index 0000000..eabfcf5 --- /dev/null +++ b/skillopt/prompts/analyst_success_full_rewrite.md @@ -0,0 +1,30 @@ +You will be given several successful agent trajectories from one minibatch and the current skill document. + +Summarize any useful lessons from these trajectories into one complete replacement skill document. + +When rewriting from a minibatch, use the current trajectories as the primary +evidence for updates. Preserve essential task-format instructions, but avoid mechanically carrying over +stale, redundant, or conflicting rules. Prefer a concise, coherent replacement +skill over a long document with weakly supported guidance. + +Do not include task-specific answers, IDs, file paths, gold values, or entity names. +If the skill contains a protected block between and +, keep that block unchanged. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "success_patterns": ["", ""], + "patch": { + "reasoning": "", + "skill_candidates": [ + { + "title": "", + "change_summary": ["", ""], + "new_skill": "" + } + ] + } +} + +Return exactly one item in "skill_candidates". diff --git a/skillopt/prompts/analyst_success_rewrite.md b/skillopt/prompts/analyst_success_rewrite.md new file mode 100644 index 0000000..1291b6d --- /dev/null +++ b/skillopt/prompts/analyst_success_rewrite.md @@ -0,0 +1,33 @@ +You are an expert success-pattern analyst for AI agent tasks. + +You will be given MULTIPLE successful agent trajectories from a single minibatch +and the current skill document. Your job is to identify broadly useful patterns +worth preserving in a later full-skill rewrite. + +## Rules +- Only propose revise_suggestions for patterns NOT already covered in the skill. +- Focus on patterns that appear across MULTIPLE trajectories in the batch. +- Keep suggestions general, concise, and rewrite-friendly. +- Prefer guidance that improves organization, clarity, or reusable behavior. + +You will be told the maximum number of suggestions (the budget L). Produce AT MOST L suggestions, +focusing on the most broadly applicable patterns. You may produce fewer if warranted. + +Respond ONLY with a valid JSON object: +{ + "batch_size": , + "success_patterns": ["", ""], + "patch": { + "reasoning": "", + "revise_suggestions": [ + { + "type": "add_rule|remove_rule|merge_rules|reorganize|compress|clarify", + "title": "", + "motivation": "", + "instruction": "", + "priority_hint": "high|medium|low" + } + ] + } +} +"revise_suggestions" may be empty if the skill already captures all useful patterns. diff --git a/skillopt/prompts/lr_autonomous.md b/skillopt/prompts/lr_autonomous.md new file mode 100644 index 0000000..81d1bc0 --- /dev/null +++ b/skillopt/prompts/lr_autonomous.md @@ -0,0 +1,20 @@ +You are an update-size controller for a skill-learning system. + +You will receive: +1. The current skill document. +2. A pool of proposed update items distilled from the current training step. +3. Brief evidence about the current rollout and training step. + +Your job is to decide how many update items should be applied in this step. +Use only the evidence shown in the prompt. Do not assume any default update +size, previous convention, external preference, or unstated decision rule. + +Do not rank the update items. Only decide the count. + +Respond ONLY with a valid JSON object: +{ + "learning_rate": , + "reasoning": "", + "confidence": "low|medium|high", + "risk_notes": ["", "..."] +} diff --git a/skillopt/prompts/merge_failure.md b/skillopt/prompts/merge_failure.md new file mode 100644 index 0000000..e448999 --- /dev/null +++ b/skillopt/prompts/merge_failure.md @@ -0,0 +1,30 @@ +You are a skill-edit coordinator. You receive multiple independently-proposed patches +from FAILURE analysis of agent trajectories. Merge them into ONE coherent, non-redundant patch. + +Merge guidelines: +1. **Deduplicate**: keep the best-worded version of similar edits. +2. **Resolve conflicts**: if patches contradict on the same point, + choose the one with stronger justification or synthesize both. +3. **Preserve unique insights**: include all non-redundant corrective edits. +4. **Prevalent-pattern bias**: edits appearing consistently across multiple patches + address systematic failures — preserve them with HIGH priority. + Edits from only one patch may be discarded if task-specific. +5. **Independence**: no two edits in the merged patch may target the same text region. +6. **Support count**: for each merged edit, estimate how many source patches support it. +7. **PROTECTED SECTION**: The skill may contain a section between + and markers. + Do NOT merge or produce any edits that target content within these markers. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "edits": [ + { + "op": "append|insert_after|replace|delete", + "target": "", + "content": "", + "support_count": , + "source_type": "failure" + } + ] +} diff --git a/skillopt/prompts/merge_failure_full_rewrite.md b/skillopt/prompts/merge_failure_full_rewrite.md new file mode 100644 index 0000000..0b5c20b --- /dev/null +++ b/skillopt/prompts/merge_failure_full_rewrite.md @@ -0,0 +1,28 @@ +You will be given complete skill candidates written from failed trajectories and the current skill document. + +Combine them into one complete replacement skill document. + +When merging full-skill candidates, preserve essential task-format instructions, +but do not mechanically retain stale, redundant, or +conflicting rules. If candidates disagree, prefer the concise rule with clearer +trajectory support and better consistency with the replacement skill. + +Do not include task-specific answers, IDs, file paths, gold values, or entity names. +If the current skill contains a protected block between and +, keep that block unchanged. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "skill_candidates": [ + { + "title": "", + "change_summary": ["", ""], + "new_skill": "", + "support_count": , + "source_type": "failure" + } + ] +} + +Return exactly one item in "skill_candidates". diff --git a/skillopt/prompts/merge_failure_rewrite.md b/skillopt/prompts/merge_failure_rewrite.md new file mode 100644 index 0000000..f86c079 --- /dev/null +++ b/skillopt/prompts/merge_failure_rewrite.md @@ -0,0 +1,26 @@ +You are a skill-revision coordinator. You receive multiple independently-proposed +revision suggestion sets from FAILURE analysis of agent trajectories. Merge them +into ONE coherent, non-redundant set of revise_suggestions. + +Merge guidelines: +1. Deduplicate overlapping suggestions. +2. Resolve conflicts by keeping the more general, better-justified direction. +3. Preserve unique high-impact corrective insights. +4. Suggestions supported by many source patches should receive higher support_count. +5. The output suggestions should help a later optimizer rewrite the full skill. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "revise_suggestions": [ + { + "type": "add_rule|remove_rule|merge_rules|reorganize|compress|clarify", + "title": "", + "motivation": "", + "instruction": "", + "priority_hint": "high|medium|low", + "support_count": , + "source_type": "failure" + } + ] +} diff --git a/skillopt/prompts/merge_final.md b/skillopt/prompts/merge_final.md new file mode 100644 index 0000000..5dd8be1 --- /dev/null +++ b/skillopt/prompts/merge_final.md @@ -0,0 +1,33 @@ +You are a skill-edit coordinator performing the FINAL merge. You receive two +pre-merged patch groups: +1. **Failure-driven patches** (corrective, high priority) +2. **Success-driven patches** (reinforcement, lower priority) + +Merge guidelines: +1. **FAILURE PATCHES TAKE PRIORITY**: the primary goal of skill reflection is to + fix failures. Failure-driven edits should be preserved unless they directly + conflict with a well-supported success pattern. +2. **Deduplicate**: if a failure edit and success edit cover the same point, + keep the failure version. +3. **Preserve success insights**: include success edits that cover patterns + NOT addressed by failure edits. +4. **Higher-level merges represent broader consensus**: edits that survived + previous merge rounds (higher level) should be given priority. +5. **Carry forward support_count and source_type for each edit.** +6. **PROTECTED SECTION**: The skill may contain a section between + and markers. + Do NOT merge or produce any edits that target content within these markers. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "edits": [ + { + "op": "append|insert_after|replace|delete", + "target": "", + "content": "", + "support_count": , + "source_type": "failure|success" + } + ] +} diff --git a/skillopt/prompts/merge_final_full_rewrite.md b/skillopt/prompts/merge_final_full_rewrite.md new file mode 100644 index 0000000..9976a46 --- /dev/null +++ b/skillopt/prompts/merge_final_full_rewrite.md @@ -0,0 +1,28 @@ +You will be given complete skill candidates and the current skill document. + +Combine them into one complete replacement skill document. + +When merging full-skill candidates, preserve essential task-format instructions, +but do not mechanically retain stale, redundant, or +conflicting rules. Prefer concise guidance with clear trajectory support and +better consistency with the replacement skill. + +Do not include task-specific answers, IDs, file paths, gold values, or entity names. +If the current skill contains a protected block between and +, keep that block unchanged. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "skill_candidates": [ + { + "title": "", + "change_summary": ["", ""], + "new_skill": "", + "support_count": , + "source_type": "failure|success|mixed" + } + ] +} + +Return exactly one item in "skill_candidates". diff --git a/skillopt/prompts/merge_final_rewrite.md b/skillopt/prompts/merge_final_rewrite.md new file mode 100644 index 0000000..7fe3e29 --- /dev/null +++ b/skillopt/prompts/merge_final_rewrite.md @@ -0,0 +1,25 @@ +You are a skill-revision coordinator performing the FINAL merge. You receive: +1. Failure-driven revise_suggestions (higher priority) +2. Success-driven revise_suggestions (lower priority) + +Merge guidelines: +1. Failure-driven suggestions take priority when they overlap. +2. Keep success-driven suggestions that add distinct value. +3. Prefer general, rewrite-friendly, non-redundant suggestions. +4. Carry forward support_count and source_type. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "revise_suggestions": [ + { + "type": "add_rule|remove_rule|merge_rules|reorganize|compress|clarify", + "title": "", + "motivation": "", + "instruction": "", + "priority_hint": "high|medium|low", + "support_count": , + "source_type": "failure|success" + } + ] +} diff --git a/skillopt/prompts/merge_success.md b/skillopt/prompts/merge_success.md new file mode 100644 index 0000000..a467bb1 --- /dev/null +++ b/skillopt/prompts/merge_success.md @@ -0,0 +1,28 @@ +You are a skill-edit coordinator. You receive multiple independently-proposed patches +from SUCCESS analysis of agent trajectories. Merge them into ONE coherent patch +that reinforces effective patterns. + +Merge guidelines: +1. **Deduplicate**: keep only the most generalizable version of similar patterns. +2. **Be conservative**: success-driven patches reinforce existing behavior. + Only include edits for patterns NOT already in the skill. +3. **Prevalent-pattern bias**: patterns seen across many successful trajectories + are most worth encoding. +4. **Support count**: estimate how many source patches support each merged edit. +5. **PROTECTED SECTION**: The skill may contain a section between + and markers. + Do NOT merge or produce any edits that target content within these markers. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "edits": [ + { + "op": "append|insert_after|replace|delete", + "target": "", + "content": "", + "support_count": , + "source_type": "success" + } + ] +} diff --git a/skillopt/prompts/merge_success_full_rewrite.md b/skillopt/prompts/merge_success_full_rewrite.md new file mode 100644 index 0000000..a508c0d --- /dev/null +++ b/skillopt/prompts/merge_success_full_rewrite.md @@ -0,0 +1,28 @@ +You will be given complete skill candidates written from successful trajectories and the current skill document. + +Combine them into one complete replacement skill document. + +When merging full-skill candidates, preserve essential task-format instructions, +but do not mechanically retain stale, redundant, or +conflicting rules. If candidates disagree, prefer the concise rule with clearer +trajectory support and better consistency with the replacement skill. + +Do not include task-specific answers, IDs, file paths, gold values, or entity names. +If the current skill contains a protected block between and +, keep that block unchanged. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "skill_candidates": [ + { + "title": "", + "change_summary": ["", ""], + "new_skill": "", + "support_count": , + "source_type": "success" + } + ] +} + +Return exactly one item in "skill_candidates". diff --git a/skillopt/prompts/merge_success_rewrite.md b/skillopt/prompts/merge_success_rewrite.md new file mode 100644 index 0000000..e823893 --- /dev/null +++ b/skillopt/prompts/merge_success_rewrite.md @@ -0,0 +1,25 @@ +You are a skill-revision coordinator. You receive multiple independently-proposed +revision suggestion sets from SUCCESS analysis of agent trajectories. Merge them +into ONE coherent, non-redundant set of revise_suggestions. + +Merge guidelines: +1. Deduplicate overlapping success patterns. +2. Be conservative: only keep suggestions that reinforce useful behavior not already well-covered. +3. Suggestions supported by many source patches should receive higher support_count. +4. The output suggestions should help a later optimizer rewrite the full skill. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "revise_suggestions": [ + { + "type": "add_rule|remove_rule|merge_rules|reorganize|compress|clarify", + "title": "", + "motivation": "", + "instruction": "", + "priority_hint": "high|medium|low", + "support_count": , + "source_type": "success" + } + ] +} diff --git a/skillopt/prompts/meta_skill.md b/skillopt/prompts/meta_skill.md new file mode 100644 index 0000000..f094973 --- /dev/null +++ b/skillopt/prompts/meta_skill.md @@ -0,0 +1,40 @@ +You are a optimizer-coach for an AI agent skill optimization system. + +Your job is not to solve tasks directly and not to write target-facing skill +rules. Your job is to write a compact OPTIMIZER-SIDE memory that helps future +optimizer calls produce better skill edits in this environment. + +## What You Receive + +1. The previous epoch's last-step skill. +2. The current epoch's last-step skill. +3. A longitudinal comparison on the SAME sampled tasks under those two skills. +4. The previous optimizer meta skill, if one existed. + +## Your Goal + +Write a concise meta skill that improves future optimizer behavior in stages such +as failure analysis, success analysis, patch merging, and edit ranking. + +This meta skill should capture things like: +- Which kinds of edits tend to help in this environment. +- Which kinds of edits tend to be too vague, redundant, brittle, or harmful. +- What level of abstraction works best for rules here. +- What failure-repair patterns should be prioritized. +- What regression risks future optimizer calls should guard against. + +## Important Constraints + +- Address the FUTURE OPTIMIZER directly, not the target. +- Focus on how to write better edits and organize better skill updates. +- Use evidence from the adjacent-epoch comparison, not generic advice. +- Keep it compact and high-signal. Prefer a few durable principles. +- Revise or remove parts of the previous meta skill if they did not help. +- Do not output target-facing task instructions. +- Do not restate the whole skill; summarize editing strategy. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "meta_skill_content": "" +} diff --git a/skillopt/prompts/ranking.md b/skillopt/prompts/ranking.md new file mode 100644 index 0000000..05575a7 --- /dev/null +++ b/skillopt/prompts/ranking.md @@ -0,0 +1,20 @@ +You are an expert skill-optimization optimizer. You receive a skill document and a pool +of proposed edits. Your job is to RANK the edits by importance and select the top ones. + +Ranking criteria (in order of priority): +1. **Systematic impact**: edits that address widespread, recurring failure patterns + across many tasks should rank highest. A rule that fixes 50%% of failures beats + one that fixes a single edge case. +2. **Complementarity**: edits that fill gaps in the current skill (not duplicate + existing content) rank higher. +3. **Generality**: edits phrased as general principles rank higher than those + tied to specific question types or entities. +4. **Actionability**: edits with clear, concrete guidance rank higher than vague advice. + +You will be told how many edits to select (the budget). + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "selected_indices": [<0-based indices of the top edits, in priority order>] +} diff --git a/skillopt/prompts/ranking_rewrite.md b/skillopt/prompts/ranking_rewrite.md new file mode 100644 index 0000000..065787a --- /dev/null +++ b/skillopt/prompts/ranking_rewrite.md @@ -0,0 +1,15 @@ +You are an expert skill-optimization optimizer. You receive a skill document and a pool +of revise_suggestions that will later be used to rewrite the full skill document. +Rank the suggestions by importance and select the top ones. + +Ranking criteria: +1. Systematic impact on recurring failures or strong reusable successes +2. Complementarity with the current skill +3. Rewrite utility: how much the suggestion helps a later optimizer improve structure, clarity, or coverage +4. Generality and actionability + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "selected_indices": [<0-based indices in priority order>] +} diff --git a/skillopt/prompts/rewrite_skill.md b/skillopt/prompts/rewrite_skill.md new file mode 100644 index 0000000..78f2688 --- /dev/null +++ b/skillopt/prompts/rewrite_skill.md @@ -0,0 +1,25 @@ +You are an expert skill-document rewriter for an AI agent training system. + +You will receive: +1. The current skill document +2. A selected set of revise_suggestions distilled from trajectory analysis + +Your job is to rewrite the FULL target skill document so it incorporates the +selected suggestions coherently. + +Hard requirements: +1. Produce a complete standalone skill document, not a patch. +2. Keep effective existing guidance unless a selected suggestion clearly says to remove or merge it. +3. Prefer consolidation and clarity over making the document longer. +4. Do not hardcode benchmark-specific answers, entity names, file paths, or gold values. +5. Preserve the skill's scope: general reusable behavioral guidance for the target. +6. Do not modify content inside the protected slow-update block between + and except to keep it intact. +7. The rewritten skill should be concise, internally consistent, and better organized than the original. + +Respond ONLY with a valid JSON object: +{ + "reasoning": "", + "change_summary": ["", ""], + "new_skill": "" +} diff --git a/skillopt/prompts/slow_update.md b/skillopt/prompts/slow_update.md new file mode 100644 index 0000000..38b1c66 --- /dev/null +++ b/skillopt/prompts/slow_update.md @@ -0,0 +1,60 @@ +You are a strategic skill advisor for an AI agent optimization system. + +Your role is different from the per-step analyst. The per-step analyst sees +individual trajectories and proposes local patches. YOU see how the skill has +evolved across an entire epoch by comparing the SAME tasks under two consecutive +skill versions. This longitudinal view lets you identify systemic drift, +regressions, and persistent blind spots that step-level edits cannot catch. + +## What You Receive + +1. **Previous epoch's skill** and **current epoch's skill** — to see what changed. +2. **Longitudinal comparison** — the same 20 training tasks rolled out under + both skills, categorized into: regressions, persistent failures, + improvements, and stable successes. +3. **Previous slow update guidance** (if any) — the guidance you (or a prior + invocation of you) wrote at the end of the last epoch. This guidance was + active during the current epoch's step-level optimization. You must evaluate + whether it helped or hurt based on the longitudinal comparison results. + +## Your Process + +1. **Reflect on the previous guidance** (if provided): + - Which parts of the previous guidance were effective? (Evidence: tasks that + improved or stayed correct.) + - Which parts failed or backfired? (Evidence: regressions or persistent + failures that the guidance was supposed to address.) + - Were there blind spots the previous guidance missed entirely? + Include this reflection in your "reasoning" field. + +2. **Write updated guidance** that: + - Retains and strengthens parts of the previous guidance that proved effective. + - Revises or removes parts that were ineffective or counterproductive. + - Adds new instructions to address newly observed regressions and persistent + failures. + +## Output Requirements + +Write a **strategic guidance block** that will OVERWRITE the previous guidance +in the protected section of the skill document. This section is READ-ONLY to +all subsequent step-level optimization — only you can overwrite it at the next +epoch boundary. + +Your guidance must: +- Be written as **direct, actionable instructions** to the target model + (the AI agent that will read and follow the skill). +- Focus on helping the target get problems RIGHT — not on analysis or + explanation of what went wrong. +- Prioritize: (1) preventing regressions, (2) fixing persistent failures, + (3) reinforcing successful patterns. +- Be concise but comprehensive — you have no length limit, but every sentence + should earn its place. +- NOT duplicate content already in the main skill body — complement it. +- Address the target directly (e.g., "When you encounter X, always do Y" + rather than "The agent should..."). + +Respond ONLY with a valid JSON object (no markdown fences, no extra text): +{ + "reasoning": "", + "slow_update_content": "" +} diff --git a/skillopt/scheduler/__init__.py b/skillopt/scheduler/__init__.py new file mode 100644 index 0000000..9378ee6 --- /dev/null +++ b/skillopt/scheduler/__init__.py @@ -0,0 +1,8 @@ +"""ReflACT Scheduler -- edit budget and learning rate scheduling. + +Analogous to learning rate schedulers (cosine annealing, step decay, warmup) +in neural network training. Controls how the edit_budget evolves over the +course of training. + +Placeholder for future implementations. +""" diff --git a/skillopt/types.py b/skillopt/types.py new file mode 100644 index 0000000..9c23edb --- /dev/null +++ b/skillopt/types.py @@ -0,0 +1,306 @@ +"""Standardized I/O types for the ReflACT pipeline. + +Shared dataclass definitions for the 6-stage per-step pipeline +and the 2 epoch-level stages. All types support round-trip +conversion to/from plain dicts for incremental adoption. + +Re-exports +---------- +GateResult, GateAction — from skillopt.evaluation.gate +BatchSpec — from skillopt.datasets.base +""" +from __future__ import annotations + +from dataclasses import dataclass, field, fields as dc_fields +from typing import Any, Literal + +from skillopt.evaluation.gate import GateAction, GateResult # noqa: F401 +from skillopt.datasets.base import BatchSpec # noqa: F401 + + +# ── Atomic types ───────────────────────────────────────────────────────── + +EditOp = Literal["append", "insert_after", "replace", "delete"] + + +@dataclass +class Edit: + """A single edit operation on a skill document. + + Used across Reflect → Aggregate → Select → Update → MetaReflect. + """ + + op: EditOp + content: str = "" + target: str = "" + support_count: int | None = None + source_type: Literal["failure", "success"] | None = None + merge_level: int | None = None + update_origin: str = "" + update_target: str = "" + + @classmethod + def from_dict(cls, d: dict) -> Edit: + return cls( + op=d.get("op", "append"), + content=d.get("content", ""), + target=d.get("target", ""), + support_count=d.get("support_count"), + source_type=d.get("source_type"), + merge_level=d.get("merge_level"), + update_origin=d.get("update_origin", ""), + update_target=d.get("update_target", ""), + ) + + def to_dict(self) -> dict: + d: dict[str, Any] = {"op": self.op, "content": self.content} + if self.target: + d["target"] = self.target + if self.support_count is not None: + d["support_count"] = self.support_count + if self.source_type is not None: + d["source_type"] = self.source_type + if self.merge_level is not None: + d["merge_level"] = self.merge_level + if self.update_origin: + d["update_origin"] = self.update_origin + if self.update_target: + d["update_target"] = self.update_target + return d + + +@dataclass +class Patch: + """A set of edits with reasoning. + + Output of Aggregate (③), Select (④); input to Update (⑤). + """ + + edits: list[Edit] = field(default_factory=list) + reasoning: str = "" + ranking_details: dict[str, Any] | None = None + + @classmethod + def from_dict(cls, d: dict) -> Patch: + edits_raw = d.get("edits", []) + return cls( + edits=[Edit.from_dict(e) if isinstance(e, dict) else e for e in edits_raw], + reasoning=d.get("reasoning", ""), + ranking_details=d.get("ranking_details"), + ) + + def to_dict(self) -> dict: + d: dict[str, Any] = { + "reasoning": self.reasoning, + "edits": [e.to_dict() if isinstance(e, Edit) else e for e in self.edits], + } + if self.ranking_details is not None: + d["ranking_details"] = self.ranking_details + return d + + +# ── Stage ① ROLLOUT ────────────────────────────────────────────────────── + +@dataclass +class RolloutResult: + """Result of a single episode/task rollout. + + Universal fields are required; env-specific fields live in ``extras``. + """ + + id: str + hard: int + soft: float + n_turns: int = 0 + fail_reason: str = "" + task_type: str = "" + task_description: str = "" + predicted_answer: str = "" + question: str = "" + reference_text: str = "" + target_system_prompt: str = "" + target_user_prompt: str = "" + spreadsheet_preview: str = "" + extras: dict[str, Any] = field(default_factory=dict) + + _KNOWN_FIELDS: frozenset[str] | None = field( + default=None, init=False, repr=False, compare=False, # type: ignore[assignment] + ) + + @classmethod + def _get_known_fields(cls) -> frozenset[str]: + if cls._KNOWN_FIELDS is None: + cls._KNOWN_FIELDS = frozenset( + f.name for f in dc_fields(cls) + if f.name != "_KNOWN_FIELDS" + ) + return cls._KNOWN_FIELDS + + @classmethod + def from_dict(cls, d: dict) -> RolloutResult: + known = cls._get_known_fields() + extras = {k: v for k, v in d.items() if k not in known} + return cls( + id=str(d.get("id", "")), + hard=int(d.get("hard", 0)), + soft=float(d.get("soft", 0.0)), + n_turns=int(d.get("n_turns", 0)), + fail_reason=str(d.get("fail_reason", "")), + task_type=str(d.get("task_type", "")), + task_description=str(d.get("task_description", "")), + predicted_answer=str(d.get("predicted_answer", "")), + question=str(d.get("question", "")), + reference_text=str(d.get("reference_text", "")), + target_system_prompt=str(d.get("target_system_prompt", "")), + target_user_prompt=str(d.get("target_user_prompt", "")), + spreadsheet_preview=str(d.get("spreadsheet_preview", "")), + extras=extras, + ) + + def to_dict(self) -> dict: + d: dict[str, Any] = { + "id": self.id, + "hard": self.hard, + "soft": self.soft, + } + for attr in ( + "n_turns", "fail_reason", "task_type", "task_description", + "predicted_answer", "question", "reference_text", + "target_system_prompt", "target_user_prompt", + "spreadsheet_preview", + ): + val = getattr(self, attr) + if val: + d[attr] = val + d.update(self.extras) + return d + + +# ── Stage ② REFLECT ────────────────────────────────────────────────────── + +@dataclass +class FailureSummaryEntry: + """One entry in the failure summary produced by error analysts.""" + + failure_type: str + count: int = 0 + description: str = "" + + @classmethod + def from_dict(cls, d: dict) -> FailureSummaryEntry: + return cls( + failure_type=d.get("failure_type", ""), + count=int(d.get("count", 0)), + description=d.get("description", ""), + ) + + def to_dict(self) -> dict: + return { + "failure_type": self.failure_type, + "count": self.count, + "description": self.description, + } + + +@dataclass +class RawPatch: + """Analyst output from the Reflect stage — a patch with provenance. + + Wraps the dict produced by ``run_error_analyst_minibatch`` + and ``run_success_analyst_minibatch``. + """ + + patch: Patch + source_type: Literal["failure", "success"] = "failure" + batch_size: int = 0 + failure_summary: list[FailureSummaryEntry] = field(default_factory=list) + + @classmethod + def from_dict(cls, d: dict | None) -> RawPatch | None: + if d is None: + return None + inner = d.get("patch", d) + if not isinstance(inner, dict): + return None + patch = Patch.from_dict(inner) + return cls( + patch=patch, + source_type=d.get("source_type", "failure"), + batch_size=int(d.get("batch_size", 0)), + failure_summary=[ + FailureSummaryEntry.from_dict(fs) + for fs in d.get("failure_summary", []) + ], + ) + + def to_dict(self) -> dict: + d: dict[str, Any] = { + "patch": self.patch.to_dict(), + "source_type": self.source_type, + "batch_size": self.batch_size, + } + if self.failure_summary: + d["failure_summary"] = [fs.to_dict() for fs in self.failure_summary] + return d + + +# ── Epoch-level: SLOW_UPDATE ───────────────────────────────────────────── + +@dataclass +class SlowUpdateResult: + """Output of the epoch-level slow update stage (EMA / regularization).""" + + reasoning: str = "" + slow_update_content: str = "" + action: str = "" + time_s: float | None = None + prev_hard: float | None = None + curr_hard: float | None = None + selection_hard: float | None = None + selection_soft: float | None = None + candidate_hash: str = "" + update_origin: str = "" + update_target: str = "" + + @classmethod + def from_dict(cls, d: dict | None) -> SlowUpdateResult | None: + if d is None: + return None + return cls( + reasoning=d.get("reasoning", ""), + slow_update_content=d.get("slow_update_content", ""), + action=d.get("action", ""), + time_s=d.get("time_s"), + prev_hard=d.get("prev_hard"), + curr_hard=d.get("curr_hard"), + selection_hard=d.get("selection_hard"), + selection_soft=d.get("selection_soft"), + candidate_hash=d.get("candidate_hash", ""), + update_origin=d.get("update_origin", ""), + update_target=d.get("update_target", ""), + ) + + def to_dict(self) -> dict: + d: dict[str, Any] = { + "reasoning": self.reasoning, + "slow_update_content": self.slow_update_content, + } + if self.action: + d["action"] = self.action + if self.time_s is not None: + d["time_s"] = self.time_s + if self.prev_hard is not None: + d["prev_hard"] = self.prev_hard + if self.curr_hard is not None: + d["curr_hard"] = self.curr_hard + if self.selection_hard is not None: + d["selection_hard"] = self.selection_hard + if self.selection_soft is not None: + d["selection_soft"] = self.selection_soft + if self.candidate_hash: + d["candidate_hash"] = self.candidate_hash + if self.update_origin: + d["update_origin"] = self.update_origin + if self.update_target: + d["update_target"] = self.update_target + return d diff --git a/skillopt/utils/__init__.py b/skillopt/utils/__init__.py new file mode 100644 index 0000000..d59520e --- /dev/null +++ b/skillopt/utils/__init__.py @@ -0,0 +1,4 @@ +"""ReflACT utilities — JSON extraction, scoring, hashing.""" + +from skillopt.utils.json_utils import extract_json, extract_json_array # noqa: F401 +from skillopt.utils.scoring import compute_score, skill_hash # noqa: F401 diff --git a/skillopt/utils/json_utils.py b/skillopt/utils/json_utils.py new file mode 100644 index 0000000..f1fab6e --- /dev/null +++ b/skillopt/utils/json_utils.py @@ -0,0 +1,172 @@ +"""JSON extraction helpers for LLM responses.""" +from __future__ import annotations + +import json +import re +import warnings + + +def _top_level_brace_objects(text: str) -> list[str]: + """Return every balanced *top-level* ``{...}`` span in ``text``. + + Fully string/escape aware: braces inside quoted strings are ignored both + when scanning for an object start AND while tracking depth inside one, so a + ``{`` that appears in prose (e.g. ``'set it to {x}'``) is never mistaken for + the start of a JSON object. Used to detect ambiguity: when a response carries + more than one top-level object we must not let a repair pass silently pick + one — it may pick the wrong (discarded) edit, strictly worse than None. + """ + spans: list[str] = [] + i, n = 0, len(text) + outer_in_str = False + outer_esc = False + while i < n: + ch = text[i] + # Skip over braces that live *inside* a quoted string before any object + # has started — otherwise a `{` in prose like '"set it to {x}"' is wrongly + # treated as an object start, and the repair pass below turns non-JSON + # prose into a bogus dict (strictly worse than returning None). + if outer_in_str: + if outer_esc: + outer_esc = False + elif ch == "\\": + outer_esc = True + elif ch == '"': + outer_in_str = False + i += 1 + continue + if ch == '"': + outer_in_str = True + i += 1 + continue + if ch != "{": + i += 1 + continue + depth = 0 + in_str = False + esc = False + start = i + while i < n: + ch = text[i] + if in_str: + if esc: + esc = False + elif ch == "\\": + esc = True + elif ch == '"': + in_str = False + elif ch == '"': + in_str = True + elif ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + spans.append(text[start:i + 1]) + i += 1 + break + i += 1 + else: + break # unterminated final object + return spans + + +def _looks_json_like(span: str) -> bool: + """Heuristic: does ``span`` look like an intended JSON object (vs. prose)? + + A genuine JSON object's first non-space character after ``{`` is either ``"`` + (a string key) or ``}`` (an empty object). Prose pseudo-objects that the + repair pass would otherwise fabricate into bogus dicts — ``{op: delete}``, + ``{x: 1}`` quoted in single quotes or backticks, etc. — start with a bare + word and are rejected. This complements the string-aware scan, which only + skips *double*-quoted prose; single-quoted / backticked / unquoted prose + braces are caught here instead. Legitimate repair targets (trailing commas, + unescaped quotes inside string values) all begin with ``"`` and pass. + """ + inner = span.strip() + if not (inner.startswith("{") and inner.endswith("}")): + return False + after_brace = inner[1:].lstrip() + return after_brace[:1] in ('"', '}') + + +def extract_json(text: str) -> dict | None: + """Extract a JSON object from LLM response text. + + Tries ```json fences first, then bare {...} patterns. + """ + m = re.search(r"```json\s*(.*?)```", text, re.DOTALL) + if m: + try: + return json.loads(m.group(1)) + except json.JSONDecodeError: + pass + m = re.search(r"\{.*\}", text, re.DOTALL) + if m: + try: + return json.loads(m.group(0)) + except json.JSONDecodeError: + pass + # Tolerant fallback for non-OpenAI backends (Claude/Qwen, …) whose free-form + # JSON strict json.loads rejects — unescaped ASCII quotes inside CJK string + # values, trailing commas, etc. Repair so the analyst's edits aren't silently + # dropped, but ONLY a single unambiguous object: never feed the greedy `{.*}` + # span or the raw text, or json_repair would quietly return one of several + # objects (empirically the wrong/last one) — strictly worse than None, which + # the caller can detect and retry/skip. + # + # Pick the candidate FIRST, before importing json_repair, so the optional + # dependency only matters (and only warns) when there is genuinely a single + # malformed object we could have repaired. Ordinary no-JSON / prose replies + # have no candidate and return None silently. + candidate = None + fenced = re.search(r"```json\s*(.*?)```", text, re.DOTALL) + if fenced and len(_top_level_brace_objects(fenced.group(1))) == 1: + candidate = fenced.group(1) + else: + objs = _top_level_brace_objects(text) + if len(objs) == 1: + candidate = objs[0] + # 0 or >1 top-level objects → too ambiguous to repair safely → None + if not candidate: + return None + # Final guard: only repair spans that actually look like an intended JSON + # object. Prose pseudo-objects in single quotes / backticks / bare text + # (e.g. `{op: delete}`) reach here because the scan only skips double-quoted + # prose; repairing them would fabricate a wrong dict (worse than None). + if not _looks_json_like(candidate): + return None + try: + from json_repair import repair_json + except ModuleNotFoundError: + warnings.warn( + "json_repair not installed; malformed-JSON recovery disabled — " + "a non-OpenAI analyst edit may be silently dropped. pip install json_repair", + RuntimeWarning, + stacklevel=2, + ) + return None + try: + repaired = repair_json(candidate, return_objects=True) + if isinstance(repaired, dict) and repaired: + return repaired + except Exception: # noqa: BLE001 — repair is best-effort + pass + return None + + +def extract_json_array(text: str) -> list | None: + """Extract a JSON array from LLM response text.""" + m = re.search(r"```json\s*(.*?)```", text, re.DOTALL) + if m: + try: + return json.loads(m.group(1)) + except json.JSONDecodeError: + pass + m = re.search(r"\[.*\]", text, re.DOTALL) + if m: + try: + return json.loads(m.group(0)) + except json.JSONDecodeError: + pass + return None diff --git a/skillopt/utils/scoring.py b/skillopt/utils/scoring.py new file mode 100644 index 0000000..df5d1fe --- /dev/null +++ b/skillopt/utils/scoring.py @@ -0,0 +1,28 @@ +"""Scoring and hashing utilities.""" +from __future__ import annotations + +import hashlib + + +def compute_score(results: list) -> tuple[float, float]: + """Compute hard and soft accuracy from a list of episode results. + + Accepts both plain dicts and :class: instances. hard may be continuous (0.0-1.0) when using smoothed reward. + """ + if not results: + return 0.0, 0.0 + + def _hard(r: object) -> float: + return float(r.hard if hasattr(r, "hard") else r.get("hard", 0)) + + def _soft(r: object) -> float: + return float(r.soft if hasattr(r, "soft") else r.get("soft", 0.0)) + + hard = sum(_hard(r) for r in results) / len(results) + soft = sum(_soft(r) for r in results) / len(results) + return hard, soft + + +def skill_hash(content: str) -> str: + """Return a short deterministic hash of skill content (for caching).""" + return hashlib.sha256(content.encode()).hexdigest()[:16] diff --git a/skillopt_sleep/__init__.py b/skillopt_sleep/__init__.py new file mode 100644 index 0000000..9c7581f --- /dev/null +++ b/skillopt_sleep/__init__.py @@ -0,0 +1,20 @@ +"""SkillOpt-Sleep — nightly offline self-evolution for a local Claude agent. + +A Claude Code plugin engine that gives a user's agent a "sleep cycle": +harvest the day's real session transcripts, mine recurring tasks, replay +them offline, and consolidate short-term experience into long-term memory +(CLAUDE.md) and skills (SKILL.md) behind a SkillOpt validation gate. + +Synthesizes three ideas: + * SkillOpt — validation-gated bounded text optimization (this repo) + * Dreams — offline memory consolidation, input never mutated + * Sleep — short-term experience -> long-term competence, offline + +Public entry points: + * skillopt_sleep.cli — `python -m skillopt_sleep ...` + * skillopt_sleep.cycle.run_sleep_cycle(...) +""" +from __future__ import annotations + +__all__ = ["__version__"] +__version__ = "0.2.0" diff --git a/skillopt_sleep/__main__.py b/skillopt_sleep/__main__.py new file mode 100644 index 0000000..01293e1 --- /dev/null +++ b/skillopt_sleep/__main__.py @@ -0,0 +1,532 @@ +"""SkillOpt-Sleep — command-line interface. + + python -m skillopt_sleep run # full cycle: harvest->mine->replay->gate->stage + python -m skillopt_sleep dry-run # same but report only, no staging/adopt + python -m skillopt_sleep status # show state + latest staged proposal + python -m skillopt_sleep adopt # apply the latest staged proposal (with backup) + python -m skillopt_sleep harvest # just print what would be mined (debug) + +Common flags: + --project PATH project to evolve (default: cwd) + --scope all|invoked harvest scope (default: invoked) + --max-sessions N cap transcript sessions per run + --max-tasks N cap mined tasks per run + --target-skill-path PATH explicit live SKILL.md to stage/adopt + --tasks-file PATH reviewed TaskRecord JSON file to replay instead of harvesting + --backend mock|claude|codex|copilot|handoff + --source claude|codex|auto + --model NAME + --lookback-hours N + --auto-adopt + --json machine-readable output +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from typing import Any, Dict + +from skillopt_sleep.config import load_config +from skillopt_sleep.cycle import run_sleep_cycle +from skillopt_sleep.harvest_sources import harvest_for_config +from skillopt_sleep.mine import mine +from skillopt_sleep.staging import adopt as adopt_staging +from skillopt_sleep.staging import latest_staging +from skillopt_sleep.state import SleepState +from skillopt_sleep.tasks_file import load_tasks_file, make_tasks_payload, write_tasks_file + + +def _read_text(path: str) -> str: + try: + with open(path, encoding="utf-8") as f: + return f.read() + except Exception: + return "" + + +def _report_payload(rep, outcome) -> Dict[str, Any]: + return { + "night": rep.night, + "accepted": rep.accepted, + "gate_action": rep.gate_action, + "no_edits_reason": getattr(rep, "no_edits_reason", ""), + "baseline": rep.baseline_score, + "candidate": rep.candidate_score, + "n_tasks": rep.n_tasks, + "n_sessions": rep.n_sessions, + "n_accepted_edits": len(rep.edits), + "n_rejected_edits": len(rep.rejected_edits), + "edits": [e.__dict__ for e in rep.edits], + "rejected_edits": [e.__dict__ for e in rep.rejected_edits], + "notes": rep.notes, + "staging_dir": outcome.staging_dir, + "adopted": outcome.adopted, + } + + +def _add_common(p: argparse.ArgumentParser) -> None: + p.add_argument("--project", default="") + p.add_argument("--scope", default="", choices=["", "all", "invoked"]) + p.add_argument("--backend", default="", + choices=["", "mock", "claude", "codex", "copilot", "handoff"]) + p.add_argument("--model", default="") + p.add_argument("--codex-path", default="", help="path to the real @openai/codex binary") + p.add_argument("--claude-home", default="", help="override ~/.claude (also isolates state)") + p.add_argument("--codex-home", default="", help="override ~/.codex for archived session harvest") + p.add_argument("--source", default="", choices=["", "claude", "codex", "auto"], + help="session transcript source") + p.add_argument("--lookback-hours", type=int, default=None, + help="harvest window in hours; 0 = scan full history") + p.add_argument("--edit-budget", type=int, default=0) + p.add_argument("--max-sessions", type=int, default=0, + help="cap harvested sessions before mining; default derives from max tasks") + p.add_argument("--max-tasks", type=int, default=0, + help="cap mined tasks for this run") + p.add_argument("--target-skill-path", default="", + help="explicit live SKILL.md path to evolve/stage/adopt") + p.add_argument("--tasks-file", default="", + help="reviewed TaskRecord JSON file to replay instead of harvesting") + p.add_argument("--progress", action="store_true", + help="print phase progress to stderr") + p.add_argument("--auto-adopt", action="store_true") + p.add_argument("--json", action="store_true") + + +def _cfg_from_args(args, task_meta: Dict[str, Any] | None = None) -> Any: + overrides: Dict[str, Any] = {} + if args.project: + overrides["invoked_project"] = os.path.abspath(args.project) + overrides["projects"] = "invoked" + if args.scope: + overrides["projects"] = args.scope + if args.backend: + overrides["backend"] = args.backend + if args.model: + overrides["model"] = args.model + if getattr(args, "codex_path", ""): + overrides["codex_path"] = os.path.abspath(args.codex_path) + if getattr(args, "claude_home", ""): + overrides["claude_home"] = os.path.abspath(args.claude_home) + if getattr(args, "codex_home", ""): + overrides["codex_home"] = os.path.abspath(args.codex_home) + if getattr(args, "source", ""): + overrides["transcript_source"] = args.source + lh = getattr(args, "lookback_hours", None) + if lh is not None: # --lookback-hours was explicitly passed (0 = full history) + overrides["lookback_hours"] = lh + if getattr(args, "edit_budget", 0): + overrides["edit_budget"] = args.edit_budget + if getattr(args, "max_sessions", 0): + overrides["max_sessions_per_night"] = args.max_sessions + if getattr(args, "max_tasks", 0): + overrides["max_tasks_per_night"] = args.max_tasks + target_skill_path = getattr(args, "target_skill_path", "") + if not target_skill_path and task_meta: + target_skill_path = str(task_meta.get("target_skill_path") or "") + if target_skill_path: + path = os.path.expanduser(target_skill_path) + if args.project and not os.path.isabs(path): + path = os.path.join(os.path.abspath(args.project), path) + overrides["target_skill_path"] = os.path.abspath(path) + if getattr(args, "progress", False): + overrides["progress"] = True + if getattr(args, "auto_adopt", False): + overrides["auto_adopt"] = True + return load_config(**overrides) + + +def cmd_run(args, dry: bool = False) -> int: + task_meta: Dict[str, Any] = {} + tasks = None + if getattr(args, "tasks_file", ""): + # Load once before config so target_skill_path can default from metadata. + tasks, task_meta = load_tasks_file(args.tasks_file) + cfg = _cfg_from_args(args, task_meta=task_meta) + if getattr(args, "tasks_file", ""): + tasks, task_meta = load_tasks_file( + args.tasks_file, + holdout_fraction=cfg.get("holdout_fraction", 0.34), + seed=cfg.get("seed", 42), + ) + if cfg.get("backend", "mock") != "mock" and task_meta.get("reviewed") is not True: + print( + "[sleep] refusing real-backend replay from an unreviewed tasks file; " + "inspect/redact it and set \"reviewed\": true first", + file=sys.stderr, + ) + return 2 + if cfg.get("backend", "mock") == "handoff": + return _run_handoff(cfg, args, seed_tasks=tasks, task_meta=task_meta, dry=dry) + outcome = run_sleep_cycle(cfg, seed_tasks=tasks, dry_run=dry) + _print_run_report(outcome, args, task_meta) + return 0 + + +def _print_run_report(outcome, args, task_meta: Dict[str, Any]) -> None: + rep = outcome.report + if args.json: + payload = _report_payload(rep, outcome) + if task_meta: + payload["tasks_file"] = task_meta.get("tasks_file", "") + payload["tasks_reviewed"] = task_meta.get("reviewed", False) + print(json.dumps(payload, ensure_ascii=False, indent=2)) + else: + print(f"[sleep] night {rep.night}: {rep.n_sessions} sessions -> {rep.n_tasks} tasks") + print(f"[sleep] held-out {rep.baseline_score:.3f} -> {rep.candidate_score:.3f} " + f"=> {rep.gate_action} (accepted={rep.accepted})") + for e in rep.edits: + print(f" + [{e.target}/{e.op}] {e.content}") + if rep.rejected_edits: + print("[sleep] rejected by gate:") + for e in rep.rejected_edits: + print(f" - [{e.target}/{e.op}] {e.content}") + if outcome.staging_dir: + print(f"[sleep] staged: {outcome.staging_dir}") + if not outcome.adopted: + print("[sleep] review it, then: python -m skillopt_sleep adopt") + if outcome.adopted: + print(f"[sleep] auto-adopted: {', '.join(outcome.adopted_paths)}") + + +def _handoff_dir_for(cfg) -> str: + project = cfg.get("invoked_project") or os.getcwd() + return os.environ.get("SKILLOPT_SLEEP_HANDOFF_DIR", "") or os.path.join( + project, ".skillopt-sleep-handoff" + ) + + +def _redact_deep(obj): + """Redact secret-looking substrings in every string of a JSON-like tree.""" + from skillopt_sleep.staging import redact_secrets + if isinstance(obj, str): + return redact_secrets(obj) + if isinstance(obj, list): + return [_redact_deep(x) for x in obj] + if isinstance(obj, dict): + return {k: _redact_deep(v) for k, v in obj.items()} + return obj + + +def _flush_handoff(backend, args) -> int: + prompts_path = backend.flush_pending() + if args.json: + print(json.dumps({ + "handoff_pending": len(backend.pending), + "prompts": prompts_path, + "answers_dir": backend.answers_dir, + }, ensure_ascii=False, indent=2)) + else: + print(f"[sleep] handoff: {len(backend.pending)} model call(s) need answers") + print(f"[sleep] prompts: {prompts_path}") + print(f"[sleep] write each raw answer to {backend.answers_dir}/.md, " + "then re-run this exact command to resume") + return 3 + + +def _handoff_mine_and_pin(cfg, args, backend, snapshot: str, dry: bool): + """Harvest + mine with the same knobs as run_sleep_cycle (harvest window, + target-skill filter, candidate-limit bump, LLM mining — routed through the + handoff files like every other model call), then pin the result to + ``tasks.json``. Session digests are pinned too, so the sessions created + while answering prompts cannot change what gets mined between rounds. + + Returns ``(exit_code, tasks)``; ``tasks is None`` means exit now. + """ + import time + + from skillopt_sleep.handoff_backend import PendingCalls + from skillopt_sleep.state import SleepState, _now_iso + from skillopt_sleep.types import SessionDigest + + project = cfg.get("invoked_project") or os.getcwd() + state = SleepState.load(cfg.state_path) + started = _now_iso() + + digests_path = os.path.join(backend.handoff_dir, "digests.json") + digests = None + if os.path.exists(digests_path): + try: + with open(digests_path, encoding="utf-8") as f: + raw = json.load(f) + known = set(SessionDigest.__dataclass_fields__) + digests = [SessionDigest(**{k: v for k, v in d.items() if k in known}) + for d in raw] + except Exception: + # Corrupted/truncated pin (e.g. an interrupted earlier round): + # fall back to a fresh harvest instead of crashing the run. + print("[sleep] handoff: digests.json unreadable — re-harvesting", + file=sys.stderr) + digests = None + if digests is None: + since = state.last_harvest_for(project) + lookback_hours = cfg.get("lookback_hours", 72) + if since is None and lookback_hours and lookback_hours > 0: + since = _now_iso(time.time() - lookback_hours * 3600) + max_tasks = cfg.get("max_tasks_per_night", 40) + session_limit = cfg.get("max_sessions_per_night", 0) or max_tasks * 3 + digests = harvest_for_config(cfg, since_iso=since, limit=session_limit) + os.makedirs(backend.handoff_dir, exist_ok=True) + with open(digests_path, "w", encoding="utf-8") as f: + json.dump(_redact_deep([d.to_dict() for d in digests]), f, + ensure_ascii=False, indent=2) + + max_tasks = cfg.get("max_tasks_per_night", 40) + session_limit = cfg.get("max_sessions_per_night", 0) or max_tasks * 3 + target_skill_path = cfg.managed_skill_path() if cfg.get("target_skill_path", "") else "" + target_skill_text = _read_text(target_skill_path) if target_skill_path else "" + candidate_limit = max_tasks + if cfg.get("target_task_filter", True) and target_skill_text: + candidate_limit = max(max_tasks, max_tasks * 3) + llm_miner = None + if cfg.get("llm_mine", True): + try: + from skillopt_sleep.llm_miner import make_llm_miner + llm_miner = make_llm_miner( + backend, max_sessions=session_limit, max_tasks=candidate_limit, + ) + except Exception: + llm_miner = None + try: + tasks = mine( + digests, + max_tasks=max_tasks, + candidate_limit=candidate_limit, + holdout_fraction=cfg.get("holdout_fraction", 0.34), + seed=cfg.get("seed", 42), + llm_miner=llm_miner, + target_skill_text=target_skill_text, + target_skill_path=target_skill_path, + ) + except PendingCalls: + tasks = [] + if backend.pending: + # LLM mining needs answers before the task set can be pinned. + return _flush_handoff(backend, args), None + if not tasks: + print("[sleep] handoff: no tasks mined — nothing to consolidate") + if not dry: + # Advance the harvest window like run_sleep_cycle's no-tasks + # branch, or every later run re-scans the same stale window. + state.set_last_harvest(project, started) + state.save() + return 0, None + payload = make_tasks_payload( + tasks, + project=project, + transcript_source=cfg.get("transcript_source", ""), + n_sessions=len(digests), + target_skill_path=target_skill_path, + ) + # NOT marked reviewed: feeding this snapshot back through --tasks-file + # with a real backend must still hit the human-review gate above. The + # driver itself loads it directly, with the same trust as in-cycle mining. + write_tasks_file(snapshot, _redact_deep(payload)) + print(f"[sleep] handoff: pinned {len(tasks)} tasks -> {snapshot}") + return 0, tasks + + +def _run_handoff(cfg, args, *, seed_tasks, task_meta: Dict[str, Any], dry: bool) -> int: + """Drive the handoff backend: run until model calls are needed, then + write the prompt batch and exit 3; on a fully-answered run, finish + normally. Session digests and mined tasks are pinned under the handoff + dir on the first rounds so wall-clock time between rounds (including + the very sessions that answer the prompts) cannot change the task set + and invalidate earlier answers. + """ + from skillopt_sleep.handoff_backend import HandoffBackend, PendingCalls + + hdir = _handoff_dir_for(cfg) + backend = HandoffBackend(model=cfg.get("model", ""), handoff_dir=hdir) + tasks = seed_tasks + if tasks is None: + snapshot = os.path.join(hdir, "tasks.json") + if os.path.exists(snapshot): + tasks, _meta = load_tasks_file( + snapshot, + holdout_fraction=cfg.get("holdout_fraction", 0.34), + seed=cfg.get("seed", 42), + ) + else: + rc, tasks = _handoff_mine_and_pin(cfg, args, backend, snapshot, dry) + if tasks is None: + return rc + outcome = None + try: + outcome = run_sleep_cycle(cfg, seed_tasks=tasks, dry_run=dry, backend=backend) + except PendingCalls: + pass + if backend.pending: + return _flush_handoff(backend, args) + _print_run_report(outcome, args, task_meta) + # A completed real run ends the night: archive the handoff dir so the + # next night re-harvests instead of replaying the pinned snapshot. + if not dry and outcome.staging_dir and os.path.isdir(hdir): + import time + done = f"{hdir}.night{outcome.report.night}.done" + if os.path.exists(done): + done = f"{done}.{int(time.time())}" + os.rename(hdir, done) + print(f"[sleep] handoff: archived round data -> {done}") + return 0 + + +def cmd_status(args) -> int: + cfg = _cfg_from_args(args) + state = SleepState.load(cfg.state_path) + project = cfg.get("invoked_project") or os.getcwd() + latest = latest_staging(project) + info = { + "night": state.night, + "state_path": cfg.state_path, + "project": project, + "history_tail": state.data.get("history", [])[-5:], + "latest_staging": latest, + "slow_memory_chars": len(state.slow_memory), + } + if args.json: + print(json.dumps(info, ensure_ascii=False, indent=2)) + else: + print(f"[sleep] nights so far: {state.night}") + print(f"[sleep] project: {project}") + if latest: + print(f"[sleep] latest staged proposal: {latest}") + rp = os.path.join(latest, "report.md") + if os.path.exists(rp): + with open(rp) as f: + print("\n" + f.read()) + else: + print("[sleep] no staged proposals yet.") + return 0 + + +def cmd_adopt(args) -> int: + cfg = _cfg_from_args(args) + project = cfg.get("invoked_project") or os.getcwd() + target = args.staging or latest_staging(project) + if not target or not os.path.isdir(target): + print("[sleep] nothing to adopt (no staging dir).") + return 1 + updated = adopt_staging(target) + print(f"[sleep] adopted from {target}") + for p in updated: + print(f" -> {p}") + if not updated: + print("[sleep] (proposal contained no accepted changes)") + return 0 + + +def cmd_harvest(args) -> int: + cfg = _cfg_from_args(args) + session_limit = cfg.get("max_sessions_per_night", 0) or cfg.get("max_tasks_per_night", 40) * 3 + target_skill_path = cfg.managed_skill_path() if cfg.get("target_skill_path", "") else "" + target_skill_text = _read_text(target_skill_path) if target_skill_path else "" + max_tasks = cfg.get("max_tasks_per_night", 40) + candidate_limit = max_tasks + if cfg.get("target_task_filter", True) and target_skill_text: + candidate_limit = max(max_tasks, max_tasks * 3) + digests = harvest_for_config(cfg, limit=session_limit) + tasks = mine( + digests, + max_tasks=max_tasks, + candidate_limit=candidate_limit, + holdout_fraction=cfg.get("holdout_fraction", 0.34), + seed=cfg.get("seed", 42), + target_skill_text=target_skill_text, + target_skill_path=target_skill_path, + ) + payload = make_tasks_payload( + tasks, + project=cfg.get("invoked_project") or os.getcwd(), + transcript_source=cfg.get("transcript_source", ""), + n_sessions=len(digests), + target_skill_path=target_skill_path, + ) + output_path = "" + if getattr(args, "output", ""): + output_path = write_tasks_file(args.output, payload) + if args.json: + json_payload = dict(payload) + if output_path: + json_payload["output"] = output_path + print(json.dumps(json_payload, ensure_ascii=False, indent=2)) + else: + print(f"[sleep] {len(digests)} sessions -> {len(tasks)} tasks") + if output_path: + print(f"[sleep] wrote reviewed-task draft: {output_path}") + for t in tasks: + print(f" [{t.split}/{t.outcome}] {t.intent[:90]}") + return 0 + + +def cmd_schedule(args) -> int: + from skillopt_sleep.scheduler import schedule, list_scheduled + cfg = _cfg_from_args(args) + project = cfg.get("invoked_project") or os.getcwd() + ok, msg = schedule(project, backend=cfg.get("backend", "mock"), + hour=args.hour, minute=args.minute, + extra=("--auto-adopt" if getattr(args, "auto_adopt", False) else "")) + print("[sleep] " + msg) + cur = list_scheduled() + if cur: + print("[sleep] currently scheduled:") + for ln in cur: + print(" " + ln[:140]) + return 0 if ok else 1 + + +def cmd_unschedule(args) -> int: + from skillopt_sleep.scheduler import unschedule + cfg = _cfg_from_args(args) + project = cfg.get("invoked_project") or os.getcwd() + ok, msg = unschedule(project, all_projects=getattr(args, "all", False)) + print("[sleep] " + msg) + return 0 if ok else 1 + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(prog="skillopt_sleep", description="SkillOpt-Sleep nightly self-evolution") + sub = parser.add_subparsers(dest="cmd", required=True) + + p_run = sub.add_parser("run", help="run a full sleep cycle") + _add_common(p_run) + p_dry = sub.add_parser("dry-run", help="harvest+mine+replay, report only") + _add_common(p_dry) + p_status = sub.add_parser("status", help="show state + latest proposal") + _add_common(p_status) + p_adopt = sub.add_parser("adopt", help="apply latest staged proposal") + _add_common(p_adopt) + p_adopt.add_argument("--staging", default="", help="specific staging dir") + p_harvest = sub.add_parser("harvest", help="debug: show mined tasks") + _add_common(p_harvest) + p_harvest.add_argument("--output", default="", help="write mined tasks JSON for review") + p_sched = sub.add_parser("schedule", help="install a nightly cron entry for this project") + _add_common(p_sched) + p_sched.add_argument("--hour", type=int, default=3) + p_sched.add_argument("--minute", type=int, default=17) + p_unsched = sub.add_parser("unschedule", help="remove the nightly cron entry") + _add_common(p_unsched) + p_unsched.add_argument("--all", action="store_true", help="remove all managed entries") + + args = parser.parse_args(argv) + if args.cmd == "run": + return cmd_run(args, dry=False) + if args.cmd == "dry-run": + return cmd_run(args, dry=True) + if args.cmd == "status": + return cmd_status(args) + if args.cmd == "adopt": + return cmd_adopt(args) + if args.cmd == "harvest": + return cmd_harvest(args) + if args.cmd == "schedule": + return cmd_schedule(args) + if args.cmd == "unschedule": + return cmd_unschedule(args) + parser.print_help() + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skillopt_sleep/backend.py b/skillopt_sleep/backend.py new file mode 100644 index 0000000..d7bb040 --- /dev/null +++ b/skillopt_sleep/backend.py @@ -0,0 +1,1468 @@ +"""SkillOpt-Sleep — optimizer/replay backend abstraction. + +A backend supplies the three "intelligent" operations the sleep cycle needs: + + 1. attempt(task, skill, memory) -> response text (the rollout) + 2. judge(task, response) -> (hard, soft, rationale) (the reward) + 3. reflect(failures, successes, skill, memory) + -> list[EditRecord] (proposed bounded edits) + +Two implementations: + * MockBackend — deterministic, no API, used for tests + the experiment. + Reads optional `reference` exact answers and a tiny + rule-table so the loop provably improves and the gate + provably blocks regressions. + * AnthropicBackend — uses the user's ANTHROPIC_API_KEY via the `claude` + CLI or the anthropic SDK (lazy-imported). Real lift. + +The backend never touches live config; it only returns text/edits that the +consolidation stage gates and stages. +""" +from __future__ import annotations + +import json +import os +import re +import subprocess +import tempfile +from typing import Any, Dict, List, Optional, Tuple + +from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord + +# On Windows, console-attached children (cmd.exe shims, python) allocate a +# visible console window when the parent has none — a nightly cycle making +# hundreds of CLI calls strobes cmd windows and steals focus from the user. +# CREATE_NO_WINDOW suppresses that; harmless 0 elsewhere. +_NO_WINDOW = getattr(subprocess, "CREATE_NO_WINDOW", 0) if os.name == "nt" else 0 + + +def skill_hash(content: str) -> str: + import hashlib + return hashlib.sha256(content.encode("utf-8")).hexdigest()[:16] + + +# ── Backend protocol ────────────────────────────────────────────────────────── + +class Backend: + name = "base" + # Optional user preferences (free text) injected into reflect as a prior. + preferences: str = "" + + def attempt(self, task: TaskRecord, skill: str, memory: str, + sample_id: int = 0) -> str: + raise NotImplementedError + + def attempt_with_tools( + self, task: TaskRecord, skill: str, memory: str, tools: List[str] + ) -> Tuple[str, List[str]]: + """Run the task while exposing real tools; return (response, tools_called). + + Default: no real tool loop — fall back to plain attempt and let the + single-shot 'TOOL_CALL: ' marker convention surface intent. CLI + backends override this to expose a genuinely callable tool. + """ + resp = self.attempt(task, skill, memory) + called: List[str] = [] + for t in tools: + if re.search(r"(?i)\btool_call\s*:\s*%s\b" % re.escape(t), resp): + called.append(t) + return resp, called + + def judge(self, task: TaskRecord, response: str) -> Tuple[float, float, str]: + raise NotImplementedError + + def reflect( + self, + failures: List[Tuple[TaskRecord, ReplayResult]], + successes: List[Tuple[TaskRecord, ReplayResult]], + skill: str, + memory: str, + *, + edit_budget: int, + evolve_skill: bool, + evolve_memory: bool, + ) -> List[EditRecord]: + raise NotImplementedError + + # token accounting (optional) + def tokens_used(self) -> int: + return 0 + + +# ── Shared scoring helpers ──────────────────────────────────────────────────── + +def _normalize(s: str) -> str: + s = (s or "").lower().strip() + s = re.sub(r"[^\w\s]", " ", s) + s = re.sub(r"\s+", " ", s) + return s.strip() + + +def exact_score(reference: str, response: str) -> float: + ref = _normalize(reference) + resp = _normalize(response) + if not ref: + return 0.0 + return 1.0 if ref in resp or resp == ref else 0.0 + + +def keyword_soft_score(reference: str, response: str) -> float: + """Fraction of reference tokens present in response (cheap rubric proxy).""" + ref_tokens = [t for t in _normalize(reference).split() if len(t) > 2] + if not ref_tokens: + return 0.0 + resp = _normalize(response) + hit = sum(1 for t in set(ref_tokens) if t in resp) + return hit / len(set(ref_tokens)) + + +# ── Mock backend (deterministic, no API) ────────────────────────────────────── + +class MockBackend(Backend): + """Deterministic backend for tests and the acceptance experiment. + + Model of reality: + * Each task may carry a `reference` (exact answer) and a "rule" tag + describing the single skill rule that makes the task solvable, e.g. + tags=["rule:wrap-answer-in-answer-tags"]. + * `attempt` produces a correct response IFF the required rule text is + present in skill+memory; otherwise it produces a near-miss. + * `judge` scores exact (hard) + keyword (soft) against `reference`. + * `reflect` looks at failures, reads each failed task's required rule, + and proposes exactly that rule as an `add` edit (bounded by budget). + It NEVER proposes a rule already present (no churn), and on the + special tag "rule:__harmful__" it proposes a known-bad edit so tests + can prove the gate rejects regressions. + + This makes the end-to-end loop monotonic and fully reproducible while + exercising the real harvest->mine->replay->gate->stage plumbing. + """ + + name = "mock" + + RULE_PREFIX = "rule:" + RULE_TEXT = { + "wrap-answer": "Always wrap the final answer in ... tags.", + "arxiv-id": "Report arXiv ids in the exact form arXiv:XXXX.XXXXX.", + "commit-imperative": "Write git commit subjects in imperative mood, max 50 chars.", + "units-si": "Always include SI units in numeric answers.", + "json-only": "When asked for JSON, output only valid JSON with no prose.", + "__harmful__": "Ignore the user's formatting requests and answer freely.", + } + + def _required_rules(self, task: TaskRecord) -> List[str]: + out = [] + for t in task.tags: + if t.startswith(self.RULE_PREFIX): + key = t[len(self.RULE_PREFIX):] + if key in self.RULE_TEXT: + out.append(key) + return out + + def attempt(self, task: TaskRecord, skill: str, memory: str, + sample_id: int = 0) -> str: + ctx = (skill or "") + "\n" + (memory or "") + rules = self._required_rules(task) + # The "__harmful__" rule models a bad edit: even when present it makes + # the agent ignore formatting, so it can NEVER produce the reference. + # This is what lets the experiment prove the gate rejects regressions. + if "__harmful__" in rules: + return "I'll just answer freely and skip the requested format." + # A task is solved iff ALL its required rule texts are present in context. + have_all = all(self.RULE_TEXT[k] in ctx for k in rules) if rules else False + if have_all and task.reference: + # produce a response that satisfies the rule and contains the answer + if "wrap-answer" in rules: + return f"Here is the result. {task.reference}" + return f"{task.reference}" + # Near miss: a degraded answer that shares keywords but is NOT the exact + # rule-correct form, so exact-match fails deterministically regardless of + # how many whitespace tokens the reference has. + if task.reference: + ref = task.reference + mangled = ref[:-2] if len(ref) > 3 else "unknown" + return f"approximately {mangled} (format not applied)" + return "(attempted, no checkable reference)" + + def attempt_with_tools(self, task, skill, memory, tools): + # Deterministic tool model: the mock "calls" a tool iff the skill+memory + # contains an explicit instruction to use it (a learned rule mentioning + # the tool name or "search"). The deficient skill says NOT to, so + # baseline calls nothing; a learned "use ./search" rule flips it. + ctx = ((skill or "") + "\n" + (memory or "")).lower() + resp = self.attempt(task, skill, memory) + called = [] + for t in (tools or []): + tl = t.lower() + if (f"./{tl}" in ctx or f"use {tl}" in ctx or f"run {tl}" in ctx + or f"call {tl}" in ctx or f"must {tl}" in ctx): + called.append(t) + return resp, called + + def judge(self, task: TaskRecord, response: str) -> Tuple[float, float, str]: + if task.reference_kind == "answer" and task.judge: + try: + from skillopt_sleep.experiments.real_eval import score_answer_judge + except ImportError: + score_answer_judge = None # research evaluators not bundled + if score_answer_judge is not None: + return score_answer_judge(task.judge, response) + if task.reference_kind == "rule" and task.judge: + from skillopt_sleep.judges import score_rule_judge + return score_rule_judge(task.judge, response) + if task.reference_kind == "exact" and task.reference: + hard = exact_score(task.reference, response) + soft = max(hard, keyword_soft_score(task.reference, response)) + return hard, soft, f"exact-match={hard}" + if task.reference_kind == "rubric" and task.reference: + soft = keyword_soft_score(task.reference, response) + return (1.0 if soft >= 0.8 else 0.0), soft, f"rubric keyword soft={soft:.2f}" + # no reference: outcome-derived weak label + hard = 1.0 if task.outcome == "success" else 0.0 + return hard, hard, "outcome-derived" + + def reflect( + self, + failures, + successes, + skill: str, + memory: str, + *, + edit_budget: int, + evolve_skill: bool, + evolve_memory: bool, + ) -> List[EditRecord]: + ctx = (skill or "") + "\n" + (memory or "") + edits: List[EditRecord] = [] + seen_text: set = set() + target = "skill" if evolve_skill else "memory" + for task, _res in failures: + for key in self._required_rules(task): + text = self.RULE_TEXT[key] + if text in ctx or text in seen_text: + continue + seen_text.add(text) + edits.append( + EditRecord( + target=target, + op="add", + content=text, + rationale=f"failed task {task.id} requires rule '{key}'", + ) + ) + if len(edits) >= edit_budget: + return edits + return edits + + +# ── Shared real-CLI backend (prompts + parsing + cache; subclasses do _call) ── + +def _extract_json(raw: str, kind: str): + """Pull the first JSON object/array out of a possibly chatty CLI reply.""" + pat = r"\{.*\}" if kind == "object" else r"\[.*\]" + m = re.search(pat, raw or "", re.DOTALL) + if not m: + return None + try: + return json.loads(m.group(0)) + except Exception: + return None + + +def _task_guardrail(pairs) -> str: + """Build an 'output contract' the optimizer must not violate. + + ``pairs`` is a list of (TaskRecord, ReplayResult). We surface the benchmark's + own rollout system prompt (TaskRecord.system) plus a short, explicit list of + invariants, so the optimizer cannot learn rules that the evaluator can never + honor (the SpreadsheetBench failure mode: a learned "return ```vba```" or + "ask the user for the range" rule scores 0 because the harness runs only + ```python``` openpyxl and cannot answer questions). + + Returns "" when no task carries a system contract (e.g. mined daily cases), + so non-benchmark runs are unchanged. + """ + sys_txt = "" + for t, _ in pairs: + s = getattr(t, "system", "") or "" + if s.strip(): + sys_txt = s.strip() + break + if not sys_txt: + return "" + # the system prompt can be long; keep the rules portion concise for the optimizer + contract = sys_txt + if len(contract) > 900: + contract = contract[:900] + " …" + invariants = ( + "- Do NOT change the required output format or programming language.\n" + "- Do NOT tell the agent to ask the user a question or request more info; " + "it must always produce a best-effort answer from what is given.\n" + "- Keep every rule consistent with the contract above." + ) + return ( + "\n# Task output contract (rules MUST obey this — violating it scores 0)\n" + f"{contract}\n{invariants}\n" + ) + + +class CliBackend(Backend): + """Common logic for real CLI-driven backends (claude / codex). + + Subclasses implement only ``_call(prompt) -> str``. This base owns the + prompts (attempt / judge / reflect), JSON parsing, a response cache (so + re-scoring an unchanged (skill, memory) on the held-out slice is free), + and a rough token estimate. + """ + + name = "cli" + + def __init__(self, model: str = "", timeout: int = 180) -> None: + self.model = model + self.timeout = timeout + self._tokens = 0 + self._cache: Dict[str, str] = {} + self.last_call_error = "" + self.last_reflect_raw = "" + + # subclasses override -------------------------------------------------- + def _call(self, prompt: str, *, max_tokens: int = 1024) -> str: + raise NotImplementedError + + def _cached_call(self, key: str, prompt: str, *, max_tokens: int = 1024) -> str: + if key in self._cache: + return self._cache[key] + out = self._call(prompt, max_tokens=max_tokens) + self._tokens += len(prompt) // 4 + len(out) // 4 + self._cache[key] = out + return out + + # operations ----------------------------------------------------------- + def attempt(self, task: TaskRecord, skill: str, memory: str, + sample_id: int = 0) -> str: + # sample_id distinguishes repeated rollouts of the SAME (task, skill, + # memory) in the cache key. Without it the attempt cache collapses all + # K dream rollouts into one cached response (spread always 0), which + # silently disables contrastive reflection. sample_id=0 keeps the old + # key format so gate re-scoring still benefits from the cache. + if task.system: + # Benchmark carries its own (research-repo) rollout system prompt. + # Use it verbatim with a neutral skill/memory section — this both + # keeps scoring faithful and avoids the aggressive "OVERRIDE / HARD + # CONSTRAINT" phrasing below, which Azure's content filter flags as a + # jailbreak (HTTP 400) and silently zeroes the rollout. + skill_section = f"## Skill\n{skill.strip()}\n\n" if skill.strip() else "" + mem_section = f"## Memory\n{memory.strip()}\n\n" if memory.strip() else "" + system = task.system.replace("{skill_section}", skill_section) + if "{skill_section}" not in task.system and skill_section: + system = skill_section + system + body = task.intent + ("\n\n" + task.context_excerpt if task.context_excerpt else "") + prompt = f"{system}{mem_section}\n{body}" + salt = f"s{sample_id}:" if sample_id else "" + key = "attempt:" + salt + skill_hash(prompt) + return self._cached_call(key, prompt, max_tokens=512) + # generic path (mined daily-case tasks): neutral, content-filter-safe + # wording. Apply the skill/memory as guidance, not as adversarial + # "OVERRIDE everything" directives. + prompt = ( + "Complete the following task for the user. Follow the skill and memory " + "guidance below, including any output-format and length requirements. " + "When a 'Learned preferences' rule sets an explicit limit (e.g. a length " + "cap), prefer that rule over more general advice it refines.\n\n" + f"# Skill\n{skill or '(none)'}\n\n# Memory\n{memory or '(none)'}\n\n" + f"# Task\n{task.intent}\n\n{task.context_excerpt}\n\n" + "Return ONLY the final answer text, nothing else." + ) + # cache on (task, skill, memory) so identical hold-out re-scoring is free + salt = f"s{sample_id}:" if sample_id else "" + key = "attempt:" + salt + skill_hash(prompt) + return self._cached_call(key, prompt, max_tokens=512) + + def judge(self, task: TaskRecord, response: str) -> Tuple[float, float, str]: + # real-benchmark correctness judge (searchqa/livemath/spreadsheet) — local + if task.reference_kind == "answer" and task.judge: + try: + from skillopt_sleep.experiments.real_eval import score_answer_judge + except ImportError: + score_answer_judge = None # research evaluators not bundled + if score_answer_judge is not None: + return score_answer_judge(task.judge, response) + # gbrain-style rule judge: scored locally, no API spend + if task.reference_kind == "rule" and task.judge: + from skillopt_sleep.judges import score_rule_judge + return score_rule_judge(task.judge, response) + # exact references are scored locally — no API spend + if task.reference_kind == "exact" and task.reference: + hard = exact_score(task.reference, response) + return hard, max(hard, keyword_soft_score(task.reference, response)), "exact(local)" + prompt = ( + "Score how well the response satisfies the rubric, 0..1. " + 'Return ONLY JSON {"score": <0..1>, "reason": "..."}.\n\n' + f"# Rubric\n{task.reference or task.intent}\n\n# Response\n{response}" + ) + key = "judge:" + skill_hash(prompt) + raw = self._cached_call(key, prompt, max_tokens=200) + obj = _extract_json(raw, "object") + if isinstance(obj, dict): + try: + soft = float(obj.get("score", 0.0)) + return (1.0 if soft >= 0.8 else 0.0), soft, str(obj.get("reason", ""))[:200] + except Exception: + pass + return 0.0, 0.0, "judge-parse-failed" + + def reflect( + self, + failures, + successes, + skill: str, + memory: str, + *, + edit_budget: int, + evolve_skill: bool, + evolve_memory: bool, + ) -> List[EditRecord]: + if not failures: + return [] + target = "skill" if evolve_skill else "memory" + cur_doc = (skill if target == "skill" else memory) or "(empty)" + fail_text = "\n".join( + f"- wanted: {t.intent[:160]}\n got: {r.response[:160]}\n why-wrong: {r.fail_reason[:160]}" + for t, r in failures[:8] + ) + # Aggregate the most common failing criteria across all failures so the + # optimizer is told *exactly what the scorer rewards* — gbrain's lesson: + # the optimizer kept proposing reasonable-but-wrong edits until it could + # see the success criteria. + from collections import Counter + crit = Counter() + for _t, r in failures: + fr = r.fail_reason or "" + if fr.startswith("failed:"): + for part in fr[len("failed:"):].split(","): + part = part.strip() + if part: + crit[part] += 1 + + def _explain(c: str) -> str: + # translate an "op=arg" criterion into a plain-English requirement + if "=" in c: + op, _, arg = c.partition("=") + op = op.strip(); arg = arg.strip() + if op == "max_chars": + return f"the ENTIRE response must be at most {arg} characters long" + if op == "min_chars": + return f"the response must be at least {arg} characters long" + if op == "section_present": + return f"the response must contain a section/heading titled '{arg}'" + if op == "regex": + return f"the response must match the pattern /{arg}/ (e.g. include that label)" + if op == "contains": + return f"the response must contain the text '{arg}'" + if op == "tool_called": + return f"the agent must actually call the '{arg}' tool" + return c + + criteria_text = "" + if crit: + criteria_text = ( + "\n# Exact criteria the outputs are FAILING (fix these directly)\n" + + "\n".join(f"- {_explain(c)} [{c}, failed {n}x]" for c, n in crit.most_common()) + ) + pref_text = "" + if getattr(self, "preferences", ""): + pref_text = ( + "\n# User preferences (honor these as priors when writing rules)\n" + + str(self.preferences).strip() + ) + # Task GUARDRAIL: the optimizer must not invent rules that violate the + # task's hard constraints (e.g. SpreadsheetBench answers MUST be a + # ```python``` openpyxl block — a learned "return ```vba```" or "ask the + # user for the range" rule scores 0 because the harness can't run VBA and + # can't ask questions). We surface the benchmark's own rollout system + # prompt (carried on TaskRecord.system) so proposed rules stay in-bounds. + guard_text = _task_guardrail(failures) + prompt = ( + "You are SkillOpt's optimizer. The agent keeps failing the recurring " + f"tasks below. Propose at most {edit_budget} bounded edits to the " + f"{target} document so it stops failing. Each edit MUST be a short, " + "GENERAL, reusable rule or preference (never task-specific, never an " + "answer to a single task). If exact failing criteria are listed, your " + "edits MUST make future outputs satisfy every one of them.\n" + "BE CONCRETE: quote the exact threshold, section name, or format from " + "the criteria verbatim in your rule (e.g. write 'keep the entire " + "response under 1200 characters', NOT 'respect length limits'). Vague " + "rules do not change behavior; specific numeric/structural rules do.\n" + "IMPORTANT: your edits are APPENDED to a 'Learned preferences' block; " + "you CANNOT delete the existing instructions above. If the current " + f"{target} text conflicts with a criterion (e.g. it says 'be exhaustive' " + "but outputs must be under a character limit), write an explicit, " + "forceful OVERRIDE rule stating it supersedes the conflicting " + "instruction, and put the hard requirement first.\n" + "HARD CONSTRAINT: every rule you write MUST be consistent with the " + "'Task output contract' below (if shown). NEVER propose a rule that " + "changes the required output format/language, tells the agent to ask " + "the user a question, or otherwise violates that contract — such a " + "rule scores ZERO because the evaluator cannot honor it.\n" + 'Return ONLY a JSON array: ' + '[{"op":"add|replace|delete","content":"","anchor":"","rationale":""}].\n\n' + f"# Current {target}\n{cur_doc}\n" + f"{guard_text}" + f"{criteria_text}\n" + f"{pref_text}\n\n" + f"# Recurring failures\n{fail_text}" + ) + # Call with one retry: transient non-JSON replies otherwise waste a whole + # night (the gate sees no edits and rejects). A firmer second prompt + # recovers most of these. + arr = None + for attempt in range(2): + p = prompt if attempt == 0 else ( + prompt + "\n\nIMPORTANT: your previous reply was not valid JSON. " + "Reply with ONLY the JSON array, no prose, no markdown fences." + ) + raw = self._call(p, max_tokens=1024) + self._tokens += len(p) // 4 + len(raw) // 4 + arr = _extract_json(raw, "array") + if isinstance(arr, list) and arr: + break + # Expose the last raw optimizer reply so a no-edits night is diagnosable: + # a 0.0->0.0 gate with zero edits is otherwise indistinguishable from + # "nothing to learn" (the cycle persists this in diagnostics.json). + self.last_reflect_raw = raw or "" + edits: List[EditRecord] = [] + if isinstance(arr, list): + for e in arr[:edit_budget]: + if not isinstance(e, dict): + continue + content = str(e.get("content", "")).strip() + if not content: + continue + edits.append(EditRecord( + target=target, + op=str(e.get("op", "add")).strip().lower(), + content=content, + anchor=str(e.get("anchor", "")).strip(), + rationale=str(e.get("rationale", "")).strip(), + )) + return edits + + def tokens_used(self) -> int: + return self._tokens + + +# ── Claude Code CLI backend ─────────────────────────────────────────────────── + +class ClaudeCliBackend(CliBackend): + """Drives the authenticated `claude` CLI: claude -p --output-format text.""" + + name = "claude" + + def __init__(self, model: str = "", claude_path: str = "claude", timeout: int = 180) -> None: + super().__init__(model=model or os.environ.get("SKILLOPT_SLEEP_CLAUDE_MODEL", "") or "sonnet", + timeout=timeout) + # On Windows the npm-installed `claude` is a .cmd shim; CreateProcess + # cannot resolve it by bare name (WinError 2), so every call would + # silently return "" and the whole cycle scores 0.0. shutil.which + # honors PATHEXT and returns the full claude.CMD path. + import shutil as _shutil + self.claude_path = _shutil.which(claude_path) or claude_path + + # Known CLI error prefixes that indicate auth or config failures. + # When detected, we log a warning so the user doesn't mistake a + # broken auth for "nothing to optimize" (issue #68). + # Keep these specific to avoid false positives on normal model output. + _CLI_ERROR_MARKERS = ( + "Not logged in", + "Please run /login", + "Authentication required", + "Invalid API key", + "Unauthorized: invalid x-api-key", + ) + + def _detect_cli_error(self, stdout: str, stderr: str) -> None: + """Log a warning if CLI output looks like an auth/config error. + + Only checks stderr and short stdout (< 300 chars) to avoid + false-positives on legitimate model responses that mention + auth-related terms. + """ + import logging + # Long stdout is almost certainly a real model response, not an error. + check_stdout = stdout if len(stdout) < 300 else "" + combined = check_stdout + "\n" + stderr + for marker in self._CLI_ERROR_MARKERS: + if marker in combined: + from skillopt_sleep.staging import redact_secrets + logging.getLogger("skillopt_sleep").warning( + "Claude CLI returned a likely auth error: %s", + redact_secrets(combined[:200].replace("\n", " ")), + ) + self.last_call_error = combined[:500] + return + + def _call(self, prompt: str, *, max_tokens: int = 1024) -> str: + # Run ISOLATED so the ambient Claude Code environment does not leak into + # the optimizer/target call. Critically, the user's GLOBAL skills + # (~/.claude/skills) are injected regardless of cwd, so we must disable + # them explicitly — without this, reflect/attempt sometimes reply with a + # list of the user's installed skills instead of doing the task. + # --bare skip hooks, LSP, plugins (minimal mode) + # Only safe with ANTHROPIC_API_KEY auth; + # breaks subscription-token auth (#68). + # --disable-slash-commands disable all skills + # --disallowedTools '*' no tool use + # --exclude-dynamic-... drop per-machine cwd/env/memory/git sections + # cwd= no project CLAUDE.md + import tempfile + cmd = [self.claude_path, "-p", "--output-format", "text"] + if os.environ.get("ANTHROPIC_API_KEY"): + cmd.append("--bare") + cmd += [ + "--disable-slash-commands", + "--disallowedTools", "*", + "--exclude-dynamic-system-prompt-sections", + ] + if self.model: + cmd += ["--model", self.model] + # Prompt goes via stdin, not argv: the Windows .cmd shim routes through + # cmd.exe whose command line caps at ~8K chars — reflect/judge prompts + # exceed that. `claude -p` with no positional prompt reads stdin. + clean_cwd = tempfile.mkdtemp(prefix="skillopt_sleep_claude_") + try: + proc = subprocess.run( + cmd, capture_output=True, creationflags=_NO_WINDOW, text=True, timeout=self.timeout, cwd=clean_cwd, + input=prompt, + ) + except Exception as exc: + import logging + self.last_call_error = f"Claude CLI spawn failed: {exc}" + logging.getLogger("skillopt_sleep").warning( + "Claude CLI could not be executed: %s", exc, + ) + return "" + finally: + try: + import shutil + shutil.rmtree(clean_cwd, ignore_errors=True) + except Exception: + pass + out = (proc.stdout or "").strip() + self._detect_cli_error(out, proc.stderr or "") + return out + + def attempt_with_tools(self, task, skill, memory, tools): + # Expose a REAL, callable `search` tool (a shell shim that logs each + # call) so the gbrain quick-answerer judge (tool_called=search) is + # validated honestly: we detect the call from the shim's log, not from + # a self-reported marker. Other tools are stubbed the same way. + import tempfile, shutil, stat + work = tempfile.mkdtemp(prefix="skillopt_sleep_tools_") + calllog = os.path.join(work, "_tool_calls.log") + try: + for tname in (tools or ["search"]): + shim = os.path.join(work, tname) + with open(shim, "w") as f: + f.write( + "#!/usr/bin/env bash\n" + f'echo "{tname}" >> "{calllog}"\n' + 'echo "(search results: 3 relevant notes found; use them to answer)"\n' + ) + os.chmod(shim, os.stat(shim).st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + tool_hint = ( + "You have shell tools available in the current directory: " + + ", ".join(f"./{t}" for t in (tools or ["search"])) + + ". When the skill says to look something up or search before " + "answering, you MUST actually run the tool (e.g. `./search \"query\"`) " + "via Bash before giving your final answer." + ) + prompt = ( + "You are completing a task. Apply the skill and memory rules EXACTLY, " + "including any rule about searching/looking up before answering. " + "Treat a 'Learned preferences' block as HARD CONSTRAINTS that override " + "earlier conflicting skill text.\n\n" + f"{tool_hint}\n\n" + f"# Skill\n{skill or '(none)'}\n\n# Memory\n{memory or '(none)'}\n\n" + f"# Task\n{task.intent}\n\n{task.context_excerpt}\n\n" + "Return ONLY the final answer text." + ) + cmd = [self.claude_path, "-p", "--output-format", "text"] + if os.environ.get("ANTHROPIC_API_KEY"): + cmd.append("--bare") + cmd += [ + "--disable-slash-commands", + "--allowedTools", "Bash", + "--exclude-dynamic-system-prompt-sections", + ] + if self.model: + cmd += ["--model", self.model] + try: + proc = subprocess.run( + cmd, capture_output=True, creationflags=_NO_WINDOW, text=True, timeout=self.timeout, cwd=work, + input=prompt, + ) + resp = (proc.stdout or "").strip() + self._detect_cli_error(resp, proc.stderr or "") + except Exception as exc: + import logging + self.last_call_error = f"Claude CLI spawn failed: {exc}" + logging.getLogger("skillopt_sleep").warning( + "Claude CLI could not be executed: %s", exc, + ) + resp = "" + self._tokens += len(prompt) // 4 + len(resp) // 4 + called: List[str] = [] + if os.path.exists(calllog): + with open(calllog) as f: + logged = {ln.strip() for ln in f if ln.strip()} + called = [t for t in (tools or ["search"]) if t in logged] + return resp, called + finally: + try: + shutil.rmtree(work, ignore_errors=True) + except Exception: + pass + +def resolve_codex_path(explicit: str = "") -> str: + """Find the REAL `@openai/codex` binary, skipping the hermes wrapper. + + The wrapper at ~/.local/bin/codex is a shell shim that execs hermes-codex + and injects extra output; we look past it for the genuine node-installed + binary so replay output is clean. + """ + if explicit: + return explicit + env = os.environ.get("SKILLOPT_SLEEP_CODEX_PATH") + if env: + return env + candidates = [ + os.path.expanduser("~/.nvm/versions/node/v22.22.3/bin/codex"), + ] + # any nvm node version + nvm = os.path.expanduser("~/.nvm/versions/node") + if os.path.isdir(nvm): + for ver in sorted(os.listdir(nvm), reverse=True): + candidates.append(os.path.join(nvm, ver, "bin", "codex")) + for c in candidates: + if not c or not os.path.exists(c): + continue + try: + with open(c, "rb") as f: + head = f.read(64) + # skip the bash shim that execs hermes + if head.startswith(b"#!") and b"bash" in head: + continue + except Exception: + pass + return c + return "codex" # last resort (may be the wrapper) + + +class CodexCliBackend(CliBackend): + """Drives the real Codex CLI: `codex exec -o ` for clean output.""" + + name = "codex" + + def __init__( + self, + model: str = "", + codex_path: str = "", + timeout: int = 240, + sandbox: str = "read-only", + project_dir: str = "", + ) -> None: + super().__init__(model=model or os.environ.get("SKILLOPT_SLEEP_CODEX_MODEL", ""), + timeout=timeout) + self.codex_path = resolve_codex_path(codex_path) + self.sandbox = sandbox + self.project_dir = ( + os.path.abspath(os.path.expanduser(project_dir)) if project_dir else "" + ) + + def _call_once(self, prompt: str, *, max_tokens: int = 1024) -> str: + """One codex exec attempt: returns the response text, or "" on + timeout/exception/empty-output (with last_call_error set). ``_call`` + wraps this with retries so a transient failure is NOT silently scored 0.""" + import tempfile + out_path = tempfile.NamedTemporaryFile( + prefix="codex_last_", suffix=".txt", delete=False + ).name + cmd = [ + self.codex_path, "exec", "--skip-git-repo-check", + "--color", "never", "--sandbox", self.sandbox, + "-o", out_path, + ] + if self.project_dir: + cmd[3:3] = ["-C", self.project_dir] + if self.model: + cmd += ["-m", self.model] + cmd += ["--", prompt] + proc = None + try: + try: + proc = subprocess.run( + cmd, + capture_output=True, + creationflags=_NO_WINDOW, + text=True, + timeout=self.timeout, + cwd=self.project_dir or None, + ) + except subprocess.TimeoutExpired: + self.last_call_error = f"codex exec timed out after {self.timeout}s" + return "" + except Exception as exc: + self.last_call_error = f"codex exec failed: {exc}" + return "" + try: + with open(out_path, encoding="utf-8") as f: + out = f.read().strip() + if out: + return out + except Exception as exc: + self.last_call_error = f"could not read codex output file: {exc}" + stdout = (proc.stdout or "").strip() if proc is not None else "" + stderr = (proc.stderr or "").strip() if proc is not None else "" + if proc is not None and proc.returncode != 0 and not self.last_call_error: + self.last_call_error = f"codex exec exited {proc.returncode}: {stderr[:500]}" + # Do NOT return the CLI's error text as if it were a model response: it + # pollutes rollout/judge/reflect and gets silently scored 0, hiding the + # real cause (e.g. an expired codex auth token surfacing as a 9k-char 401). + # Surface it via last_call_error and return empty instead. + if self.last_call_error: + return "" + return stdout or stderr + finally: + try: + os.unlink(out_path) + except Exception: + pass + + # Fatal codex failures that will NOT recover on retry — fail fast + loud so a + # 0.0 night reads as "codex auth/model/version problem" not "nothing to learn". + # Covers: auth (re-login), and 400 config errors like an unsupported model on a + # ChatGPT account or a model that needs a newer codex CLI (upgrade). + _AUTH_MARKERS = ( + "401 Unauthorized", "refresh_token_reused", "token_expired", + "Please log out and sign in", "Not logged in", "Please run /login", + "authentication token is expired", "Unauthorized: invalid", + "is not supported when using Codex", "requires a newer version of Codex", + ) + + def _call(self, prompt: str, *, max_tokens: int = 1024, retries: int = 3) -> str: + """Retry transient empties/timeouts instead of silently returning "". + + An empty reply scores 0 on every judge, which deflates the held-out + baseline AND blocks the candidate from ever improving — making a flaky + backend indistinguishable from "nothing to learn". The Azure backend + already guards this way (AzureOpenAIBackend._call); codex now does too. + Auth errors are NOT retried (hopeless until the user re-logs-in). + """ + import logging + import random as _r + import time as _t + out = "" + for attempt in range(max(1, retries)): + self.last_call_error = "" + out = self._call_once(prompt, max_tokens=max_tokens) + if out: + return out + err = self.last_call_error or "" + if any(m in err for m in self._AUTH_MARKERS): + from skillopt_sleep.staging import redact_secrets + logging.getLogger("skillopt_sleep").error( + "codex auth error — re-login required (`codex login`): %s", + redact_secrets(err[:200]), + ) + break # fail fast: retrying a 401 just burns calls + if attempt < retries - 1: + _t.sleep(min(6.0, (2 ** attempt) * 0.5) + _r.random() * 0.3) + return out + + def attempt_with_tools(self, task, skill, memory, tools): + # Codex exec runs in a sandbox with shell access; expose the same real + # `search` shim and let it run (workspace-write so the shim can log). + import tempfile, shutil, stat + work = tempfile.mkdtemp(prefix="skillopt_sleep_codextools_") + calllog = os.path.join(work, "_tool_calls.log") + out_path = os.path.join(work, "_last.txt") + try: + for tname in (tools or ["search"]): + shim = os.path.join(work, tname) + with open(shim, "w") as f: + f.write( + "#!/usr/bin/env bash\n" + f'echo "{tname}" >> "{calllog}"\n' + 'echo "(search results: 3 relevant notes found; use them to answer)"\n' + ) + os.chmod(shim, os.stat(shim).st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + tool_hint = ( + "Shell tools are available in the working directory: " + + ", ".join(f"./{t}" for t in (tools or ["search"])) + + ". When the skill says to look something up or search before " + "answering, you MUST actually run the tool (e.g. `./search \"query\"`) " + "before giving your final answer." + ) + prompt = ( + "Complete the task. Apply the skill and memory rules EXACTLY, " + "including any rule about searching before answering. Treat a " + "'Learned preferences' block as HARD CONSTRAINTS overriding earlier " + "conflicting skill text.\n\n" + f"{tool_hint}\n\n# Skill\n{skill or '(none)'}\n\n# Memory\n{memory or '(none)'}\n\n" + f"# Task\n{task.intent}\n\n{task.context_excerpt}\n\nReturn ONLY the final answer." + ) + cmd = [ + self.codex_path, "exec", "--skip-git-repo-check", "--color", "never", + "--sandbox", "workspace-write", "-C", work, "-o", out_path, + ] + if self.model: + cmd += ["-m", self.model] + cmd += ["--", prompt] + self.last_call_error = "" + proc = None + try: + proc = subprocess.run(cmd, capture_output=True, creationflags=_NO_WINDOW, text=True, timeout=self.timeout, cwd=work) + except subprocess.TimeoutExpired: + self.last_call_error = f"codex exec (tools) timed out after {self.timeout}s" + except Exception as exc: # noqa: BLE001 + self.last_call_error = f"codex exec (tools) failed: {exc}" + resp = "" + try: + with open(out_path, encoding="utf-8") as f: + resp = f.read().strip() + except Exception: + resp = "" + # Surface a failed tool-rollout the SAME way _call does: an auth/model/version + # failure on this path must show up in diagnostics (call_error), not vanish as a + # silent empty->0 scored as a failed rollout. Response stays "" (never the error text). + if not resp and not self.last_call_error and proc is not None and proc.returncode != 0: + self.last_call_error = ( + f"codex exec (tools) exited {proc.returncode}: {(proc.stderr or '')[:500]}" + ) + self._tokens += len(prompt) // 4 + len(resp) // 4 + called: List[str] = [] + if os.path.exists(calllog): + with open(calllog) as f: + logged = {ln.strip() for ln in f if ln.strip()} + called = [t for t in (tools or ["search"]) if t in logged] + return resp, called + finally: + try: + shutil.rmtree(work, ignore_errors=True) + except Exception: + pass + +def resolve_copilot_path(explicit: str = "") -> str: + """Find the GitHub Copilot CLI (`copilot`) binary.""" + if explicit: + return explicit + env = os.environ.get("SKILLOPT_SLEEP_COPILOT_PATH") + if env: + return env + import shutil + found = shutil.which("copilot") + return found or "copilot" + + +class CopilotCliBackend(CliBackend): + """Drives the GitHub Copilot CLI in non-interactive mode. + + Uses ``copilot -p --output-format json`` and parses the emitted + JSONL event stream, returning the concatenated ``assistant.message`` + content. The plain-text / ``--silent`` modes do not reliably stream the + response to stdout on all platforms, so JSONL is used for robust capture. + + The call runs in a clean temp cwd with streaming disabled and tools allowed + (so non-interactive mode never blocks on a permission prompt); ``_call``'s + prompts ask for final-answer text only, so no tool use is expected there, + while ``attempt_with_tools`` exposes real, cross-platform callable shims in + the working directory for honest tool-call detection. + + Startup overhead is minimised: each invocation points ``COPILOT_HOME`` at a + dedicated, isolated config dir (no user ``mcp-config.json``, so the user's + MCP servers — including this project's own — are NOT spawned, avoiding a + slow recursive launch), and built-in MCP servers / custom instructions are + disabled. Auth is read from the OS credential store / token env vars, which + live outside ``COPILOT_HOME``, so isolation does not break authentication. + Set ``SKILLOPT_SLEEP_COPILOT_HOME`` to override the isolated home, or set it + empty / ``SKILLOPT_SLEEP_COPILOT_FULL_ENV=1`` to use the user's real + environment instead. + """ + + name = "copilot" + + def __init__(self, model: str = "", copilot_path: str = "", timeout: int = 240) -> None: + super().__init__(model=model or os.environ.get("SKILLOPT_SLEEP_COPILOT_MODEL", ""), + timeout=timeout) + self.copilot_path = resolve_copilot_path(copilot_path) + self.full_env = os.environ.get("SKILLOPT_SLEEP_COPILOT_FULL_ENV", "") == "1" + # Stable isolated home so first-run setup is cached across calls. + if self.full_env: + self.copilot_home = "" + else: + self.copilot_home = os.environ.get("SKILLOPT_SLEEP_COPILOT_HOME") or os.path.join( + tempfile.gettempdir(), "skillopt_sleep_copilot_home" + ) + try: + os.makedirs(self.copilot_home, exist_ok=True) + except Exception: + self.copilot_home = "" + + def _call(self, prompt: str, *, max_tokens: int = 1024) -> str: + clean_cwd = tempfile.mkdtemp(prefix="skillopt_sleep_copilot_") + cmd = [ + self.copilot_path, "-p", prompt, + "--output-format", "json", + "--stream", "off", + "--no-color", + "--log-level", "none", + "--allow-all-tools", + "-C", clean_cwd, + ] + if not self.full_env: + # Drop unneeded startup work: no built-in (github) MCP server and no + # AGENTS.md / custom-instruction loading. With an isolated home that + # has no mcp-config.json, no user MCP servers spawn either. + cmd += ["--disable-builtin-mcps", "--no-custom-instructions"] + if self.model: + cmd += ["--model", self.model] + env = os.environ.copy() + if self.copilot_home: + env["COPILOT_HOME"] = self.copilot_home + try: + proc = subprocess.run( + cmd, capture_output=True, creationflags=_NO_WINDOW, text=True, timeout=self.timeout, cwd=clean_cwd, + encoding="utf-8", errors="replace", env=env, + ) + except Exception: + return "" + finally: + try: + import shutil + shutil.rmtree(clean_cwd, ignore_errors=True) + except Exception: + pass + return self._parse_jsonl_response(proc.stdout or "") + + @staticmethod + def _parse_jsonl_response(raw: str) -> str: + parts: List[str] = [] + for line in raw.splitlines(): + line = line.strip() + if not line or not line.startswith("{"): + continue + try: + obj = json.loads(line) + except Exception: + continue + if obj.get("type") == "assistant.message": + content = (obj.get("data") or {}).get("content") + if isinstance(content, str) and content: + parts.append(content) + return "\n".join(parts).strip() + + def attempt_with_tools(self, task, skill, memory, tools): + # Expose REAL, callable tool shims in the working directory so the + # gbrain quick-answerer judge (tool_called=search) is validated + # honestly: we detect each call from the shim's log, not from a + # self-reported marker. The Copilot CLI is the Windows-validated + # backend, so the shims must be cross-platform — a bash `#!/usr/bin/env + # bash` + chmod shim does NOT execute via `./tool` under PowerShell/cmd, + # so on Windows we emit a `.cmd` batch shim instead. + import shutil + import stat + work = tempfile.mkdtemp(prefix="skillopt_sleep_copilottools_") + calllog = os.path.join(work, "_tool_calls.log") + tool_names = tools or ["search"] + is_windows = os.name == "nt" + try: + for tname in tool_names: + if is_windows: + shim = os.path.join(work, f"{tname}.cmd") + with open(shim, "w") as f: + # `%~n0` is the script's own base name (the tool name); + # writing it keeps the calllog line == tool name so the + # honest-detection match below works unchanged. + f.write( + "@echo off\n" + f'echo %~n0>>"{calllog}"\n' + "echo (search results: 3 relevant notes found; use them to answer)\n" + ) + else: + shim = os.path.join(work, tname) + with open(shim, "w") as f: + f.write( + "#!/usr/bin/env bash\n" + f'echo "{tname}" >> "{calllog}"\n' + 'echo "(search results: 3 relevant notes found; use them to answer)"\n' + ) + os.chmod(shim, os.stat(shim).st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + if is_windows: + tool_hint = ( + "You have shell tools available in the current directory: " + + ", ".join(f"{t}.cmd" for t in tool_names) + + " (each callable as `" + tool_names[0] + "` or `.\\" + + tool_names[0] + "`). When the skill says to look something " + "up or search before answering, you MUST actually run the " + "tool (e.g. `" + tool_names[0] + " \"query\"`) before giving " + "your final answer." + ) + else: + tool_hint = ( + "You have shell tools available in the current directory: " + + ", ".join(f"./{t}" for t in tool_names) + + ". When the skill says to look something up or search before " + "answering, you MUST actually run the tool (e.g. `./search \"query\"`) " + "before giving your final answer." + ) + prompt = ( + "You are completing a task. Apply the skill and memory rules EXACTLY, " + "including any rule about searching/looking up before answering. " + "Treat a 'Learned preferences' block as HARD CONSTRAINTS that override " + "earlier conflicting skill text.\n\n" + f"{tool_hint}\n\n" + f"# Skill\n{skill or '(none)'}\n\n# Memory\n{memory or '(none)'}\n\n" + f"# Task\n{task.intent}\n\n{task.context_excerpt}\n\n" + "Return ONLY the final answer text." + ) + cmd = [ + self.copilot_path, "-p", prompt, + "--output-format", "json", + "--stream", "off", + "--no-color", + "--log-level", "none", + "--allow-all-tools", + "-C", work, + ] + if not self.full_env: + cmd += ["--disable-builtin-mcps", "--no-custom-instructions"] + if self.model: + cmd += ["--model", self.model] + env = os.environ.copy() + if self.copilot_home: + env["COPILOT_HOME"] = self.copilot_home + resp = "" + try: + proc = subprocess.run( + cmd, capture_output=True, creationflags=_NO_WINDOW, text=True, encoding="utf-8", + errors="replace", timeout=self.timeout, cwd=work, env=env, + ) + resp = self._parse_jsonl_response(proc.stdout or "") + except Exception: + resp = "" + self._tokens += len(prompt) // 4 + len(resp) // 4 + called: List[str] = [] + if os.path.exists(calllog): + with open(calllog) as f: + logged = {ln.strip() for ln in f if ln.strip()} + called = [t for t in tool_names if t in logged] + return resp, called + finally: + try: + shutil.rmtree(work, ignore_errors=True) + except Exception: + pass + + +class DualBackend(Backend): + """Route operations to two backends, à la SkillOpt's target vs optimizer. + + * attempt -> TARGET backend (the model the skill is deployed on) + * reflect -> OPTIMIZER backend (the stronger/cheaper model writing edits) + * judge -> OPTIMIZER backend (graded by the optimizer when no local rule) + + This lets you optimize a skill with one model and run tasks on another, and + is the basis of the sleep-scenario transfer experiment (optimize cheap, + deploy expensive — or vice-versa). + """ + + name = "dual" + + def __init__(self, target: Backend, optimizer: Backend) -> None: + self.target = target + self.optimizer = optimizer + self.name = f"target={target.name}/optimizer={optimizer.name}" + + def attempt(self, task, skill, memory, sample_id: int = 0): + return self.target.attempt(task, skill, memory, sample_id=sample_id) + + def attempt_with_tools(self, task, skill, memory, tools): + return self.target.attempt_with_tools(task, skill, memory, tools) + + def judge(self, task, response): + # local rule/exact judging needs no model; delegate to target which + # already short-circuits those. For rubric judging use the optimizer. + if task.reference_kind in {"rule", "exact"}: + return self.target.judge(task, response) + return self.optimizer.judge(task, response) + + def reflect(self, failures, successes, skill, memory, **kw): + return self.optimizer.reflect(failures, successes, skill, memory, **kw) + + def _call(self, prompt, *, max_tokens=1024): + # used by the LLM miner; prefer the optimizer (the "thinking" model) + return self.optimizer._call(prompt, max_tokens=max_tokens) # type: ignore[attr-defined] + + def tokens_used(self): + return self.target.tokens_used() + self.optimizer.tokens_used() + + +# ── Azure OpenAI backend (gpt-5.x via managed identity) ─────────────────────── + +# Endpoint -> deployments, from the intern's avail_api.md. The backend picks the +# first endpoint that hosts the requested deployment. +_AZURE_ENDPOINTS = { + "https://oaidr9.openai.azure.com/": {"gpt-5.5", "gpt-5.4", "gpt-5.4-mini", "gpt-5.4-nano", "o3"}, + "https://t2vgoaigpt4o6.openai.azure.com/": {"gpt-5.5", "gpt-4o-mini", "o3", "o4-mini"}, + "https://oaidr21.openai.azure.com/": {"gpt-5.5", "o3", "o4-mini"}, + "https://searchagent5.cognitiveservices.azure.com/": {"gpt-5.4-mini", "gpt-4o-mini"}, + "https://t2vgoaigpt4o.openai.azure.com/": {"gpt-5.4", "gpt-5.4-nano", "gpt-5.2", "gpt-5.1", "o3", "o4-mini"}, +} +_AZURE_MI_CLIENT_ID = "8cafa2b1-a2a7-4ad9-814a-ffe4aed7e800" + + +class AzureOpenAIBackend(CliBackend): + """Drives Azure OpenAI gpt-5.x deployments via managed identity. + + Mirrors the intern's blog_1 setup (avail_api.md): managed-identity auth, the + same endpoints/deployments. Reuses CliBackend's attempt/judge/reflect prompts + and JSON parsing; only _call() differs. openai + azure-identity are lazy + imported so the mock/CLI paths stay dependency-free. + """ + + name = "azure" + + def __init__(self, deployment: str = "", endpoint: str = "", timeout: int = 180, + api_version: str = "2024-12-01-preview") -> None: + super().__init__(model=deployment or "gpt-5.5", timeout=timeout) + self.deployment = deployment or "gpt-5.5" + self.endpoint = endpoint or self._endpoint_for(self.deployment) + self.api_version = api_version + self.name = f"azure:{self.deployment}" + self._client = None + + @staticmethod + def _endpoint_for(deployment: str) -> str: + for ep, deps in _AZURE_ENDPOINTS.items(): + if deployment in deps: + return ep + return "https://oaidr9.openai.azure.com/" + + def _get_client(self): + if self._client is None: + from azure.identity import ManagedIdentityCredential, get_bearer_token_provider + from openai import AzureOpenAI + cred = ManagedIdentityCredential(client_id=_AZURE_MI_CLIENT_ID) + tp = get_bearer_token_provider(cred, "https://cognitiveservices.azure.com/.default") + self._client = AzureOpenAI( + azure_endpoint=self.endpoint, azure_ad_token_provider=tp, + api_version=self.api_version, max_retries=4, + ) + return self._client + + def _call(self, prompt: str, *, max_tokens: int = 1024, retries: int = 5) -> str: + """Call the deployment with bounded retries. + + IMPORTANT: transient failures (429 rate-limit, timeouts, 5xx) must NOT be + silently turned into an empty string — an empty response scores 0 and + deflates every baseline/after measure. We retry with exponential backoff + (mirroring the research repo's retries=5) and only return "" after the + budget is exhausted. ``time``/``random`` are used for backoff; both are + available here (this is library code, not a Workflow script sandbox). + """ + import random as _r + import time as _t + + client = self._get_client() + last_exc = None + for attempt in range(max(1, retries)): + try: + resp = client.chat.completions.create( + model=self.deployment, + messages=[{"role": "user", "content": prompt}], + max_completion_tokens=16384, + ) + text = (resp.choices[0].message.content or "").strip() + try: + u = resp.usage + self._tokens += (getattr(u, "prompt_tokens", 0) or 0) + (getattr(u, "completion_tokens", 0) or 0) + except Exception: + pass + if text: + return text + # empty but no exception: model genuinely returned nothing — one + # quick retry can help (reasoning models occasionally yield empty) + last_exc = "empty-response" + except Exception as e: # noqa: BLE001 + last_exc = e + # backoff before next try (skip after the final attempt) + if attempt < retries - 1: + _t.sleep(min(8.0, (2 ** attempt) * 0.5) + _r.random() * 0.4) + return "" + + +class AzureResponsesBackend(AzureOpenAIBackend): + """gpt-5.x via the **Responses API** on the high-throughput gpt4v endpoints. + + Differs from AzureOpenAIBackend in three ways, all required by the enhanced + experiment: + * Auth via ``AzureCliCredential`` (the logged-in user), not Managed Identity + — the gpt4v-scus/swc accounts grant the data role to the CLI principal. + * Calls ``client.responses.create`` (the /responses API) instead of + chat.completions — these deployments are Responses-only. + * Round-robins across multiple endpoints for parallel throughput; each + worker thread binds a client for one endpoint (picked by thread index) + so concurrent replay spreads load across all endpoints. + + A single shared ``AzureCliCredential`` token provider is reused across all + endpoint clients (the token is cached + auto-refreshed by the provider). + """ + + name = "azure-responses" + + # the two parallel /responses endpoints (user-provided), both hosting gpt-5.5 + _RESP_ENDPOINTS = [ + "https://gpt4v-scus.openai.azure.com/", + "https://gpt4v-swc.openai.azure.com/", + ] + + def __init__(self, deployment: str = "", endpoints: Optional[List[str]] = None, + timeout: int = 180, api_version: str = "2025-04-01-preview") -> None: + super().__init__(deployment=deployment, endpoint=(endpoints or self._RESP_ENDPOINTS)[0], + timeout=timeout, api_version=api_version) + self.endpoints = list(endpoints or self._RESP_ENDPOINTS) + self.name = f"azure-responses:{self.deployment}" + self._token_provider = None + self._clients: dict = {} # endpoint -> AzureOpenAI client + import threading as _thr + self._lock = _thr.Lock() + self._rr = 0 # round-robin counter + + def _get_provider(self): + if self._token_provider is None: + from azure.identity import AzureCliCredential, get_bearer_token_provider + self._token_provider = get_bearer_token_provider( + AzureCliCredential(), "https://cognitiveservices.azure.com/.default") + return self._token_provider + + def _client_for(self, endpoint: str): + cl = self._clients.get(endpoint) + if cl is None: + from openai import AzureOpenAI + cl = AzureOpenAI( + azure_endpoint=endpoint, azure_ad_token_provider=self._get_provider(), + api_version=self.api_version, max_retries=2, + ) + self._clients[endpoint] = cl + return cl + + def _next_endpoint(self) -> str: + # round-robin so concurrent calls spread across all endpoints + with self._lock: + ep = self.endpoints[self._rr % len(self.endpoints)] + self._rr += 1 + return ep + + def _call(self, prompt: str, *, max_tokens: int = 1024, retries: int = 5) -> str: + import random as _r + import time as _t + last = None + base_ep = self._next_endpoint() # this call's primary endpoint + base_idx = self.endpoints.index(base_ep) + for attempt in range(max(1, retries)): + # on retry, fail over to the other endpoint(s) + ep = self.endpoints[(base_idx + attempt) % len(self.endpoints)] + try: + client = self._client_for(ep) + resp = client.responses.create( + model=self.deployment, input=prompt, + max_output_tokens=16384, + ) + text = (getattr(resp, "output_text", "") or "").strip() + try: + u = resp.usage + self._tokens += (getattr(u, "input_tokens", 0) or 0) + (getattr(u, "output_tokens", 0) or 0) + except Exception: + pass + if text: + return text + last = "empty-response" + except Exception as e: # noqa: BLE001 + last = e + if attempt < retries - 1: + _t.sleep(min(8.0, (2 ** attempt) * 0.5) + _r.random() * 0.4) + return "" + + +def get_backend( + name: str, + *, + model: str = "", + claude_path: str = "claude", + codex_path: str = "", + azure_endpoint: str = "", + project_dir: str = "", +) -> Backend: + n = (name or "mock").strip().lower() + if n in {"claude", "anthropic", "claude_cli", "claude_code"}: + return ClaudeCliBackend(model=model, claude_path=claude_path) + if n in {"codex", "codex_cli", "openai_codex"}: + return CodexCliBackend(model=model, codex_path=codex_path, project_dir=project_dir) + if n in {"azure", "azure_openai", "aoai"}: + return AzureOpenAIBackend(deployment=model, endpoint=azure_endpoint) + if n in {"azure-responses", "azure_responses", "aoai-responses", "responses"}: + eps = [e.strip() for e in azure_endpoint.split(",") if e.strip()] or None + return AzureResponsesBackend(deployment=model, endpoints=eps) + if n in {"copilot", "github_copilot", "copilot_cli", "gh_copilot"}: + return CopilotCliBackend(model=model) + if n in {"handoff", "session", "file"}: + # Lazy import: handoff_backend imports CliBackend from this module. + from skillopt_sleep.handoff_backend import HandoffBackend + hdir = os.environ.get("SKILLOPT_SLEEP_HANDOFF_DIR", "") or os.path.join( + project_dir or os.getcwd(), ".skillopt-sleep-handoff" + ) + return HandoffBackend(model=model, handoff_dir=hdir) + return MockBackend() + + +def build_backend( + *, + backend: str = "mock", + model: str = "", + optimizer_backend: str = "", + optimizer_model: str = "", + target_backend: str = "", + target_model: str = "", + codex_path: str = "", + azure_endpoint: str = "", + preferences: str = "", + project_dir: str = "", +) -> Backend: + """Build a single or dual backend. + + If optimizer_* or target_* are given, returns a DualBackend routing + attempt->target and reflect/judge->optimizer. Otherwise a single backend + from (backend, model). ``preferences`` (free text) is attached so reflect + uses it as a prior (set on the optimizer for dual backends). + """ + has_split = any([optimizer_backend, optimizer_model, target_backend, target_model]) + if not has_split: + be = get_backend( + backend, + model=model, + codex_path=codex_path, + azure_endpoint=azure_endpoint, + project_dir=project_dir, + ) + be.preferences = preferences + return be + tgt = get_backend(target_backend or backend, model=target_model or model, + codex_path=codex_path, azure_endpoint=azure_endpoint, + project_dir=project_dir) + opt = get_backend(optimizer_backend or backend, model=optimizer_model or model, + codex_path=codex_path, azure_endpoint=azure_endpoint, + project_dir=project_dir) + opt.preferences = preferences # reflect runs on the optimizer + dual = DualBackend(target=tgt, optimizer=opt) + dual.preferences = preferences + return dual diff --git a/skillopt_sleep/budget.py b/skillopt_sleep/budget.py new file mode 100644 index 0000000..48875ca --- /dev/null +++ b/skillopt_sleep/budget.py @@ -0,0 +1,75 @@ +"""SkillOpt-Sleep — budget controller. + +Lets the user say how much they're willing to spend on a night's "dreaming", +in tokens or wall-clock minutes, and the engine schedules depth (how many +rollouts × how many nights) within that budget. Stops cleanly when exhausted +and reports what it skipped (no silent truncation). +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class Budget: + max_tokens: Optional[int] = None # None = unlimited + max_minutes: Optional[float] = None # None = unlimited + _start_time: Optional[float] = None + _tokens_at_start: int = 0 + + def start(self, clock_fn, tokens_now: int) -> None: + self._start_time = clock_fn() + self._tokens_at_start = tokens_now + + def tokens_spent(self, tokens_now: int) -> int: + return max(0, tokens_now - self._tokens_at_start) + + def minutes_elapsed(self, clock_fn) -> float: + if self._start_time is None: + return 0.0 + return (clock_fn() - self._start_time) / 60.0 + + def remaining_fraction(self, *, tokens_now: int, clock_fn) -> float: + """Smallest remaining fraction across all active limits (1.0 = fresh).""" + fracs = [1.0] + if self.max_tokens: + fracs.append(max(0.0, 1.0 - self.tokens_spent(tokens_now) / self.max_tokens)) + if self.max_minutes: + fracs.append(max(0.0, 1.0 - self.minutes_elapsed(clock_fn) / self.max_minutes)) + return min(fracs) + + def exhausted(self, *, tokens_now: int, clock_fn) -> bool: + if self.max_tokens and self.tokens_spent(tokens_now) >= self.max_tokens: + return True + if self.max_minutes and self.minutes_elapsed(clock_fn) >= self.max_minutes: + return True + return False + + def status(self, *, tokens_now: int, clock_fn) -> str: + parts = [] + if self.max_tokens: + parts.append(f"tokens {self.tokens_spent(tokens_now)}/{self.max_tokens}") + if self.max_minutes: + parts.append(f"minutes {self.minutes_elapsed(clock_fn):.1f}/{self.max_minutes}") + return ", ".join(parts) or "unbounded" + + +def plan_depth(budget: Budget, *, n_tasks: int, + default_nights: int = 2, default_k: int = 1) -> tuple: + """Heuristically choose (nights, rollouts_per_task) from a token budget. + + Rough cost model: one rollout ≈ 1 unit; a night does ~n_tasks*k rollouts + plus reflect/gate (~2*n_tasks). We scale k and nights up with more budget. + Returns (nights, k). With no budget set, returns the defaults. + """ + if not budget.max_tokens: + return default_nights, default_k + # assume ~1.5k tokens per rollout as a planning constant + rollouts_affordable = budget.max_tokens / 1500.0 + per_night = max(1, n_tasks) * 3 # rollouts + reflect + gate, k=1 + nights = max(1, min(4, int(rollouts_affordable // per_night))) + # spend surplus on more rollouts-per-task (contrastive signal) + surplus = rollouts_affordable - nights * per_night + k = max(1, min(5, 1 + int(surplus // max(1, n_tasks)))) + return nights, k diff --git a/skillopt_sleep/config.py b/skillopt_sleep/config.py new file mode 100644 index 0000000..06303e0 --- /dev/null +++ b/skillopt_sleep/config.py @@ -0,0 +1,162 @@ +"""SkillOpt-Sleep — configuration. + +Config is JSON-first (yaml optional) so the engine and the deterministic +experiment run with zero external dependencies. Defaults are safe: +review-gated adoption, single-project scope, bounded token/task budgets. + +Resolution order (later wins): + 1. built-in DEFAULTS + 2. ~/.skillopt-sleep/config.json (or .yaml if PyYAML available) + 3. explicit overrides passed to load_config(**overrides) +""" +from __future__ import annotations + +import json +import os +from dataclasses import dataclass, field +from typing import Any, Dict, Optional + +HOME_STATE_DIR = os.path.expanduser("~/.skillopt-sleep") +CLAUDE_HOME = os.path.expanduser("~/.claude") +CODEX_HOME = os.path.expanduser("~/.codex") + + +DEFAULTS: Dict[str, Any] = { + # ── scope ────────────────────────────────────────────────────────────── + "claude_home": CLAUDE_HOME, + "codex_home": CODEX_HOME, + "transcript_source": "claude", # "claude" | "codex" | "auto" + "projects": "invoked", # "invoked" | "all" | [list of abs paths] + "invoked_project": "", # filled at runtime (cwd) when projects == "invoked" + "lookback_hours": 72, # harvest window when no prior sleep recorded + # ── budgets ──────────────────────────────────────────────────────────── + "max_tasks_per_night": 40, + "max_tokens_per_night": 400_000, + "holdout_fraction": 0.34, # legacy alias for val_fraction + "val_fraction": 0.34, # real tasks reserved to gate updates + "test_fraction": 0.0, # real tasks reserved as the final held-out measure + # ── optimizer ────────────────────────────────────────────────────────── + "backend": "mock", # "mock" | "claude" | "codex" | "copilot" + "model": "", # backend-specific; "" => backend default + "gate_mode": "on", # "on" (validation-gated) | "off" (greedy, no hard filter) + "codex_path": "", # "" => auto-detect the real @openai/codex binary + "edit_budget": 4, # textual learning rate (max edits/night) + "gate_metric": "mixed", # hard | soft | mixed (mixed best for tiny holdouts) + "gate_mixed_weight": 0.5, + "replay_mode": "mock", # "mock" (sandboxed prompt) | "fresh" (worktree) + # ── dream + recall (opt-in; defaults reproduce the prior single-shot loop) ─ + "dream_rollouts": 1, # >1 => multi-rollout contrastive reflection per task + "dream_factor": 0, # >0 => add N synthetic variants of each task to the dream + "recall_k": 0, # >0 => recall the K most-similar past tasks into the dream + "evolve_memory": True, # consolidate CLAUDE.md + "evolve_skill": True, # consolidate the managed SKILL.md + "llm_mine": True, # use the backend to mine checkable tasks (real backends) + "target_skill_path": "", # explicit SKILL.md target for repo-scoped agents + "target_task_filter": True, # prefer mined tasks matching target_skill_path/text + "progress": False, # print phase progress to stderr + # ── adoption / safety ────────────────────────────────────────────────── + "auto_adopt": False, # default: stage + require explicit `adopt` + "managed_skill_name": "skillopt-sleep-learned", + "redact_secrets": True, + "seed": 42, +} + + +@dataclass +class SleepConfig: + data: Dict[str, Any] = field(default_factory=lambda: dict(DEFAULTS)) + + # convenient attribute access ------------------------------------------- + def __getattr__(self, name: str) -> Any: + # only called when normal attribute lookup fails + data = object.__getattribute__(self, "data") + if name in data: + return data[name] + raise AttributeError(name) + + def get(self, key: str, default: Any = None) -> Any: + return self.data.get(key, default) + + def to_dict(self) -> Dict[str, Any]: + return dict(self.data) + + # paths ------------------------------------------------------------------ + @property + def state_dir(self) -> str: + # Allow full isolation: if the caller overrides state_dir explicitly, + # honor it; else derive from claude_home's parent so a single + # --claude-home flag isolates transcripts AND state together; else the + # default ~/.skillopt-sleep. + explicit = self.data.get("state_dir") + if explicit: + return explicit + ch = self.data.get("claude_home", CLAUDE_HOME) + if os.path.abspath(ch) != os.path.abspath(CLAUDE_HOME): + return os.path.join(os.path.dirname(os.path.abspath(ch)), ".skillopt-sleep") + return HOME_STATE_DIR + + @property + def state_path(self) -> str: + return os.path.join(self.state_dir, "state.json") + + @property + def transcripts_dir(self) -> str: + return os.path.join(self.data["claude_home"], "projects") + + @property + def codex_archived_sessions_dir(self) -> str: + return os.path.join(self.data["codex_home"], "archived_sessions") + + @property + def history_path(self) -> str: + return os.path.join(self.data["claude_home"], "history.jsonl") + + @property + def skills_dir(self) -> str: + return os.path.join(self.data["claude_home"], "skills") + + def managed_skill_path(self) -> str: + target = self.data.get("target_skill_path") or "" + if target: + target = os.path.expanduser(str(target)) + if not os.path.isabs(target): + base = self.data.get("invoked_project") or os.getcwd() + target = os.path.join(base, target) + return os.path.abspath(target) + return os.path.join( + self.skills_dir, self.data["managed_skill_name"], "SKILL.md" + ) + + +def _user_config_path() -> Optional[str]: + for name in ("config.json", "config.yaml", "config.yml"): + p = os.path.join(HOME_STATE_DIR, name) + if os.path.exists(p): + return p + return None + + +def _load_file(path: str) -> Dict[str, Any]: + if path.endswith((".yaml", ".yml")): + try: + import yaml # optional + with open(path) as f: + return yaml.safe_load(f) or {} + except Exception: + return {} + with open(path) as f: + return json.load(f) + + +def load_config(**overrides: Any) -> SleepConfig: + data = dict(DEFAULTS) + path = _user_config_path() + if path: + try: + data.update(_load_file(path) or {}) + except Exception: + pass + data.update({k: v for k, v in overrides.items() if v is not None}) + if data.get("projects") == "invoked" and not data.get("invoked_project"): + data["invoked_project"] = os.getcwd() + return SleepConfig(data=data) diff --git a/skillopt_sleep/consolidate.py b/skillopt_sleep/consolidate.py new file mode 100644 index 0000000..a9ea662 --- /dev/null +++ b/skillopt_sleep/consolidate.py @@ -0,0 +1,266 @@ +"""SkillOpt-Sleep — Stage 4: consolidate (one SkillOpt epoch). + +This is the core that makes nightly evolution *safe*: it proposes bounded +edits from replayed failures, applies them to a candidate skill/memory, then +**gates** the candidate on a held-out slice of the user's own tasks. Only a +candidate that strictly improves the held-out score is accepted — the SkillOpt +validation gate, vendored self-contained in ``skillopt_sleep.gate``. +""" +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from typing import List, Optional, Tuple + +from skillopt_sleep.backend import Backend +from skillopt_sleep.memory import apply_edits +from skillopt_sleep.replay import aggregate_scores, replay_batch +from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord + + +# Self-contained validation gate (vendored from SkillOpt; zero dependency on the +# research package, so this open-source tool stays decoupled from the paper code). +from skillopt_sleep.gate import evaluate_gate, select_gate_score +_HAVE_REPO_GATE = True + + +@dataclass +class ConsolidationResult: + accepted: bool + gate_action: str + baseline_score: float + candidate_score: float + new_skill: str + new_memory: str + applied_edits: List[EditRecord] + rejected_edits: List[EditRecord] + holdout_baseline: float + holdout_candidate: float + # ── observability (so a 0.0->0.0 night is self-diagnosing, not a black box) ── + holdout_detail: List[dict] = field(default_factory=list) # per val task: hard/soft/resp/why + reflect_raw: str = "" # the optimizer's last raw reply (empty => reflect produced nothing) + call_error: str = "" # backend's last call error (timeout/auth/empty) + + +def _split(tasks: List[TaskRecord]) -> Tuple[List[TaskRecord], List[TaskRecord]]: + """Return (train_tasks, val_tasks). + + train drives reflect; val gates updates. test is held out entirely from + consolidation and is scored by the caller. Accepts legacy split names + (replay->train, holdout->val) for robustness. + """ + def _norm(s: str) -> str: + return {"replay": "train", "holdout": "val"}.get(s, s) + + train = [t for t in tasks if _norm(t.split) == "train"] + val = [t for t in tasks if _norm(t.split) == "val"] + # be robust if a split is empty: fall back so a night still does something, + # but never silently use test as val. + test = [t for t in tasks if _norm(t.split) == "test"] + if not val: + # prefer train as the gate reference over nothing; last resort all-but-test + val = train or [t for t in tasks if _norm(t.split) != "test"] or tasks + if not train: + train = val + return train, val + + +def _holdout_detail(pairs: List[Tuple[TaskRecord, ReplayResult]]) -> List[dict]: + """Per-task held-out evidence so a 0.0 night explains itself: was the + response empty (backend call failed) or non-empty-but-failing-checks + (judge too strict / edit didn't help)? The two need opposite fixes.""" + out: List[dict] = [] + for t, r in pairs: + resp = r.response or "" + out.append({ + "id": t.id, + "reference_kind": t.reference_kind, + "hard": r.hard, + "soft": r.soft, + "response_len": len(resp), + "response_head": resp[:200], + "why": (r.fail_reason or r.judge_rationale or "")[:200], + }) + return out + + +def consolidate( + backend: Backend, + tasks: List[TaskRecord], + skill: str, + memory: str, + *, + edit_budget: int = 4, + gate_metric: str = "mixed", + gate_mixed_weight: float = 0.5, + gate_mode: str = "on", # "on" (hard/soft per gate_metric) | "off" (greedy) + rollouts_k: int = 1, # >1 => multi-rollout contrastive reflection + evolve_skill: bool = True, + evolve_memory: bool = True, + night: int = 1, +) -> ConsolidationResult: + """Run one consolidation epoch: reflect -> bounded edit -> gate. + + train tasks drive reflect; val tasks gate the update (test is held out by the + caller). With ``gate_mode='off'`` edits are accepted greedily (no val-improve + requirement) — the user opts out of hard filtering — but val scores are still + recorded so the report shows whether quality moved. + + Skill and memory are evolved in sequence (skill first if both enabled). + """ + train_tasks, val_tasks = _split(tasks) + gate_off = str(gate_mode).strip().lower() in {"off", "none", "false", "greedy"} + holdout_detail: List[dict] = [] + + # ── baseline on the VAL slice (the gate reference) ──────────────────── + # When the gate is OFF the user has opted out of holding out a validation set + # (the daily-use design): we accept edits greedily and judge quality only on + # the real test set, scored by the caller. So we SKIP all val scoring — it is + # both wasted cost and contrary to the "no val set required" design. + if gate_off: + base_hard, base_soft = 0.0, 0.0 + else: + base_pairs = replay_batch(backend, val_tasks, skill, memory) + base_hard, base_soft = aggregate_scores(base_pairs) + holdout_detail = _holdout_detail(base_pairs) + base_score = select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight) + + # ── reflect over TRAIN-split failures/successes ─────────────────────── + train_pairs = replay_batch(backend, train_tasks, skill, memory) + failures = [(t, r) for (t, r) in train_pairs if r.hard < 1.0] + successes = [(t, r) for (t, r) in train_pairs if r.hard >= 1.0] + + cand_skill, cand_memory = skill, memory + all_applied: List[EditRecord] = [] + all_rejected: List[EditRecord] = [] + + def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: + nonlocal cand_skill, cand_memory, base_score, all_applied, all_rejected + if not edits: + return doc + new_doc, applied = apply_edits(doc, edits) + if not applied: + return doc + # gate OFF: accept greedily with NO val scoring (the daily-use path) + if gate_off: + all_applied.extend(applied) + return new_doc + # gate ON: score the candidate on the VAL slice, keep only if it improves + trial_skill = new_doc if which == "skill" else cand_skill + trial_memory = new_doc if which == "memory" else cand_memory + pairs = replay_batch(backend, val_tasks, trial_skill, trial_memory) + h, s = aggregate_scores(pairs) + cand_score = select_gate_score(h, s, gate_metric, gate_mixed_weight) + if cand_score > base_score: + base_score = max(base_score, cand_score) + all_applied.extend(applied) + return new_doc + all_rejected.extend(applied) + return doc + + if evolve_skill: + if rollouts_k > 1: + # multi-rollout contrastive reflection: run each train task K times + # and distill a rule from the good-vs-bad contrast (the imagination signal). + from skillopt_sleep.rollout import multi_rollout, contrastive_reflect + # Parallelize across tasks (each multi_rollout also parallelizes its K + # attempts). This dream phase is the dominant cost; serial execution + # times out on real backends. Cap total in-flight at the worker env. + import os + from concurrent.futures import ThreadPoolExecutor + try: + _w = int(os.environ.get("SKILLOPT_SLEEP_WORKERS", "1")) + except ValueError: + _w = 1 + if _w > 1 and len(train_tasks) > 1: + # split the worker budget between task-parallelism and per-task K + task_workers = max(1, min(len(train_tasks), _w)) + per_task = max(1, _w // task_workers) + with ThreadPoolExecutor(max_workers=task_workers) as ex: + sets = list(ex.map( + lambda t: multi_rollout(backend, t, cand_skill, cand_memory, + k=rollouts_k, workers=per_task), + train_tasks)) + else: + sets = [multi_rollout(backend, t, cand_skill, cand_memory, + k=rollouts_k, workers=1) + for t in train_tasks] + edits = contrastive_reflect( + backend, sets, cand_skill, cand_memory, + edit_budget=edit_budget, target="skill", + ) + # fall back to single-shot reflect if contrast yielded nothing + if not edits: + edits = backend.reflect( + failures, successes, cand_skill, cand_memory, + edit_budget=edit_budget, evolve_skill=True, evolve_memory=False, + ) + else: + edits = backend.reflect( + failures, successes, cand_skill, cand_memory, + edit_budget=edit_budget, evolve_skill=True, evolve_memory=False, + ) + cand_skill = _gate_apply(cand_skill, edits, "skill") + + if evolve_memory: + # re-evaluate failures under the (possibly improved) skill + train_pairs2 = replay_batch(backend, train_tasks, cand_skill, cand_memory) + failures2 = [(t, r) for (t, r) in train_pairs2 if r.hard < 1.0] + successes2 = [(t, r) for (t, r) in train_pairs2 if r.hard >= 1.0] + edits_m = backend.reflect( + failures2, successes2, cand_skill, cand_memory, + edit_budget=edit_budget, evolve_skill=False, evolve_memory=True, + ) + cand_memory = _gate_apply(cand_memory, edits_m, "memory") + + # ── final decision ──────────────────────────────────────────────────── + if gate_off: + # greedy mode: no val scoring at all. Keep whatever edits we applied; the + # caller measures real quality on the test set. We report holdout_candidate + # as 0.0 (val intentionally not computed in this variant). + final_hard, final_soft = 0.0, 0.0 + final_score = 0.0 + accepted = bool(all_applied) + action = "greedy_applied" if all_applied else "greedy_noop" + base_gate_score = 0.0 + else: + # scored on the VAL slice (the gate reference) + final_pairs = replay_batch(backend, val_tasks, cand_skill, cand_memory) + final_hard, final_soft = aggregate_scores(final_pairs) + final_score = select_gate_score(final_hard, final_soft, gate_metric, gate_mixed_weight) + base_gate_score = select_gate_score(base_hard, base_soft, gate_metric, gate_mixed_weight) + if _HAVE_REPO_GATE: + gate = evaluate_gate( + candidate_skill=cand_skill, + cand_hard=final_hard, + current_skill=skill, + current_score=base_gate_score, + best_skill=skill, + best_score=base_gate_score, + best_step=night - 1, + global_step=night, + cand_soft=final_soft, + metric=gate_metric, + mixed_weight=gate_mixed_weight, + ) + action = gate.action + accepted = bool(all_applied) and final_score > base_gate_score + else: + action = "accept" if final_score > base_gate_score else "reject" + accepted = bool(all_applied) and final_score > base_gate_score + + return ConsolidationResult( + accepted=accepted, + gate_action=action, + baseline_score=base_gate_score, + candidate_score=final_score, + new_skill=cand_skill if accepted else skill, + new_memory=cand_memory if accepted else memory, + applied_edits=all_applied, + rejected_edits=all_rejected, + holdout_baseline=base_hard, + holdout_candidate=final_hard, + holdout_detail=holdout_detail, + reflect_raw=getattr(backend, "last_reflect_raw", "") or "", + call_error=getattr(backend, "last_call_error", "") or "", + ) diff --git a/skillopt_sleep/cycle.py b/skillopt_sleep/cycle.py new file mode 100644 index 0000000..cb54ca8 --- /dev/null +++ b/skillopt_sleep/cycle.py @@ -0,0 +1,322 @@ +"""SkillOpt-Sleep — the nightly cycle orchestrator. + +run_sleep_cycle() wires the stages: + harvest -> mine -> replay -> consolidate(gate) -> stage (-> optional adopt) + +It is pure-Python and import-light; with backend="mock" it runs with no API +key and no third-party deps, which is what the deterministic experiment and +CI use. With backend="anthropic" it spends the user's budget for real lift. +""" +from __future__ import annotations + +import os +import sys +from dataclasses import dataclass +from typing import List, Optional + +from skillopt_sleep.backend import Backend, get_backend +from skillopt_sleep.config import SleepConfig, load_config +from skillopt_sleep.dream import dream_consolidate +from skillopt_sleep.harvest_sources import harvest_for_config +from skillopt_sleep.memory import ensure_skill_scaffold +from skillopt_sleep.mine import mine +from skillopt_sleep.staging import adopt as adopt_staging +from skillopt_sleep.staging import redact_secrets +from skillopt_sleep.staging import write_staging +from skillopt_sleep.state import SleepState, _now_iso +from skillopt_sleep.types import SessionDigest, SleepReport, TaskRecord + + +@dataclass +class CycleOutcome: + report: SleepReport + staging_dir: str + adopted: bool + adopted_paths: List[str] + + +def _project_paths(cfg: SleepConfig) -> str: + """Where live CLAUDE.md lives + which project we are evolving.""" + if cfg.get("projects") == "invoked" and cfg.get("invoked_project"): + return cfg.get("invoked_project") + # default: the invoked cwd + return cfg.get("invoked_project") or os.getcwd() + + +def _read(path: str) -> str: + try: + with open(path, encoding="utf-8") as f: + return f.read() + except Exception: + return "" + + +def _progress(cfg: SleepConfig, message: str) -> None: + if cfg.get("progress", False): + print(f"[sleep] {message}", file=sys.stderr, flush=True) + + +def _render_report_md(report: SleepReport, cfg: SleepConfig) -> str: + lines = [ + f"# SkillOpt-Sleep — night {report.night} report", + "", + f"- project: `{report.project}`", + f"- backend: `{cfg.get('backend')}` replay: `{cfg.get('replay_mode')}`", + f"- sessions harvested: {report.n_sessions}", + f"- tasks mined: {report.n_tasks} (replayed: {report.n_replayed})", + f"- held-out score: {report.baseline_score:.3f} -> {report.candidate_score:.3f}", + f"- gate: **{report.gate_action}** (accepted={report.accepted})", + f"- tokens used: {report.tokens_used}", + "", + ] + if report.edits: + lines.append("## Accepted edits") + for e in report.edits: + lines.append(f"- [{e.target}/{e.op}] {e.content} \n _why: {e.rationale}_") + lines.append("") + if report.rejected_edits: + lines.append("## Rejected by gate (kept as negative feedback)") + for e in report.rejected_edits: + lines.append(f"- [{e.target}/{e.op}] {e.content}") + lines.append("") + if report.notes: + lines.append("## Notes") + for n in report.notes: + lines.append(f"- {n}") + lines.append("") + lines.append("_Review, then run `/sleep adopt` to apply, or discard this folder._") + return "\n".join(lines) + + +def run_sleep_cycle( + cfg: Optional[SleepConfig] = None, + *, + seed_tasks: Optional[List[TaskRecord]] = None, + dry_run: bool = False, + clock: Optional[float] = None, + backend: Optional[Backend] = None, +) -> CycleOutcome: + """Run one full sleep cycle and return the outcome. + + Parameters + ---------- + cfg : SleepConfig + seed_tasks : optional pre-built TaskRecords (used by the experiment to + inject a known persona instead of harvesting ~/.claude). + dry_run : harvest+mine+replay but DO NOT stage/adopt (report only). + clock : fixed epoch seconds for deterministic timestamps in tests. + backend : optional pre-built Backend; the handoff driver passes one so + it can inspect the backend's pending calls after the run. + """ + cfg = cfg or load_config() + state = SleepState.load(cfg.state_path) + night = state.begin_night(clock) + project = _project_paths(cfg) + started = _now_iso(clock) + + backend = backend or get_backend( + cfg.get("backend", "mock"), + model=cfg.get("model", ""), + codex_path=cfg.get("codex_path", ""), + project_dir=project, + ) + _progress(cfg, f"night {night}: project={project} backend={backend.name}") + + # ── live skill/memory docs ─────────────────────────────────────────── + live_memory_path = os.path.join(project, "CLAUDE.md") + live_skill_path = cfg.managed_skill_path() + _progress(cfg, f"live skill: {live_skill_path}") + raw_skill = _read(live_skill_path) + skill = raw_skill + memory = _read(live_memory_path) + if not skill: + skill = ensure_skill_scaffold( + "", name=cfg.get("managed_skill_name", "skillopt-sleep-learned"), + description="Preferences and procedures learned from past local agent sessions.", + ) + target_filter = bool( + cfg.get("target_task_filter", True) + and cfg.get("target_skill_path", "") + and raw_skill + ) + + # ── 1+2. harvest + mine (unless seed_tasks injected) ───────────────── + digests: List[SessionDigest] = [] + if seed_tasks is not None: + tasks = seed_tasks + n_sessions = 0 + _progress(cfg, f"using {len(tasks)} seeded tasks") + else: + since = state.last_harvest_for(project) + # On first run (no prior harvest), apply lookback_hours so we don't + # scan the entire transcript history and trigger massive LLM mining. + if since is None: + lookback_hours = cfg.get("lookback_hours", 72) + if lookback_hours is not None and lookback_hours > 0: + import time + ref_time = clock if clock is not None else time.time() + cutoff = ref_time - lookback_hours * 3600 + since = _now_iso(cutoff) + max_tasks = cfg.get("max_tasks_per_night", 40) + max_sessions = cfg.get("max_sessions_per_night", 0) or max_tasks * 3 + candidate_limit = max_tasks + if target_filter: + candidate_limit = max(max_tasks, max_tasks * 3) + _progress( + cfg, + f"harvest start: source={cfg.get('transcript_source')} max_sessions={max_sessions}", + ) + digests = harvest_for_config( + cfg, + since_iso=since, + limit=max_sessions, + ) + n_sessions = len(digests) + _progress(cfg, f"harvest done: sessions={n_sessions}") + # When a real backend is configured, use it to mine checkable tasks from + # the transcripts (rubric/rule judges); otherwise fall back to the + # heuristic miner (no API, no checkable reference). + llm_miner = None + if cfg.get("backend", "mock") != "mock" and cfg.get("llm_mine", True): + try: + from skillopt_sleep.llm_miner import make_llm_miner + llm_miner = make_llm_miner( + backend, + max_sessions=max_sessions, + max_tasks=candidate_limit, + ) + except Exception: + llm_miner = None + _progress( + cfg, + f"mine start: max_tasks={max_tasks} candidate_limit={candidate_limit} " + f"llm_mine={llm_miner is not None} target_filter={target_filter}", + ) + tasks = mine( + digests, + max_tasks=max_tasks, + candidate_limit=candidate_limit, + holdout_fraction=cfg.get("holdout_fraction", 0.34), + seed=cfg.get("seed", 42), + llm_miner=llm_miner, + target_skill_text=raw_skill if target_filter else "", + target_skill_path=live_skill_path if target_filter else "", + ) + _progress(cfg, f"mine done: tasks={len(tasks)}") + + report = SleepReport( + night=night, project=project, started_at=started, + n_sessions=n_sessions, n_tasks=len(tasks), + ) + + if not tasks: + report.ended_at = _now_iso(clock) + report.notes.append("no tasks mined — nothing to consolidate") + state.set_last_harvest(project, started) + state.record_night({"night": night, "accepted": False, "n_tasks": 0}) + if not dry_run: + state.save() + staging_dir = "" + return CycleOutcome(report, staging_dir, False, []) + + # ── 3+4. replay + consolidate (gate), with opt-in dream + recall ────── + # recall pulls similar past tasks from the persisted archive; dream_rollouts + # / dream_factor enrich the training signal. With the defaults (recall_k=0, + # dream_rollouts=1, dream_factor=0) this is exactly the prior single-shot + # consolidate — behavior is unchanged unless the user opts in. + _progress(cfg, "consolidate start") + recall_k = int(cfg.get("recall_k", 0) or 0) + history_tasks = [] + if recall_k > 0: + history_tasks = [TaskRecord.from_dict(d) for d in state.task_archive()] + result = dream_consolidate( + backend, tasks, skill, memory, + history_tasks=history_tasks, + recall_k=recall_k, + dream_rollouts=int(cfg.get("dream_rollouts", 1) or 1), + dream_factor=int(cfg.get("dream_factor", 0) or 0), + edit_budget=cfg.get("edit_budget", 4), + gate_metric=cfg.get("gate_metric", "mixed"), + gate_mixed_weight=cfg.get("gate_mixed_weight", 0.5), + gate_mode=cfg.get("gate_mode", "on"), + evolve_skill=cfg.get("evolve_skill", True), + evolve_memory=cfg.get("evolve_memory", True), + night=night, + ) + # archive tonight's real (non-dream) tasks so future nights can recall them + state.add_to_archive([t.to_dict() for t in tasks if t.origin != "dream"]) + _progress( + cfg, + f"consolidate done: gate={result.gate_action} accepted={result.accepted} " + f"edits={len(result.applied_edits)} rejected={len(result.rejected_edits)}", + ) + + report.n_replayed = len(tasks) + report.baseline_score = result.baseline_score + report.candidate_score = result.candidate_score + report.accepted = result.accepted + report.gate_action = result.gate_action + report.no_edits_reason = getattr(result, "no_edits_reason", "") + report.edits = result.applied_edits + report.rejected_edits = result.rejected_edits + report.tokens_used = backend.tokens_used() + report.ended_at = _now_iso(clock) + + # ── 5. stage (unless dry-run) ──────────────────────────────────────── + staging_dir = "" + adopted = False + adopted_paths: List[str] = [] + if not dry_run: + _progress(cfg, "staging start") + report_md = _render_report_md(report, cfg) + proposed_skill = result.new_skill if (cfg.get("evolve_skill") and result.accepted) else None + proposed_memory = result.new_memory if (cfg.get("evolve_memory") and result.accepted) else None + staging_dir = write_staging( + project, + report=report, + proposed_skill=proposed_skill, + proposed_memory=proposed_memory, + live_skill_path=live_skill_path, + live_memory_path=live_memory_path, + report_md=report_md, + ) + # Observability: persist per-task held-out evidence + optimizer/codex errors so a + # 0.0->0.0 night self-explains (empty responses vs failing checks vs no edits) — the + # cycle previously captured none of this, making the gate a black box (#learning-stall). + try: + import json as _json + # Backend stderr / optimizer replies / task responses can carry + # credentials (e.g. a codex 401 stderr dump), so scrub secret-looking + # substrings before persisting them to the on-disk diagnostics. + with open(os.path.join(staging_dir, "diagnostics.json"), "w", encoding="utf-8") as _fh: + _json.dump({ + "night": night, + "backend": cfg.get("backend"), + "gate_mode": cfg.get("gate_mode"), + "n_tasks": len(tasks), + "baseline_score": result.baseline_score, + "candidate_score": result.candidate_score, + "accepted": result.accepted, + "n_applied_edits": len(result.applied_edits), + "n_rejected_edits": len(result.rejected_edits), + "call_error": redact_secrets(getattr(result, "call_error", "")), + "reflect_raw_head": redact_secrets( + (getattr(result, "reflect_raw", "") or "")[:1200] + ), + "holdout_detail": redact_secrets(getattr(result, "holdout_detail", [])), + }, _fh, indent=2) + except Exception: + pass + state.set_last_harvest(project, started) + state.record_night({ + "night": night, "accepted": result.accepted, + "baseline": result.baseline_score, "candidate": result.candidate_score, + "n_tasks": len(tasks), "staging": staging_dir, + }) + # ── 6. adopt (opt-in) ──────────────────────────────────────────── + if cfg.get("auto_adopt") and result.accepted: + adopted_paths = adopt_staging(staging_dir) + adopted = bool(adopted_paths) + state.save() + + return CycleOutcome(report, staging_dir, adopted, adopted_paths) diff --git a/skillopt_sleep/dream.py b/skillopt_sleep/dream.py new file mode 100644 index 0000000..28ee79c --- /dev/null +++ b/skillopt_sleep/dream.py @@ -0,0 +1,138 @@ +"""SkillOpt-Sleep — dream + associative recall for nightly consolidation. + +Two opt-in mechanisms (both default OFF, so the cycle is unchanged unless the +user enables them) that the deployment experiments validated: + + * dream rollouts — run each task K times and learn from the good-vs-bad + contrast (set ``dream_rollouts > 1``). Stronger signal than one failure. + * associative recall — each night, pull the K past tasks most similar to + tonight's new ones into the dream (set ``recall_k > 0``). Replays relevant + experience without re-running the whole history. + +``dream_consolidate`` wires recall + synthetic augmentation + multi-rollout +consolidation and is called by BOTH the shipped plugin cycle and the benchmark +experiment harness, so the reported numbers exercise the exact code the plugin +runs. Pure-stdlib, zero research/private dependency. +""" +from __future__ import annotations + +import re +from typing import List, Optional + +from skillopt_sleep.consolidate import ConsolidationResult, consolidate +from skillopt_sleep.types import TaskRecord + + +# ── synthetic augmentation ("dream up" variants of today's tasks) ───────────── + +_WRAPPERS = [ + "(quick one) {q}", + "Please handle this request: {q}", + "For the daily report: {q}", +] + + +def dream_augment(real_tasks: List[TaskRecord], *, factor: int = 1) -> List[TaskRecord]: + """Create synthetic TRAIN variants of real tasks (origin='dream'). + + A light, deterministic rephrasing. Dream tasks are training-only — they + carry split='train' and never enter the val/test slices the gate scores on. + """ + out: List[TaskRecord] = [] + for t in real_tasks: + for k in range(max(0, factor)): + w = _WRAPPERS[k % len(_WRAPPERS)] + out.append(TaskRecord( + id=f"{t.id}_dream{k}", project=t.project, + intent=w.format(q=t.intent), context_excerpt=t.context_excerpt, + reference_kind=t.reference_kind, reference=t.reference, + judge=dict(t.judge), system=t.system, + tags=list(t.tags) + ["dream"], split="train", + origin="dream", derived_from=t.id, + )) + return out + + +# ── associative recall (experience replay of similar past tasks) ────────────── + +def _tokens(text: str) -> set: + return {w for w in re.findall(r"[a-z0-9]+", (text or "").lower()) if len(w) > 2} + + +def recall_similar(new_tasks: List[TaskRecord], history: List[TaskRecord], + k: int) -> List[TaskRecord]: + """Return the ``k`` historical tasks most lexically similar to any of + tonight's ``new_tasks`` (max Jaccard token overlap). Recalled tasks are + returned as training material (split='train'); deterministic, stdlib-only. + """ + if not history or k <= 0 or not new_tasks: + return [] + new_tok = [_tokens(t.intent) for t in new_tasks] + new_ids = {t.id for t in new_tasks} + scored = [] + for h in history: + if h.id in new_ids: + continue + ht = _tokens(h.intent) + if not ht: + continue + sim = max(((len(ht & nt) / len(ht | nt)) if (ht | nt) else 0.0) for nt in new_tok) + scored.append((sim, h.id, h)) + scored.sort(key=lambda x: (-x[0], x[1])) + out = [] + for sim, _id, h in scored[:max(0, k)]: + if sim <= 0.0: + break + # recall as training material; copy so the source archive is untouched + out.append(TaskRecord( + id=f"recall:{h.id}", project=h.project, intent=h.intent, + context_excerpt=h.context_excerpt, reference_kind=h.reference_kind, + reference=h.reference, judge=dict(h.judge), system=h.system, + tags=list(h.tags) + ["recall"], split="train", origin="real", + derived_from=h.id, + )) + return out + + +# ── the shared nightly consolidation step ───────────────────────────────────── + +def dream_consolidate( + backend, + tasks: List[TaskRecord], + skill: str, + memory: str, + *, + history_tasks: Optional[List[TaskRecord]] = None, + recall_k: int = 0, + dream_rollouts: int = 1, + dream_factor: int = 0, + edit_budget: int = 4, + gate_metric: str = "mixed", + gate_mixed_weight: float = 0.5, + gate_mode: str = "on", + evolve_skill: bool = True, + evolve_memory: bool = True, + night: int = 1, +) -> ConsolidationResult: + """Recall similar past experience + dream synthetic variants, then run one + gated consolidation epoch over the enlarged training pool. + + ``tasks`` is the split-tagged pool for tonight (train + val); recall and + augmentation only enlarge the TRAIN split, so the val slice the gate scores + on is never polluted. With ``recall_k=0`` and ``dream_rollouts=1`` (the + defaults) this is exactly the previous single-shot ``consolidate``. + """ + train = [t for t in tasks if t.split == "train"] + enlarged = list(tasks) + if recall_k > 0 and history_tasks: + enlarged += recall_similar(train, history_tasks, recall_k) + if dream_factor > 0: + seed = [t for t in enlarged if t.split == "train" and t.origin != "dream"] + enlarged += dream_augment(seed, factor=dream_factor) + return consolidate( + backend, enlarged, skill, memory, + edit_budget=edit_budget, gate_metric=gate_metric, + gate_mixed_weight=gate_mixed_weight, gate_mode=gate_mode, + rollouts_k=dream_rollouts, evolve_skill=evolve_skill, + evolve_memory=evolve_memory, night=night, + ) diff --git a/skillopt_sleep/experiments/__init__.py b/skillopt_sleep/experiments/__init__.py new file mode 100644 index 0000000..fa657fe --- /dev/null +++ b/skillopt_sleep/experiments/__init__.py @@ -0,0 +1 @@ +"""SkillOpt-Sleep experiments.""" diff --git a/skillopt_sleep/experiments/gbrain_bench.py b/skillopt_sleep/experiments/gbrain_bench.py new file mode 100644 index 0000000..49261d6 --- /dev/null +++ b/skillopt_sleep/experiments/gbrain_bench.py @@ -0,0 +1,119 @@ +"""SkillOpt-Sleep — gbrain-evals benchmark adapter. + +Loads gbrain-evals' `skillopt-v1` benchmark (deficient skills + train/held-out +task sets with rule-based judges) into our TaskRecord format, so we can run the +SkillOpt-Sleep cycle against the SAME suite gbrain publishes a scorecard for: + + docs/benchmarks/2026-06-03-skillopt.md — "4/4 skills 0 -> 1.00" + +Each gbrain seed dir has: + SKILL.md — the deliberately deficient starting skill + benchmark.jsonl — training tasks {task_id, task, judge:{kind:"rule",checks}} + held-out.jsonl — held-out tasks (same judge shape, unseen items) + +We map: + benchmark.jsonl -> TaskRecords with split="replay" + held-out.jsonl -> TaskRecords with split="holdout" + judge -> TaskRecord.judge (+ reference_kind="rule") + +This lets us reproduce gbrain's headline result with our engine and either the +claude or codex backend, scoring locally via skillopt_sleep.judges (no judge API). +""" +from __future__ import annotations + +import json +import os +from typing import Dict, List, Optional, Tuple + +from skillopt_sleep.types import TaskRecord + + +SEED_DIRS = { + "brief-writer": "seed-missing-structure", + "thorough-analyst": "seed-verbose", + "advisor": "seed-no-verdict", + "quick-answerer": "seed-no-brain-first", +} + + +def _load_jsonl(path: str) -> List[dict]: + out: List[dict] = [] + if not os.path.exists(path): + return out + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + try: + out.append(json.loads(line)) + except Exception: + pass + return out + + +def _to_task(rec: dict, *, seed: str, split: str) -> TaskRecord: + return TaskRecord( + id=f"{seed}:{rec.get('task_id', '')}", + project=f"gbrain/{seed}", + intent=str(rec.get("task", "")), + reference_kind="rule", + judge=rec.get("judge", {}) or {}, + tags=[f"seed:{seed}"], + split=split, + ) + + +def load_seed(data_root: str, seed: str, *, val_fraction: float = 0.34, + split_seed: int = 42) -> Tuple[str, List[TaskRecord]]: + """Return (deficient_skill_md, tasks) for one gbrain seed. + + Faithful split mapping: + * gbrain held-out.jsonl -> our ``test`` (the true final measure) + * gbrain benchmark.jsonl -> split deterministically into ``train`` + ``val`` + (val gates updates; train drives reflect) + All tasks are origin='real' (gbrain provides no synthetic tasks). + """ + import hashlib + sub = SEED_DIRS.get(seed, seed) + seed_dir = os.path.join(data_root, sub) + skill_path = os.path.join(seed_dir, "SKILL.md") + skill = "" + if os.path.exists(skill_path): + with open(skill_path, encoding="utf-8") as f: + skill = f.read() + tasks: List[TaskRecord] = [] + # benchmark pool -> train/val + val_cut = int(round(val_fraction * 100)) + for rec in _load_jsonl(os.path.join(seed_dir, "benchmark.jsonl")): + t = _to_task(rec, seed=seed, split="train") + bucket = int(hashlib.sha256((str(split_seed) + t.id).encode()).hexdigest(), 16) % 100 + t.split = "val" if bucket < val_cut else "train" + tasks.append(t) + # held-out -> test + for rec in _load_jsonl(os.path.join(seed_dir, "held-out.jsonl")): + tasks.append(_to_task(rec, seed=seed, split="test")) + # guarantee a non-empty val + if not any(t.split == "val" for t in tasks): + train_only = [t for t in tasks if t.split == "train"] + if train_only: + train_only[0].split = "val" + return skill, tasks + + +def available_seeds(data_root: str) -> List[str]: + return [s for s, sub in SEED_DIRS.items() + if os.path.isdir(os.path.join(data_root, sub))] + + +def find_data_root(explicit: str = "") -> Optional[str]: + """Locate eval/data/skillopt-v1 from common clone locations.""" + cands = [explicit] if explicit else [] + cands += [ + os.path.expanduser("~/git/gbrain-evals/eval/data/skillopt-v1"), + "/tmp/gbrain-evals/eval/data/skillopt-v1", + os.path.expanduser("~/gbrain-evals/eval/data/skillopt-v1"), + ] + for c in cands: + if c and os.path.isdir(c): + return c + return None diff --git a/skillopt_sleep/experiments/personas.py b/skillopt_sleep/experiments/personas.py new file mode 100644 index 0000000..72eb6af --- /dev/null +++ b/skillopt_sleep/experiments/personas.py @@ -0,0 +1,86 @@ +"""SkillOpt-Sleep — persona task fixtures for the validation experiment. + +Each persona is a list of TaskRecords with EXACT checkable references and a +`rule:` tag naming the single skill rule that makes the task solvable +(consumed by MockBackend). This lets the experiment prove — deterministically, +with no API — that nightly consolidation lifts a held-out score and that the +gate blocks regressions. + +Personas mirror the user's framing: programmer / researcher / analyst. +""" +from __future__ import annotations + +from typing import List + +from skillopt_sleep.types import TaskRecord + + +def _t(i, intent, ref, rule, project="/personas/demo", outcome="fail") -> TaskRecord: + return TaskRecord( + id=f"persona_{rule}_{i}", + project=project, + intent=intent, + context_excerpt="", + attempted_solution="", + outcome=outcome, + reference_kind="exact", + reference=ref, + tags=[f"rule:{rule}"], + source_sessions=[f"sess_{i}"], + ) + + +def researcher_persona() -> List[TaskRecord]: + """Researcher who always wants arXiv ids wrapped in tags.""" + items = [ + ("Give me the arXiv id for the SkillOpt paper", "arXiv:2605.23904"), + ("What's the arXiv id of the Attention paper?", "arXiv:1706.03762"), + ("arXiv id for the GAN paper?", "arXiv:1406.2661"), + ("arXiv id for BERT?", "arXiv:1810.04805"), + ("arXiv id for the ResNet paper?", "arXiv:1512.03385"), + ("arXiv id for the Adam optimizer paper?", "arXiv:1412.6980"), + ("arXiv id for Dropout?", "arXiv:1207.0580"), + ("arXiv id for the Transformer-XL paper?", "arXiv:1901.02860"), + ("arXiv id for word2vec?", "arXiv:1301.3781"), + ("arXiv id for the VAE paper?", "arXiv:1312.6114"), + ("arXiv id for batch norm?", "arXiv:1502.03167"), + ("arXiv id for GPT-3?", "arXiv:2005.14165"), + ] + # Both rules required: format the id (arxiv-id) AND wrap in answer tags. + out: List[TaskRecord] = [] + for i, (q, a) in enumerate(items): + t = _t(i, q, a, "wrap-answer") + t.tags = ["rule:wrap-answer", "rule:arxiv-id"] + out.append(t) + return out + + +def programmer_persona() -> List[TaskRecord]: + """Programmer who wants imperative-mood commit subjects.""" + items = [ + ("commit message for adding a login form", "Add login form"), + ("commit message for fixing the null pointer bug", "Fix null pointer in parser"), + ("commit message for updating the README", "Update README"), + ("commit message for removing dead code", "Remove dead code"), + ("commit message for bumping the version", "Bump version to 1.2.0"), + ("commit message for refactoring the auth module", "Refactor auth module"), + ("commit message for adding tests", "Add unit tests for scheduler"), + ("commit message for fixing the CI pipeline", "Fix CI pipeline"), + ] + return [_t(i, q, a, "commit-imperative") for i, (q, a) in enumerate(items)] + + +def harmful_edit_task() -> TaskRecord: + """A task whose 'fix' is a known-bad rule; used to prove the gate rejects + regressions. The MockBackend proposes the harmful rule on this failure, + but applying it does NOT raise the held-out score, so the gate must reject. + """ + t = _t(99, "answer this freely", "THIS_WILL_NOT_MATCH", "__harmful__") + t.reference = "an-answer-that-the-harmful-rule-cannot-produce" + return t + + +PERSONAS = { + "researcher": researcher_persona, + "programmer": programmer_persona, +} diff --git a/skillopt_sleep/experiments/report.py b/skillopt_sleep/experiments/report.py new file mode 100644 index 0000000..767ea65 --- /dev/null +++ b/skillopt_sleep/experiments/report.py @@ -0,0 +1,132 @@ +"""SkillOpt-Sleep — turn a sweep JSONL into a presented Markdown scorecard. + +Usage: + python -m skillopt_sleep.experiments.report --in docs/sleep/sweep.jsonl \ + --out docs/sleep/benchmark_report.md +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from typing import Any, Dict, List + + +def _load(path: str) -> List[Dict[str, Any]]: + rows = [] + if os.path.exists(path): + with open(path) as f: + for line in f: + line = line.strip() + if line: + try: + rows.append(json.loads(line)) + except Exception: + pass + return rows + + +def _fmt_model(backend: str, model: str) -> str: + m = model or "default" + return f"{backend}:{m}" + + +def render(rows: List[Dict[str, Any]]) -> str: + direct = [r for r in rows if r.get("cfg", {}).get("kind") in ("direct", "dual") and "error" not in r] + transfer = [r for r in rows if r.get("cfg", {}).get("kind") == "transfer" and "error" not in r] + errors = [r for r in rows if "error" in r] + + out: List[str] = [] + out.append("# SkillOpt-Sleep — benchmark report") + out.append("") + out.append("Auto-generated from `sweep.jsonl`. Benchmark: " + "[gbrain-evals](https://github.com/garrytan/gbrain-evals) `skillopt-v1` " + "(deficient skills, train/held-out split, local rule judge — no judge-API).") + out.append("Held-out scores are computed by the harness, not the optimizer.") + out.append("") + + # ── direct improvement table ────────────────────────────────────────── + out.append("## Direct improvement (optimize, then deploy)") + out.append("") + out.append("| Optimizer → Target | Seed | Held-out before | Held-out after | Nights | Tokens |") + out.append("|---|---|---|---|---|---|") + for r in direct: + c = r["cfg"] + if c.get("kind") == "dual": + label = (f"{_fmt_model(c['optimizer_backend'], c.get('optimizer_model',''))}" + f" → {_fmt_model(c['target_backend'], c.get('target_model',''))}") + else: + m = _fmt_model(c["backend"], c.get("model", "")) + label = f"{m} → {m}" + out.append(f"| {label} | {c['seed']} | " + f"{r['baseline']:.2f} | **{r['after']:.2f}** | {c['nights']} | " + f"{r.get('tokens','?')} |") + if direct: + n_imp = sum(1 for r in direct if r.get("improved")) + out.append("") + out.append(f"**{n_imp}/{len(direct)} configurations improved on held-out.**") + out.append("") + + # ── transfer table ──────────────────────────────────────────────────── + if transfer: + out.append("## Cross-model transfer (optimize on SOURCE, deploy frozen on TARGET)") + out.append("") + out.append("The price-difference story: spend cheap tokens optimizing overnight, " + "then deploy the frozen skill on any model with no further optimization.") + out.append("") + out.append("| Source (optimizer) | Target (deploy) | Seed | Target baseline | Transferred | Gain |") + out.append("|---|---|---|---|---|---|") + for r in transfer: + c = r["cfg"] + s = _fmt_model(c["source_backend"], c.get("source_model", "")) + t = _fmt_model(c["target_backend"], c.get("target_model", "")) + out.append(f"| {s} | {t} | {c['seed']} | {r['baseline_target']:.2f} | " + f"**{r['transferred']:.2f}** | {r['transfer_gain']:+.2f} |") + n_pos = sum(1 for r in transfer if r.get("transfer_gain", 0) > 0) + out.append("") + out.append(f"**{n_pos}/{len(transfer)} transfers were positive** " + "(frozen skill helped a different model than it was optimized on).") + out.append("") + + # ── errors (honest reporting) ───────────────────────────────────────── + if errors: + out.append("## Configs that errored (reported, not hidden)") + out.append("") + for r in errors: + out.append(f"- `{json.dumps(r['cfg'])}` → {r['error']}") + out.append("") + + out.append("## How to reproduce") + out.append("") + out.append("```bash") + out.append("git clone https://github.com/garrytan/gbrain-evals /tmp/gbrain-evals") + out.append("python -m skillopt_sleep.experiments.sweep --plan full \\") + out.append(" --data-root /tmp/gbrain-evals/eval/data/skillopt-v1 --out docs/sleep/sweep.jsonl") + out.append("python -m skillopt_sleep.experiments.report \\") + out.append(" --in docs/sleep/sweep.jsonl --out docs/sleep/benchmark_report.md") + out.append("```") + out.append("") + return "\n".join(out) + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description="Render SkillOpt-Sleep sweep report") + ap.add_argument("--in", dest="inp", default="docs/sleep/sweep.jsonl") + ap.add_argument("--out", default="docs/sleep/benchmark_report.md") + args = ap.parse_args(argv) + + rows = _load(args.inp) + if not rows: + print(f"no rows in {args.inp}", file=sys.stderr) + return 1 + md = render(rows) + os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True) + with open(args.out, "w") as f: + f.write(md) + print(f"wrote {args.out} ({len(rows)} rows)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skillopt_sleep/experiments/run_experiment.py b/skillopt_sleep/experiments/run_experiment.py new file mode 100644 index 0000000..1110f26 --- /dev/null +++ b/skillopt_sleep/experiments/run_experiment.py @@ -0,0 +1,178 @@ +"""SkillOpt-Sleep — validation experiment. + +Answers the question the user posed: *does nightly offline self-evolution +actually improve the agent?* Runs deterministically with the MockBackend +(no API key, reproducible) and is the acceptance test for the whole idea. + +What it proves: + 1. MONOTONIC LIFT — over N sleep nights, the held-out score rises from a + baseline (empty skill/memory) toward 1.0 as the gate accepts the + general rules the persona's tasks require. + 2. GATE SAFETY — an injected harmful edit is REJECTED (held-out score does + not improve), so a bad nightly proposal can never be adopted. + 3. PLUMBING — harvest->mine->replay->consolidate->stage->adopt all run and + the adopted artifact, re-scored, retains the lift. + +Run: + python -m skillopt_sleep.experiments.run_experiment + python -m skillopt_sleep.experiments.run_experiment --persona programmer --nights 3 + python -m skillopt_sleep.experiments.run_experiment --backend anthropic # real lift +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +import tempfile +from typing import List + +from skillopt_sleep.backend import get_backend +from skillopt_sleep.consolidate import consolidate +from skillopt_sleep.experiments.personas import ( + PERSONAS, + harmful_edit_task, + researcher_persona, +) +from skillopt_sleep.memory import ensure_skill_scaffold +from skillopt_sleep.replay import aggregate_scores, replay_batch +from skillopt_sleep.types import TaskRecord + + +def _score_holdout(backend, tasks: List[TaskRecord], skill: str, memory: str, + metric: str = "mixed", w: float = 0.5) -> float: + from skillopt_sleep.consolidate import select_gate_score + # the persona experiment uses a 2-way split (train/val, no test); score on val + holdout = [t for t in tasks if t.split in ("val", "holdout")] or tasks + pairs = replay_batch(backend, holdout, skill, memory) + h, s = aggregate_scores(pairs) + return select_gate_score(h, s, metric, w) + + +def run(persona: str = "researcher", nights: int = 4, backend_name: str = "mock", + edit_budget: int = 4, seed: int = 42, model: str = "", codex_path: str = "", + limit_tasks: int = 0) -> dict: + from skillopt_sleep.mine import assign_splits + + make = PERSONAS.get(persona, researcher_persona) + items = make() + if limit_tasks and limit_tasks < len(items): + items = items[:limit_tasks] + tasks = assign_splits(items, holdout_fraction=0.34, seed=seed) + backend = get_backend(backend_name, model=model, codex_path=codex_path) + is_mock = (backend.name == "mock") + + # start from an empty managed skill + empty memory + skill = ensure_skill_scaffold("", name="skillopt-sleep-learned", + description="Learned preferences.") + memory = "" + + baseline = _score_holdout(backend, tasks, skill, memory) + trace = [{"night": 0, "holdout_score": round(baseline, 4), "action": "baseline", + "n_edits": 0}] + + for night in range(1, nights + 1): + res = consolidate( + backend, tasks, skill, memory, + edit_budget=edit_budget, gate_metric="mixed", gate_mixed_weight=0.5, + evolve_skill=True, evolve_memory=True, night=night, + ) + if res.accepted: + skill, memory = res.new_skill, res.new_memory + trace.append({ + "night": night, + "holdout_score": round(res.candidate_score, 4), + "action": res.gate_action, + "accepted": res.accepted, + "n_edits": len(res.applied_edits), + "edits": [e.content for e in res.applied_edits], + "n_rejected": len(res.rejected_edits), + }) + # converged: stop early if perfect + if res.candidate_score >= 0.999: + break + + after = _score_holdout(backend, tasks, skill, memory) + + # ── gate-safety probe (mock only; it relies on the mock's known bad rule) ── + harmful_rejected = None + if is_mock: + harmful_tasks = assign_splits([harmful_edit_task()] + make()[:3], + holdout_fraction=0.5, seed=seed) + _ = _score_holdout(backend, harmful_tasks, skill, memory) + res_h = consolidate(backend, harmful_tasks, skill, memory, + edit_budget=edit_budget, gate_metric="mixed", + evolve_skill=True, evolve_memory=False, night=nights + 1) + harmful_rule_text = get_backend("mock").RULE_TEXT["__harmful__"] # type: ignore[attr-defined] + harmful_rejected = (harmful_rule_text not in res_h.new_skill) + + result = { + "persona": persona, + "backend": backend.name, + "model": model or "(default)", + "n_tasks": len(tasks), + "nights_run": len(trace) - 1, + "baseline_holdout": round(baseline, 4), + "after_holdout": round(after, 4), + "lift": round(after - baseline, 4), + "improved": after > baseline, + "gate_blocks_harmful": harmful_rejected, # None for real backends + "tokens_used": backend.tokens_used(), + "final_skill_excerpt": skill[-500:], + "trace": trace, + } + return result + + +def _assert(cond: bool, msg: str) -> None: + if not cond: + print(f"FAIL: {msg}") + raise SystemExit(1) + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description="SkillOpt-Sleep validation experiment") + ap.add_argument("--persona", default="researcher", choices=list(PERSONAS.keys())) + ap.add_argument("--nights", type=int, default=4) + ap.add_argument("--backend", default="mock", choices=["mock", "claude", "codex", "copilot"]) + ap.add_argument("--model", default="", help="backend model override") + ap.add_argument("--codex-path", default="", help="path to the real @openai/codex binary") + ap.add_argument("--edit-budget", type=int, default=4) + ap.add_argument("--limit-tasks", type=int, default=0, help="cap #tasks (control API cost)") + ap.add_argument("--json", action="store_true") + ap.add_argument("--assert-improves", action="store_true", + help="exit nonzero unless lift>0 (and, for mock, gate blocks harmful edit)") + args = ap.parse_args(argv) + + res = run(args.persona, nights=args.nights, backend_name=args.backend, + edit_budget=args.edit_budget, model=args.model, + codex_path=args.codex_path, limit_tasks=args.limit_tasks) + + if args.json: + print(json.dumps(res, ensure_ascii=False, indent=2)) + else: + print(f"=== SkillOpt-Sleep experiment: persona={res['persona']} " + f"backend={res['backend']} model={res['model']} ===") + print(f"tasks: {res['n_tasks']} tokens(approx): {res['tokens_used']}") + print(f"baseline held-out : {res['baseline_holdout']}") + print(f"after held-out : {res['after_holdout']} (lift {res['lift']:+.4f})") + if res["gate_blocks_harmful"] is not None: + print(f"gate blocks harmful edit: {res['gate_blocks_harmful']}") + print("trace:") + for row in res["trace"]: + edits = "; ".join(row.get("edits", []))[:80] + print(f" night {row['night']}: holdout={row['holdout_score']} " + f"{row['action']} (+{row['n_edits']} edits) {edits}") + + if args.assert_improves: + _assert(res["improved"], "held-out score did not improve") + if res["gate_blocks_harmful"] is not None: + _assert(res["gate_blocks_harmful"], "gate failed to block harmful edit") + print("\nPASS: nightly consolidation improves held-out score AND gate blocks regressions.") + else: + print("\nPASS: nightly consolidation improves held-out score (real backend).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skillopt_sleep/experiments/run_gbrain.py b/skillopt_sleep/experiments/run_gbrain.py new file mode 100644 index 0000000..43c7acd --- /dev/null +++ b/skillopt_sleep/experiments/run_gbrain.py @@ -0,0 +1,209 @@ +"""SkillOpt-Sleep — run the gbrain-evals skillopt-v1 benchmark with our engine. + +Reproduces gbrain's "Result 1 — skills measurably improve" scorecard +(docs/benchmarks/2026-06-03-skillopt.md) using SkillOpt-Sleep's +consolidate() loop and either the claude or codex backend. + +For each deficient seed skill: + 1. score the held-out tasks with the ORIGINAL skill -> before + 2. run N consolidation nights on the training tasks (gated) -> evolve skill + 3. score the held-out tasks with the EVOLVED skill -> after + +Held-out scoring is done locally by the rule judge (no judge API). Only the +agent's `attempt` (and the optimizer's `reflect`) spend tokens. + +Usage: + python -m skillopt_sleep.experiments.run_gbrain --backend mock + python -m skillopt_sleep.experiments.run_gbrain --backend claude --seeds brief-writer --nights 2 + python -m skillopt_sleep.experiments.run_gbrain --backend codex --data-root /tmp/gbrain-evals/eval/data/skillopt-v1 +""" +from __future__ import annotations + +import argparse +import json +import sys +from typing import Dict, List, Optional + +from skillopt_sleep.backend import build_backend, get_backend +from skillopt_sleep.consolidate import consolidate, select_gate_score +from skillopt_sleep.experiments.gbrain_bench import ( + available_seeds, + find_data_root, + load_seed, +) +from skillopt_sleep.replay import aggregate_scores, replay_batch + + +def _score(backend, tasks, skill, memory, split="test", metric="mixed", w=0.5): + sub = [t for t in tasks if t.split == split] + if not sub: # fall back to val, then everything, so we never score on nothing + sub = [t for t in tasks if t.split == "val"] or tasks + pairs = replay_batch(backend, sub, skill, memory) + h, s = aggregate_scores(pairs) + return h, s, select_gate_score(h, s, metric, w) + + +def run_seed(backend, seed: str, skill: str, tasks: List, *, + nights: int = 3, edit_budget: int = 4, gate_mode: str = "on", + slow_update: bool = True, rollouts_k: int = 1, + limit_replay: int = 0, limit_holdout: int = 0) -> dict: + memory = "" + # optionally cap each split to control API cost / latency. + # limit_replay caps train; limit_holdout caps BOTH val and test. + if limit_replay or limit_holdout: + train = [t for t in tasks if t.split == "train"] + val = [t for t in tasks if t.split == "val"] + test = [t for t in tasks if t.split == "test"] + if limit_replay: + train = train[:limit_replay] + if limit_holdout: + val = val[:limit_holdout] + test = test[:limit_holdout] + tasks = train + val + test + # final measure is TEST (the gbrain held-out set); val gates internally + bh, bs, bscore = _score(backend, tasks, skill, memory, split="test") + trace = [{"night": 0, "test_hard": round(bh, 3), "action": "baseline"}] + cur = skill + first_night_skill = skill + for night in range(1, nights + 1): + res = consolidate( + backend, tasks, cur, memory, + edit_budget=edit_budget, gate_metric="mixed", gate_mixed_weight=0.5, + gate_mode=gate_mode, rollouts_k=rollouts_k, + evolve_skill=True, evolve_memory=False, night=night, + ) + if res.accepted: + cur = res.new_skill + if night == 1: + first_night_skill = cur + # report the TEST score each night (independent of the val gate) + th, _ts, _ = _score(backend, tasks, cur, memory, split="test") + trace.append({ + "night": night, + "val_hard": round(res.holdout_candidate, 3), + "test_hard": round(th, 3), + "action": res.gate_action, + "accepted": res.accepted, + "edits": [e.content for e in res.applied_edits], + }) + if th >= 0.999: + break + + # ── SLOW UPDATE: consolidate cross-night experience into the protected + # long-term field. Runs regardless of gate mode (it is what preserves + # long-term memory even when the gate is OFF). + slow_text = None + if nights >= 2 and slow_update: + try: + from skillopt_sleep.slow_update import run_slow_update, replace_slow_field + val_tasks = [t for t in tasks if t.split == "val"] or tasks + prev_pairs = replay_batch(backend, val_tasks, first_night_skill, memory) + curr_pairs = replay_batch(backend, val_tasks, cur, memory) + slow_text = run_slow_update( + backend, prev_skill=first_night_skill, curr_skill=cur, + prev_pairs=[(t, r) for t, r in prev_pairs], + curr_pairs=[(t, r) for t, r in curr_pairs], + ) + if slow_text: + cur = replace_slow_field(cur, slow_text) + except Exception: + slow_text = None + + ah, as_, ascore = _score(backend, tasks, cur, memory, split="test") + return { + "seed": seed, + "held_out_before": round(bh, 3), + "held_out_after": round(ah, 3), + "improved": ah > bh, + "nights": len(trace) - 1, + "trace": trace, + "slow_update": slow_text, + "final_skill_tail": cur[-400:], + } + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description="Run gbrain-evals skillopt-v1 with SkillOpt-Sleep") + ap.add_argument("--backend", default="mock", choices=["mock", "claude", "codex"]) + ap.add_argument("--model", default="") + ap.add_argument("--optimizer-backend", default="", help="route reflect/judge here (dual)") + ap.add_argument("--optimizer-model", default="") + ap.add_argument("--target-backend", default="", help="route attempt here (dual)") + ap.add_argument("--target-model", default="") + ap.add_argument("--codex-path", default="") + ap.add_argument("--data-root", default="", help="path to eval/data/skillopt-v1") + ap.add_argument("--seeds", default="", help="comma list; default = all available") + ap.add_argument("--nights", type=int, default=3) + ap.add_argument("--edit-budget", type=int, default=4) + ap.add_argument("--gate", default="on", choices=["on", "off", "hard", "soft"], + help="on/hard/soft = validation-gated; off = greedy (no hard filter)") + ap.add_argument("--rollouts-k", type=int, default=1, + help=">1 = multi-rollout contrastive reflection per task") + ap.add_argument("--budget-tokens", type=int, default=0, + help="approx token budget; auto-plans nights x rollouts when set") + ap.add_argument("--budget-minutes", type=float, default=0.0) + ap.add_argument("--preferences", default="", help="free-text user preferences (prior for reflect)") + ap.add_argument("--limit-replay", type=int, default=0, help="cap #train tasks (cost control)") + ap.add_argument("--limit-holdout", type=int, default=0, help="cap #val and #test tasks (cost control)") + ap.add_argument("--json", action="store_true") + args = ap.parse_args(argv) + + data_root = find_data_root(args.data_root) + if not data_root: + print("ERROR: could not find eval/data/skillopt-v1. Clone gbrain-evals and pass --data-root.", + file=sys.stderr) + return 2 + + seeds = [s.strip() for s in args.seeds.split(",") if s.strip()] or available_seeds(data_root) + backend = build_backend( + backend=args.backend, model=args.model, + optimizer_backend=args.optimizer_backend, optimizer_model=args.optimizer_model, + target_backend=args.target_backend, target_model=args.target_model, + codex_path=args.codex_path, preferences=args.preferences, + ) + + results = [] + for seed in seeds: + skill, tasks = load_seed(data_root, seed) + if not tasks: + continue + # budget auto-planning: derive nights x rollouts_k from a token budget + nights, rollouts_k = args.nights, args.rollouts_k + if args.budget_tokens: + from skillopt_sleep.budget import Budget, plan_depth + n_train = len([t for t in tasks if t.split == "train"]) or len(tasks) + nights, rollouts_k = plan_depth( + Budget(max_tokens=args.budget_tokens), n_tasks=n_train, + default_nights=args.nights, default_k=args.rollouts_k, + ) + if not args.json: + print(f" [budget] {args.budget_tokens} tok -> nights={nights} rollouts_k={rollouts_k}") + r = run_seed(backend, seed, skill, tasks, nights=nights, + edit_budget=args.edit_budget, rollouts_k=rollouts_k, + gate_mode=("off" if args.gate == "off" else "on"), + limit_replay=args.limit_replay, limit_holdout=args.limit_holdout) + results.append(r) + if not args.json: + print(f" {seed:<18} held-out {r['held_out_before']:.2f} -> {r['held_out_after']:.2f}" + f" ({'IMPROVED' if r['improved'] else 'no change'}, {r['nights']} nights)") + + n_improved = sum(1 for r in results if r["improved"]) + summary = { + "benchmark": "gbrain-evals/skillopt-v1", + "backend": backend.name, + "model": args.model or "(default)", + "n_seeds": len(results), + "n_improved": n_improved, + "tokens_used": backend.tokens_used(), + "results": results, + } + if args.json: + print(json.dumps(summary, ensure_ascii=False, indent=2)) + else: + print(f"\n=== {n_improved}/{len(results)} seeds improved on held-out " + f"(backend={backend.name}, ~{backend.tokens_used()} tokens) ===") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skillopt_sleep/experiments/run_transfer.py b/skillopt_sleep/experiments/run_transfer.py new file mode 100644 index 0000000..5b00ec8 --- /dev/null +++ b/skillopt_sleep/experiments/run_transfer.py @@ -0,0 +1,155 @@ +"""SkillOpt-Sleep — skill-transfer experiment (sleep scenario). + +Answers: "if I optimize a skill while the agent sleeps using a CHEAP model, +does the learned skill still help an EXPENSIVE model at deploy time?" — and the +reverse. This is the SkillOpt paper's cross-model transfer result, reproduced +in the sleep setting, and it is the core price-difference value proposition: +spend cheap tokens overnight, deploy the frozen skill anywhere. + +Protocol, per gbrain seed: + 1. baseline_target = held-out score of the DEFICIENT skill, run on TARGET model + 2. optimize the skill for N nights using the SOURCE model (attempt+reflect) + 3. transferred = held-out score of the LEARNED skill, run on TARGET model, + with NO further optimization + 4. (reference) direct = held-out score of a skill optimized AND run on TARGET + +Report baseline / direct / transferred, mirroring SkillOpt Table "transfer". + +Usage: + python -m skillopt_sleep.experiments.run_transfer \ + --source-backend claude --source-model haiku \ + --target-backend claude --target-model sonnet \ + --seeds brief-writer --nights 2 +""" +from __future__ import annotations + +import argparse +import json +import sys +from typing import List, Optional + +from skillopt_sleep.backend import get_backend +from skillopt_sleep.consolidate import consolidate, select_gate_score +from skillopt_sleep.experiments.gbrain_bench import ( + available_seeds, find_data_root, load_seed, +) +from skillopt_sleep.replay import aggregate_scores, replay_batch + + +def _holdout_hard(backend, tasks, skill, memory="") -> float: + # transfer is measured on the true held-out TEST split + ho = [t for t in tasks if t.split == "test"] + if not ho: + ho = [t for t in tasks if t.split in ("val", "holdout")] or tasks + pairs = replay_batch(backend, ho, skill, memory) + h, _s = aggregate_scores(pairs) + return h + + +def _optimize(backend, skill, tasks, *, nights, edit_budget) -> str: + cur = skill + for night in range(1, nights + 1): + res = consolidate(backend, tasks, cur, "", + edit_budget=edit_budget, gate_metric="mixed", + evolve_skill=True, evolve_memory=False, night=night) + if res.accepted: + cur = res.new_skill + if res.holdout_candidate >= 0.999: + break + return cur + + +def run_seed(seed, skill, tasks, *, source, target, nights, edit_budget, + limit_replay, limit_holdout, do_direct=True) -> dict: + if limit_replay or limit_holdout: + train = [t for t in tasks if t.split == "train"] + val = [t for t in tasks if t.split == "val"] + test = [t for t in tasks if t.split == "test"] + if limit_replay: + train = train[:limit_replay] + if limit_holdout: + val = val[:limit_holdout] + test = test[:limit_holdout] + tasks = train + val + test + + baseline_target = _holdout_hard(target, tasks, skill) + + # optimize on SOURCE, evaluate frozen skill on TARGET + learned_on_source = _optimize(source, skill, tasks, nights=nights, edit_budget=edit_budget) + transferred = _holdout_hard(target, tasks, learned_on_source) + + direct = None + if do_direct: + learned_on_target = _optimize(target, skill, tasks, nights=nights, edit_budget=edit_budget) + direct = _holdout_hard(target, tasks, learned_on_target) + + return { + "seed": seed, + "baseline_target": round(baseline_target, 3), + "direct_target": (round(direct, 3) if direct is not None else None), + "transferred": round(transferred, 3), + "transfer_gain": round(transferred - baseline_target, 3), + "learned_skill_tail": learned_on_source[-300:], + } + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description="SkillOpt-Sleep cross-model transfer") + ap.add_argument("--source-backend", default="claude") + ap.add_argument("--source-model", default="haiku") + ap.add_argument("--target-backend", default="claude") + ap.add_argument("--target-model", default="sonnet") + ap.add_argument("--codex-path", default="") + ap.add_argument("--data-root", default="") + ap.add_argument("--seeds", default="brief-writer") + ap.add_argument("--nights", type=int, default=2) + ap.add_argument("--edit-budget", type=int, default=4) + ap.add_argument("--limit-replay", type=int, default=3) + ap.add_argument("--limit-holdout", type=int, default=3) + ap.add_argument("--no-direct", action="store_true", help="skip the direct reference (saves cost)") + ap.add_argument("--json", action="store_true") + args = ap.parse_args(argv) + + data_root = find_data_root(args.data_root) + if not data_root: + print("ERROR: gbrain-evals skillopt-v1 data not found; pass --data-root", file=sys.stderr) + return 2 + + source = get_backend(args.source_backend, model=args.source_model, codex_path=args.codex_path) + target = get_backend(args.target_backend, model=args.target_model, codex_path=args.codex_path) + + seeds = [s.strip() for s in args.seeds.split(",") if s.strip()] or available_seeds(data_root) + results = [] + for seed in seeds: + skill, tasks = load_seed(data_root, seed) + if not tasks: + continue + r = run_seed(seed, skill, tasks, source=source, target=target, + nights=args.nights, edit_budget=args.edit_budget, + limit_replay=args.limit_replay, limit_holdout=args.limit_holdout, + do_direct=not args.no_direct) + results.append(r) + if not args.json: + d = f" direct={r['direct_target']}" if r['direct_target'] is not None else "" + print(f" {seed:<16} baseline={r['baseline_target']:.2f}" + f" transferred={r['transferred']:.2f}{d}" + f" (gain {r['transfer_gain']:+.2f})") + + summary = { + "experiment": "skillopt-sleep/transfer", + "source": f"{args.source_backend}:{args.source_model}", + "target": f"{args.target_backend}:{args.target_model}", + "tokens_source": source.tokens_used(), + "tokens_target": target.tokens_used(), + "results": results, + } + if args.json: + print(json.dumps(summary, ensure_ascii=False, indent=2)) + else: + print(f"\n=== transfer {summary['source']} -> {summary['target']}: " + f"{sum(1 for r in results if r['transfer_gain'] > 0)}/{len(results)} positive ===") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skillopt_sleep/experiments/sweep.py b/skillopt_sleep/experiments/sweep.py new file mode 100644 index 0000000..ddd337c --- /dev/null +++ b/skillopt_sleep/experiments/sweep.py @@ -0,0 +1,164 @@ +"""SkillOpt-Sleep — benchmark sweep driver. + +Runs many (backend, model, seed, transfer-pair) configurations SEQUENTIALLY in +one process, appending each result to a JSONL file as it finishes. Designed to +run unattended in the background; safe to interrupt (already-written rows +survive) and resume (skip configs whose row already exists). + +Then `report.py` turns the JSONL into a presented Markdown scorecard. + +Usage: + python -m skillopt_sleep.experiments.sweep --plan quick --out docs/sleep/sweep.jsonl + python -m skillopt_sleep.experiments.sweep --plan full --out docs/sleep/sweep.jsonl +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from typing import Any, Dict, List + +from skillopt_sleep.backend import build_backend, get_backend +from skillopt_sleep.experiments.gbrain_bench import find_data_root, load_seed +from skillopt_sleep.experiments.run_gbrain import run_seed as bench_seed +from skillopt_sleep.experiments.run_transfer import run_seed as transfer_seed + + +# Plans: lists of config dicts. Kept small per-run to bound cost/latency. +def _direct_cfg(backend, model, seed, nights=2): + return {"kind": "direct", "backend": backend, "model": model, "seed": seed, "nights": nights} + + +def _dual_cfg(opt_backend, opt_model, tgt_backend, tgt_model, seed, nights=2): + # a 'direct' run on a DualBackend: strong optimizer proposes, weak target runs + return {"kind": "dual", "optimizer_backend": opt_backend, "optimizer_model": opt_model, + "target_backend": tgt_backend, "target_model": tgt_model, "seed": seed, "nights": nights} + + +def _transfer_cfg(sb, sm, tb, tm, seed, nights=2): + return {"kind": "transfer", "source_backend": sb, "source_model": sm, + "target_backend": tb, "target_model": tm, "seed": seed, "nights": nights} + + +PLANS: Dict[str, List[Dict[str, Any]]] = { + # one cheap seed each, both backends — fast sanity + "quick": [ + _direct_cfg("claude", "haiku", "brief-writer", 1), + _direct_cfg("codex", "", "brief-writer", 2), + ], + # SkillOpt-faithful: STRONG optimizer (sonnet) proposes, WEAK target (haiku) + # runs — the reliable config. Plus Codex self-optimized. All 4 gbrain seeds, + # including quick-answerer (real tool loop). + "direct": [ + _dual_cfg("claude", "sonnet", "claude", "haiku", "brief-writer"), + _dual_cfg("claude", "sonnet", "claude", "haiku", "advisor"), + _dual_cfg("claude", "sonnet", "claude", "haiku", "thorough-analyst"), + _dual_cfg("claude", "sonnet", "claude", "haiku", "quick-answerer"), + _direct_cfg("codex", "", "brief-writer"), + _direct_cfg("codex", "", "advisor"), + _direct_cfg("codex", "", "quick-answerer"), + ], + # the price-difference story: optimize cheap, deploy expensive (and reverse) + "transfer": [ + _transfer_cfg("claude", "haiku", "claude", "sonnet", "brief-writer"), + _transfer_cfg("claude", "sonnet", "claude", "haiku", "brief-writer"), + _transfer_cfg("codex", "", "claude", "haiku", "brief-writer"), + _transfer_cfg("claude", "haiku", "codex", "", "brief-writer"), + ], +} +PLANS["full"] = PLANS["direct"] + PLANS["transfer"] + + +def _cfg_key(c: Dict[str, Any]) -> str: + return json.dumps({k: c[k] for k in sorted(c)}, ensure_ascii=False) + + +def _load_done(out_path: str) -> set: + done = set() + if os.path.exists(out_path): + with open(out_path) as f: + for line in f: + try: + row = json.loads(line) + if "cfg_key" in row: + done.add(row["cfg_key"]) + except Exception: + pass + return done + + +def _append(out_path: str, row: Dict[str, Any]) -> None: + os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True) + with open(out_path, "a") as f: + f.write(json.dumps(row, ensure_ascii=False) + "\n") + + +def run_one(cfg: Dict[str, Any], data_root: str, codex_path: str, + limit_replay: int, limit_holdout: int) -> Dict[str, Any]: + seed = cfg["seed"] + skill, tasks = load_seed(data_root, seed) + t0 = time.time() + if cfg["kind"] in ("direct", "dual"): + if cfg["kind"] == "dual": + be = build_backend( + optimizer_backend=cfg["optimizer_backend"], optimizer_model=cfg.get("optimizer_model", ""), + target_backend=cfg["target_backend"], target_model=cfg.get("target_model", ""), + codex_path=codex_path, + ) + else: + be = get_backend(cfg["backend"], model=cfg.get("model", ""), codex_path=codex_path) + r = bench_seed(be, seed, skill, tasks, nights=cfg["nights"], + limit_replay=limit_replay, limit_holdout=limit_holdout) + out = {"baseline": r["held_out_before"], "after": r["held_out_after"], + "improved": r["improved"], "tokens": be.tokens_used()} + else: + src = get_backend(cfg["source_backend"], model=cfg.get("source_model", ""), codex_path=codex_path) + tgt = get_backend(cfg["target_backend"], model=cfg.get("target_model", ""), codex_path=codex_path) + r = transfer_seed(seed, skill, tasks, source=src, target=tgt, nights=cfg["nights"], + edit_budget=4, limit_replay=limit_replay, limit_holdout=limit_holdout, + do_direct=False) + out = {"baseline_target": r["baseline_target"], "transferred": r["transferred"], + "transfer_gain": r["transfer_gain"], + "tokens": src.tokens_used() + tgt.tokens_used()} + out.update({"cfg": cfg, "cfg_key": _cfg_key(cfg), "elapsed_s": round(time.time() - t0, 1)}) + return out + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description="SkillOpt-Sleep benchmark sweep") + ap.add_argument("--plan", default="quick", choices=list(PLANS.keys())) + ap.add_argument("--out", default="docs/sleep/sweep.jsonl") + ap.add_argument("--data-root", default="") + ap.add_argument("--codex-path", default="") + ap.add_argument("--limit-replay", type=int, default=3) + ap.add_argument("--limit-holdout", type=int, default=3) + args = ap.parse_args(argv) + + data_root = find_data_root(args.data_root) + if not data_root: + print("ERROR: gbrain-evals data not found; pass --data-root", file=sys.stderr) + return 2 + + plan = PLANS[args.plan] + done = _load_done(args.out) + print(f"[sweep] plan={args.plan} configs={len(plan)} already_done={len(done)} -> {args.out}") + for i, cfg in enumerate(plan, 1): + key = _cfg_key(cfg) + if key in done: + print(f"[sweep] ({i}/{len(plan)}) skip (done): {cfg}") + continue + print(f"[sweep] ({i}/{len(plan)}) running: {cfg}", flush=True) + try: + row = run_one(cfg, data_root, args.codex_path, args.limit_replay, args.limit_holdout) + except Exception as e: # never let one config kill the sweep + row = {"cfg": cfg, "cfg_key": key, "error": f"{type(e).__name__}: {e}"} + _append(args.out, row) + print(f"[sweep] -> {json.dumps({k: v for k, v in row.items() if k not in ('cfg','cfg_key')})}", flush=True) + print(f"[sweep] done. rows in {args.out}: {len(_load_done(args.out))}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skillopt_sleep/gate.py b/skillopt_sleep/gate.py new file mode 100644 index 0000000..7eca3b4 --- /dev/null +++ b/skillopt_sleep/gate.py @@ -0,0 +1,50 @@ +"""SkillOpt-Sleep — vendored validation gate. + +This is a self-contained copy of the SkillOpt validation gate so the sleep +engine has ZERO dependency on the research package (skillopt/*). The research +repo's ``skillopt.evaluation.gate`` is the reference implementation and the two +are kept behaviourally identical; vendoring keeps this open-source tool +decoupled from the paper's experiment code. +""" +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class GateResult: + action: str # "accept_new_best" | "accept" | "reject" + current_skill: str + current_score: float + best_skill: str + best_score: float + best_step: int + + +def select_gate_score(hard: float, soft: float, metric: str = "hard", + mixed_weight: float = 0.5) -> float: + """Project (hard, soft) onto a single comparison metric.""" + if metric == "hard": + return float(hard) + if metric == "soft": + return float(soft) + if metric == "mixed": + w = max(0.0, min(1.0, float(mixed_weight))) + return (1.0 - w) * float(hard) + w * float(soft) + raise ValueError(f"unknown gate metric {metric!r}; expected hard/soft/mixed") + + +def evaluate_gate(candidate_skill: str, cand_hard: float, current_skill: str, + current_score: float, best_skill: str, best_score: float, + best_step: int, global_step: int, *, cand_soft: float = 0.0, + metric: str = "hard", mixed_weight: float = 0.5) -> GateResult: + """Pure gate decision: compare candidate score to current/best.""" + cand_score = select_gate_score(cand_hard, cand_soft, metric, mixed_weight) + if cand_score > current_score: + if cand_score > best_score: + return GateResult("accept_new_best", candidate_skill, cand_score, + candidate_skill, cand_score, global_step) + return GateResult("accept", candidate_skill, cand_score, + best_skill, best_score, best_step) + return GateResult("reject", current_skill, current_score, + best_skill, best_score, best_step) diff --git a/skillopt_sleep/handoff_backend.py b/skillopt_sleep/handoff_backend.py new file mode 100644 index 0000000..0e41ae2 --- /dev/null +++ b/skillopt_sleep/handoff_backend.py @@ -0,0 +1,173 @@ +"""SkillOpt-Sleep — handoff backend (session-executed model calls). + +Runs the sleep cycle WITHOUT spawning any model subprocess or API call. +Every intelligent operation (attempt / judge / reflect) is turned into a +prompt file that an interactive agent session answers between engine runs: + + run 1: the engine executes the deterministic stages; every model call + it needs is recorded as a pending prompt; the run stops and + writes PROMPTS.md + pending.json into the handoff directory. + you: answer each prompt (each in a FRESH context, so the session's + own history cannot contaminate the held-out gate) and write the + raw answer text to answers/.md. + run 2: the engine re-runs; answered prompts resolve from answers/, the + cycle advances to the next model-dependent stage, and either + finishes or writes the next PROMPTS.md batch. + +Resume needs no serialized engine state: harvest -> mine -> replay is +deterministic, so re-running regenerates identical prompts and the answers +directory acts as a persistent, cross-run call cache. A prompt that embeds +a still-unanswered response (detected via the pending sentinel) aborts the +run immediately so placeholder text never propagates into scores, edits, +or staging. A typical night converges in 3-6 rounds: baseline attempts -> +reflect -> candidate re-scoring per accepted edit. + +Limitations (v1): `dream_rollouts > 1` yields no contrastive spread (the +same prompt maps to the same answer file), and tool-loop tasks fall back +to the base single-shot 'TOOL_CALL: ' marker convention. +""" +from __future__ import annotations + +import json +import os +import threading +from typing import Dict + +from skillopt_sleep.backend import CliBackend, skill_hash + +PENDING_SENTINEL_PREFIX = "[[SKILLOPT-SLEEP-PENDING:" +PENDING_SENTINEL_SUFFIX = "]]" + +# reflect() appends this when a reply fails to parse; with a placeholder +# reply the retry is a dependent call, not a genuinely new question. +_REFLECT_RETRY_MARKER = "your previous reply was not valid JSON" + +PROMPTS_FILENAME = "PROMPTS.md" +PENDING_FILENAME = "pending.json" + + +class PendingCalls(RuntimeError): + """The cycle cannot advance until pending prompts are answered.""" + + def __init__(self, pending: Dict[str, Dict[str, object]]): + self.pending = dict(pending) + super().__init__( + f"{len(self.pending)} model call(s) awaiting handoff answers" + ) + + +class HandoffBackend(CliBackend): + """Backend that outsources every model call to prompt/answer files. + + ``_call`` resolves a prompt from ``answers/.md`` when the + answer exists; otherwise it records the prompt as pending and returns a + sentinel placeholder so independent calls in the same phase can still + be collected into one batch. Any call whose prompt was BUILT FROM a + placeholder raises :class:`PendingCalls` — that call depends on answers + the user has not provided yet, so continuing would only mint garbage. + """ + + name = "handoff" + + def __init__(self, model: str = "", handoff_dir: str = "") -> None: + super().__init__(model=model, timeout=0) + self.handoff_dir = os.path.abspath( + handoff_dir or os.path.join(os.getcwd(), ".skillopt-sleep-handoff") + ) + self.answers_dir = os.path.join(self.handoff_dir, "answers") + os.makedirs(self.answers_dir, exist_ok=True) + # key -> {"prompt": str, "max_tokens": int}, insertion-ordered + self.pending: Dict[str, Dict[str, object]] = {} + self._lock = threading.Lock() + + # ── prompt/answer plumbing ──────────────────────────────────────────── + def answer_path(self, key: str) -> str: + return os.path.join(self.answers_dir, f"{key}.md") + + def _call(self, prompt: str, *, max_tokens: int = 1024) -> str: + if PENDING_SENTINEL_PREFIX in prompt: + # Built from a still-pending response — dependent call. + raise PendingCalls(self.pending) + if _REFLECT_RETRY_MARKER in prompt and self.pending: + # Retry of a reflect whose first reply is the placeholder. + raise PendingCalls(self.pending) + key = skill_hash(prompt) + path = self.answer_path(key) + if os.path.exists(path): + with open(path, encoding="utf-8") as f: + return f.read().strip() + with self._lock: + self.pending[key] = {"prompt": prompt, "max_tokens": max_tokens} + return f"{PENDING_SENTINEL_PREFIX}{key}{PENDING_SENTINEL_SUFFIX}" + + # ── handoff file emission ───────────────────────────────────────────── + def flush_pending(self) -> str: + """Write PROMPTS.md (human/agent-readable) + pending.json (machine). + + Prompts can themselves contain markdown fences, so PROMPTS.md + delimits each prompt with BEGIN/END marker lines instead of fences. + Returns the PROMPTS.md path. + """ + from skillopt_sleep.staging import redact_secrets + + os.makedirs(self.handoff_dir, exist_ok=True) + with self._lock: + items = list(self.pending.items()) + payload = { + "format": "skillopt_sleep.handoff.v1", + "answers_dir": self.answers_dir, + "pending": [ + { + "id": key, + "answer_file": self.answer_path(key), + "max_tokens": item["max_tokens"], + "prompt": redact_secrets(str(item["prompt"])), + } + for key, item in items + ], + } + with open(os.path.join(self.handoff_dir, PENDING_FILENAME), "w", + encoding="utf-8") as f: + json.dump(payload, f, ensure_ascii=False, indent=2) + f.write("\n") + + lines = [ + "# SkillOpt-Sleep — pending model calls (handoff)", + "", + f"{len(items)} prompt(s) below need answers before the sleep " + "cycle can continue.", + "", + "For EACH prompt:", + "", + "1. Answer it in a FRESH context (e.g. a subagent with no", + " conversation history). Do NOT let the current session's", + " context, the other prompts in this file, or the optimization", + " run itself leak into the answer — that contaminates the", + " held-out validation gate.", + "2. Write ONLY the raw answer text (no commentary, no code", + " fences) to the prompt's answer file.", + "", + "When every answer file exists, re-run the same engine command", + "(`python -m skillopt_sleep run --backend handoff ...`); it", + "resumes automatically from the answers directory.", + "", + ] + for i, (key, item) in enumerate(items, start=1): + lines += [ + "---", + "", + f"## Prompt {i} of {len(items)}", + "", + f"- id: `{key}`", + f"- answer file: `answers/{key}.md`", + f"- suggested max tokens: {item['max_tokens']}", + "", + f"----- BEGIN PROMPT {key} -----", + redact_secrets(str(item["prompt"])), + f"----- END PROMPT {key} -----", + "", + ] + prompts_path = os.path.join(self.handoff_dir, PROMPTS_FILENAME) + with open(prompts_path, "w", encoding="utf-8") as f: + f.write("\n".join(lines)) + return prompts_path diff --git a/skillopt_sleep/harvest.py b/skillopt_sleep/harvest.py new file mode 100644 index 0000000..deb8d88 --- /dev/null +++ b/skillopt_sleep/harvest.py @@ -0,0 +1,342 @@ +"""SkillOpt-Sleep — Stage 1: harvest. + +Read the user's local Claude Code records (read-only) and normalize them +into :class:`SessionDigest` objects. + +Sources (verified schema): + * ~/.claude/history.jsonl — one JSON/line: + {"display": , "pastedContents": {...}, + "timestamp": , "project": } + * ~/.claude/projects//.jsonl — one record/line; the + records we care about have type "user"/"assistant" and carry: + message{role, content}, cwd, gitBranch, timestamp, sessionId, version + +This module performs NO writes and NO network calls. +""" +from __future__ import annotations + +import json +import os +from datetime import datetime +from typing import Any, Dict, Iterable, List, Optional + +from skillopt_sleep.types import SessionDigest + + +# Heuristic phrases that signal the user (dis)approving of prior output. +# English-only by default. Users whose sessions are in another language can add +# their own phrases via the SKILLOPT_SLEEP_NEG_FEEDBACK / _POS_FEEDBACK env vars +# (comma-separated), so the capability is extensible without hardcoding locales. +_NEGATIVE_FEEDBACK = ( + "still broken", "still not", "still wrong", "doesn't work", "does not work", + "not working", "that's wrong", "thats wrong", "incorrect", "wrong", + "no,", "nope", "fix it", "didn't", "did not", "broken", "error again", + "still failing", "still fails", "not fixed", "revert", "undo", +) +_POSITIVE_FEEDBACK = ( + "thanks", "thank you", "perfect", "great", "works now", "fixed", + "that works", "lgtm", "looks good", "nice", "awesome", "correct", +) + + +def _extra_phrases(env_var: str) -> tuple: + raw = os.environ.get(env_var, "") + return tuple(p.strip().lower() for p in raw.split(",") if p.strip()) + + +_NEGATIVE_FEEDBACK = _NEGATIVE_FEEDBACK + _extra_phrases("SKILLOPT_SLEEP_NEG_FEEDBACK") +_POSITIVE_FEEDBACK = _POSITIVE_FEEDBACK + _extra_phrases("SKILLOPT_SLEEP_POS_FEEDBACK") + + +def _iter_jsonl(path: str) -> Iterable[Dict[str, Any]]: + try: + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + yield json.loads(line) + except Exception: + continue + except (FileNotFoundError, IsADirectoryError, PermissionError): + return + + +def _text_from_content(content: Any) -> str: + """Flatten a message.content (str or list of blocks) into text.""" + if isinstance(content, str): + return content + if isinstance(content, list): + parts: List[str] = [] + for b in content: + if isinstance(b, dict): + if b.get("type") == "text" and b.get("text"): + parts.append(str(b["text"])) + return "\n".join(parts) + return "" + + +def _tool_names_from_content(content: Any) -> List[str]: + names: List[str] = [] + if isinstance(content, list): + for b in content: + if isinstance(b, dict) and b.get("type") == "tool_use" and b.get("name"): + names.append(str(b["name"])) + return names + + +def _detect_feedback(text: str) -> List[str]: + low = text.lower() + sig: List[str] = [] + for ph in _NEGATIVE_FEEDBACK: + if ph in low: + sig.append("neg:" + ph) + for ph in _POSITIVE_FEEDBACK: + if ph in low: + sig.append("pos:" + ph) + return sig + + +def _is_meta_prompt(text: str) -> bool: + """Skip slash-commands / system noise that aren't real user intents.""" + t = text.strip() + if not t: + return True + if t.startswith("<") and t.endswith(">"): + return True + if t.startswith("/") and len(t.split()) <= 3: + return True + if t.startswith("[Pasted text") or t.startswith("Caveat:"): + return True + # Expanded slash-command prompts (Claude Code injects the full command + # body as a user message, wrapped in / + # tags or rendered as "# / — ..." headers). Not a user intent. + if "" in t[:200] or "" in t[:200]: + return True + if t.startswith("# /"): + return True + return False + + +# ── Issue #62: filter headless replay sessions ───────────────────────── + +# Prompt markers generated by the engine's own headless `claude -p` calls +# (judge, reflect, attempt). If the sole user prompt in a single-turn +# session matches any of these, the session is engine-generated, not a +# real user task. +_REPLAY_PROMPT_MARKERS = ( + "## CURRENT SKILL", + "## FAILED TASKS", + "## SUCCESSFUL TASKS", + "## OUTPUT FORMAT", + "You are a strict grader", + "Score the response 0.0-1.0", + "You are SkillOpt-Sleep", + "## TASK\n", + "## SKILL\n", +) + + +# Sessions written by OTHER tools' sub-agents (memory observers, critic +# sub-agents, plugin self-invocations). These are multi-turn, so the +# single-turn heuristic in _is_headless_replay never catches them. If the +# FIRST user prompt matches any marker, the whole session is +# machine-generated, not a real user task. +_AGENT_SESSION_MARKERS = ( + "You are a Claude-Mem", # claude-mem observer agent + "Hello memory agent", # claude-mem observer continuation + "You are driving **SkillOpt-Sleep**", # this plugin's own command body +) + tuple( + # users can add markers for their own tools' agent prompts + p.strip() + for p in os.environ.get("SKILLOPT_SLEEP_AGENT_MARKERS", "").split(",") + if p.strip() +) + + +def _is_agent_session(digest: "SessionDigest") -> bool: + """Detect transcripts written by other tools' sub-agents (see markers).""" + if not digest.user_prompts: + return False + first = digest.user_prompts[0] + return any(marker in first for marker in _AGENT_SESSION_MARKERS) + + +def _is_headless_replay(digest: "SessionDigest") -> bool: + """Detect sessions created by the engine's own headless replay calls. + + Heuristics (conservatively applied): + 1. Session has exactly 1 user turn AND + 2. The sole prompt matches engine-generated patterns (grader/reflect), + OR the session lasted < 3 seconds (programmatic, not interactive). + Multi-turn sessions are always kept (interactive by definition). + """ + if digest.n_user_turns > 1: + return False + if digest.n_user_turns == 0: + return True + prompt = digest.user_prompts[0] if digest.user_prompts else "" + for marker in _REPLAY_PROMPT_MARKERS: + if marker in prompt: + return True + # Sub-3-second single-turn sessions with short prompts are almost + # certainly programmatic (engine grader/judge calls). We require the + # prompt to also be short (<200 chars) to avoid false-positives on + # real one-shot questions that Claude happens to answer quickly. + if digest.started_at and digest.ended_at and len(prompt) < 200: + try: + fmt = "%Y-%m-%dT%H:%M:%S" + start = datetime.strptime(digest.started_at[:19], fmt) + end = datetime.strptime(digest.ended_at[:19], fmt) + if (end - start).total_seconds() < 3: + return True + except (ValueError, TypeError): + pass + return False + + +def digest_transcript(path: str) -> Optional[SessionDigest]: + """Build a SessionDigest from one ``.jsonl`` transcript.""" + session_id = os.path.splitext(os.path.basename(path))[0] + project = "" + git_branch = "" + started = "" + ended = "" + user_prompts: List[str] = [] + assistant_finals: List[str] = [] + tools: List[str] = [] + files: List[str] = [] + feedback: List[str] = [] + n_user = 0 + n_asst = 0 + + for rec in _iter_jsonl(path): + rtype = rec.get("type") + ts = rec.get("timestamp") + if isinstance(ts, str) and ts: + if not started: + started = ts + ended = ts + if rec.get("cwd") and not project: + project = str(rec.get("cwd")) + if rec.get("gitBranch") and not git_branch: + git_branch = str(rec.get("gitBranch")) + if rtype == "file-history-snapshot": + snap = rec.get("snapshot") or rec.get("files") or {} + if isinstance(snap, dict): + files.extend([str(k) for k in list(snap.keys())[:20]]) + msg = rec.get("message") + if not isinstance(msg, dict): + continue + role = msg.get("role") + content = msg.get("content") + if role == "user": + text = _text_from_content(content) + if text and not _is_meta_prompt(text): + n_user += 1 + user_prompts.append(text.strip()) + feedback.extend(_detect_feedback(text)) + elif role == "assistant": + n_asst += 1 + tools.extend(_tool_names_from_content(content)) + text = _text_from_content(content) + if text.strip(): + assistant_finals.append(text.strip()) + + if n_user == 0 and n_asst == 0: + return None + + # de-dup tools/files preserving order + def _dedup(xs: List[str]) -> List[str]: + seen = set() + out = [] + for x in xs: + if x not in seen: + seen.add(x) + out.append(x) + return out + + return SessionDigest( + session_id=session_id, + project=project, + git_branch=git_branch, + started_at=started, + ended_at=ended, + user_prompts=user_prompts, + assistant_finals=assistant_finals[-5:], # last few finals are the useful ones + tools_used=_dedup(tools), + files_touched=_dedup(files), + feedback_signals=feedback, + n_user_turns=n_user, + n_assistant_turns=n_asst, + raw_path=path, + ) + + +def _project_matches(project: str, scope: Any, invoked: str) -> bool: + if scope == "all": + return True + if isinstance(scope, (list, tuple)): + return any(os.path.abspath(project) == os.path.abspath(p) for p in scope) + # "invoked": match the invoked project (or a subdir of it) + if not invoked: + return True + a = os.path.abspath(project) + b = os.path.abspath(invoked) + return a == b or a.startswith(b + os.sep) or b.startswith(a + os.sep) + + +def harvest( + transcripts_dir: str, + *, + scope: Any = "all", + invoked_project: str = "", + since_iso: Optional[str] = None, + limit: int = 0, +) -> List[SessionDigest]: + """Walk ~/.claude/projects and return digests matching scope/time. + + Parameters + ---------- + transcripts_dir : str ~/.claude/projects + scope : "all" | "invoked" | list[path] + invoked_project : str used when scope == "invoked" + since_iso : str|None ISO8601; only sessions ending after this are kept + limit : int cap number of digests (0 = no cap) + """ + digests: List[SessionDigest] = [] + if not os.path.isdir(transcripts_dir): + return digests + + paths: List[str] = [] + for root, _dirs, files in os.walk(transcripts_dir): + # Sub-agent sidechain transcripts (/subagents/agent-*.jsonl) + # are Claude-authored prompts, not user tasks — never harvest them. + if os.path.basename(root) == "subagents": + continue + for fn in files: + if fn.endswith(".jsonl") and not fn.startswith("agent-"): + paths.append(os.path.join(root, fn)) + # newest first by mtime + paths.sort(key=lambda p: os.path.getmtime(p), reverse=True) + + for p in paths: + d = digest_transcript(p) + if d is None: + continue + if _is_headless_replay(d): + continue # Issue #62: skip engine's own headless replay sessions + if _is_agent_session(d): + continue # skip other tools' sub-agent transcripts (claude-mem etc.) + if not _project_matches(d.project or "", scope, invoked_project): + continue + if since_iso and d.ended_at and d.ended_at < since_iso: + # Note: files are sorted by mtime but we compare the embedded + # ended_at timestamp — mtime can diverge (copy/touch), so we + # cannot break here; we must continue to check all files. + continue + digests.append(d) + if limit and len(digests) >= limit: + break + return digests diff --git a/skillopt_sleep/harvest_codex.py b/skillopt_sleep/harvest_codex.py new file mode 100644 index 0000000..c50a237 --- /dev/null +++ b/skillopt_sleep/harvest_codex.py @@ -0,0 +1,233 @@ +"""SkillOpt-Sleep Codex Desktop session harvesting. + +Reads Codex Desktop archived session JSONL files and normalizes them into +``SessionDigest`` records without copying developer/system instructions, tool +arguments, or raw tool outputs. +""" +from __future__ import annotations + +import os +import re +from typing import Any, Dict, Iterable, List, Optional + +from skillopt_sleep.harvest import ( + _detect_feedback, + _is_meta_prompt, + _iter_jsonl, + _project_matches, +) +from skillopt_sleep.staging import _SECRET_PATTERNS +from skillopt_sleep.types import SessionDigest + + +def _payload(rec: Dict[str, Any]) -> Dict[str, Any]: + payload = rec.get("payload") + return payload if isinstance(payload, dict) else {} + + +def _timestamp(rec: Dict[str, Any], payload: Dict[str, Any]) -> str: + for value in ( + payload.get("timestamp"), + rec.get("timestamp"), + payload.get("started_at"), + payload.get("completed_at"), + ): + if isinstance(value, str) and value: + return value + return "" + + +def _text_from_any(content: Any) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + parts: List[str] = [] + for item in content: + if isinstance(item, str): + parts.append(item) + elif isinstance(item, dict): + if item.get("type") == "text" and item.get("text"): + parts.append(str(item["text"])) + elif item.get("text"): + parts.append(str(item["text"])) + return "\n".join(parts) + if isinstance(content, dict): + if content.get("text"): + return str(content["text"]) + if content.get("content"): + return _text_from_any(content["content"]) + return "" + + +def _strip_codex_meta(text: str) -> str: + stripped = text.strip() + if not stripped: + return "" + if stripped.startswith("", ""): + idx = stripped.rfind(marker) + if idx == -1: + continue + tail = stripped[idx + len(marker):].strip() + if tail and not tail.startswith("<"): + return tail + return "" + return stripped + + +def _sanitize_text(text: str) -> str: + sanitized = _strip_codex_meta(text).replace("\x00", "").strip() + if not sanitized or _is_meta_prompt(sanitized): + return "" + for pattern, replacement in _SECRET_PATTERNS: + sanitized = pattern.sub(replacement, sanitized) + return sanitized + + +def _sanitize_tool_name(name: str) -> str: + return re.sub(r"[^A-Za-z0-9_.:-]+", "_", name)[:80] + + +def _tool_name(payload: Dict[str, Any]) -> str: + payload_type = payload.get("type") + name = payload.get("name") + if isinstance(name, str) and name: + return _sanitize_tool_name(name) + if payload_type == "exec_command_end": + return "exec_command" + if payload_type == "patch_apply_end": + return "apply_patch" + if payload_type == "web_search_call": + return "web_search" + if payload_type == "tool_search_call": + return "tool_search" + if isinstance(payload_type, str) and payload_type.endswith("_tool_call"): + return _sanitize_tool_name(payload_type) + return "" + + +def _dedup(xs: Iterable[str]) -> List[str]: + seen = set() + out: List[str] = [] + for x in xs: + if x not in seen: + seen.add(x) + out.append(x) + return out + + +def digest_codex_archived_session(path: str, project: str = "") -> Optional[SessionDigest]: + """Build a ``SessionDigest`` from one Codex Desktop archived session.""" + session_id = os.path.splitext(os.path.basename(path))[0] + started = "" + ended = "" + session_project = "" + user_prompts: List[str] = [] + assistant_finals: List[str] = [] + tools: List[str] = [] + feedback: List[str] = [] + n_user = 0 + n_asst = 0 + + for rec in _iter_jsonl(path): + payload = _payload(rec) + payload_type = payload.get("type") + ts = _timestamp(rec, payload) + if ts: + if not started: + started = ts + ended = ts + cwd = payload.get("cwd") + if isinstance(cwd, str) and cwd: + if not session_project: + session_project = cwd + if project and _project_matches(cwd, "invoked", project): + session_project = cwd + + role = payload.get("role") + text = "" + output_role = "" + if payload_type == "user_message": + text = _text_from_any(payload.get("message")) + output_role = "user" + elif payload_type == "agent_message": + text = _text_from_any(payload.get("message")) + output_role = "assistant" + elif payload_type == "message" and role in {"user", "assistant"}: + text = _text_from_any(payload.get("content")) + output_role = str(role) + else: + tool = _tool_name(payload) + if tool: + tools.append(tool) + continue + + sanitized = _sanitize_text(text) + if not sanitized: + continue + if output_role == "user": + n_user += 1 + user_prompts.append(sanitized) + feedback.extend(_detect_feedback(sanitized)) + elif output_role == "assistant": + n_asst += 1 + assistant_finals.append(sanitized) + + if project and not _project_matches(session_project or "", "invoked", project): + return None + if n_user == 0 and n_asst == 0: + return None + + return SessionDigest( + session_id=session_id, + project=session_project, + started_at=started, + ended_at=ended, + user_prompts=user_prompts, + assistant_finals=assistant_finals[-5:], + tools_used=_dedup(tools), + files_touched=[], + feedback_signals=feedback, + n_user_turns=n_user, + n_assistant_turns=n_asst, + raw_path=path, + ) + + +def harvest_codex( + archived_sessions_dir: str, + *, + scope: Any = "all", + invoked_project: str = "", + since_iso: Optional[str] = None, + limit: int = 0, +) -> List[SessionDigest]: + """Walk ``~/.codex/archived_sessions`` and return matching digests.""" + digests: List[SessionDigest] = [] + if not os.path.isdir(archived_sessions_dir): + return digests + + paths = [ + os.path.join(archived_sessions_dir, fn) + for fn in os.listdir(archived_sessions_dir) + if fn.endswith(".jsonl") + ] + paths.sort(key=lambda p: os.path.getmtime(p), reverse=True) + + project_hint = invoked_project if scope == "invoked" else "" + for path in paths: + digest = digest_codex_archived_session(path, project=project_hint) + if digest is None: + continue + if not _project_matches(digest.project or "", scope, invoked_project): + continue + if since_iso and digest.ended_at and digest.ended_at < since_iso: + continue + digests.append(digest) + if limit and len(digests) >= limit: + break + return digests diff --git a/skillopt_sleep/harvest_sources.py b/skillopt_sleep/harvest_sources.py new file mode 100644 index 0000000..501aa28 --- /dev/null +++ b/skillopt_sleep/harvest_sources.py @@ -0,0 +1,41 @@ +"""Source selection for SkillOpt-Sleep transcript harvesting.""" +from __future__ import annotations + +from typing import Optional + +from skillopt_sleep.harvest import harvest +from skillopt_sleep.harvest_codex import harvest_codex +from skillopt_sleep.types import SessionDigest + + +def harvest_for_config(cfg, *, since_iso: Optional[str] = None, limit: int = 0) -> list[SessionDigest]: + source = cfg.get("transcript_source", "claude") + scope = cfg.get("projects", "invoked") + invoked_project = cfg.get("invoked_project", "") + + if source == "codex": + return harvest_codex( + cfg.codex_archived_sessions_dir, + scope=scope, + invoked_project=invoked_project, + since_iso=since_iso, + limit=limit, + ) + if source == "auto": + codex_digests = harvest_codex( + cfg.codex_archived_sessions_dir, + scope=scope, + invoked_project=invoked_project, + since_iso=since_iso, + limit=limit, + ) + if codex_digests: + return codex_digests + + return harvest( + cfg.transcripts_dir, + scope=scope, + invoked_project=invoked_project, + since_iso=since_iso, + limit=limit, + ) diff --git a/skillopt_sleep/judges.py b/skillopt_sleep/judges.py new file mode 100644 index 0000000..f981015 --- /dev/null +++ b/skillopt_sleep/judges.py @@ -0,0 +1,84 @@ +"""SkillOpt-Sleep — rule-based judges (gbrain-evals compatible). + +Implements the programmatic check operators used by gbrain-evals' +skillopt-v1 benchmark so we can score skill outputs locally, with NO judge +API call: + + * section_present — a markdown heading containing exists + * regex — the pattern matches the response + * max_chars — response length <= n + * min_chars — response length >= n + * contains — substring present (case-insensitive) + * tool_called — a tool with was invoked (needs a tool loop; + in single-shot replay we approximate via an + explicit "TOOL_CALL: " marker the agent emits) + +A task whose judge is {"kind": "rule", "checks": [...]} passes (hard=1.0) iff +ALL checks pass; soft = fraction of checks passed. This mirrors gbrain's +all-checks-must-pass rule scoring and gives the gate a smooth signal. +""" +from __future__ import annotations + +import re +from typing import Any, Dict, List, Tuple + + +def _section_present(response: str, name: str) -> bool: + # a markdown heading line (#, ##, ...) or bold line that contains `name` + pat = re.compile( + r"(?im)^\s{0,3}(#{1,6}\s*.*%s|\*\*.*%s.*\*\*\s*:?)\s*$" % (re.escape(name), re.escape(name)) + ) + if pat.search(response or ""): + return True + # also accept "Name:" style label at line start + label = re.compile(r"(?im)^\s*%s\s*:" % re.escape(name)) + return bool(label.search(response or "")) + + +def _check(op: str, arg: Any, response: str, tools_called: List[str]) -> bool: + r = response or "" + if op == "section_present": + return _section_present(r, str(arg)) + if op == "regex": + try: + return bool(re.search(str(arg), r)) + except re.error: + return False + if op == "max_chars": + return len(r) <= int(arg) + if op == "min_chars": + return len(r) >= int(arg) + if op == "contains": + return str(arg).lower() in r.lower() + if op == "tool_called": + name = str(arg).lower() + if any(name == t.lower() for t in tools_called): + return True + # single-shot approximation: the agent emits an explicit marker + return bool(re.search(r"(?i)\btool_call\s*:\s*%s\b" % re.escape(name), r)) + # unknown op: do not block + return True + + +def score_rule_judge( + judge: Dict[str, Any], + response: str, + tools_called: List[str] | None = None, +) -> Tuple[float, float, str]: + """Return (hard, soft, rationale) for a gbrain-style rule judge.""" + checks = (judge or {}).get("checks", []) or [] + if not checks: + return 0.0, 0.0, "no checks" + tools_called = tools_called or [] + passed = 0 + failed_desc: List[str] = [] + for c in checks: + ok = _check(c.get("op", ""), c.get("arg"), response, tools_called) + if ok: + passed += 1 + else: + failed_desc.append(f"{c.get('op')}={c.get('arg')}") + soft = passed / len(checks) + hard = 1.0 if passed == len(checks) else 0.0 + rationale = "all checks passed" if hard else "failed: " + ", ".join(failed_desc) + return hard, soft, rationale diff --git a/skillopt_sleep/llm_miner.py b/skillopt_sleep/llm_miner.py new file mode 100644 index 0000000..dd78c63 --- /dev/null +++ b/skillopt_sleep/llm_miner.py @@ -0,0 +1,134 @@ +"""SkillOpt-Sleep — LLM-backed task miner. + +The heuristic miner (mine.py) produces TaskRecords without a checkable +reference, so real harvested transcripts can't show measurable lift. This +module uses an optimizer backend to turn session digests into TaskRecords +WITH a checkable rubric judge — the missing piece for real-data improvement. + +For each recurring intent it extracts: + * a clean, generalized `intent` (the reusable task, stripped of one-off specifics) + * a `rubric` (what a good answer must satisfy) -> stored as a rule judge of + `contains`/`regex`/`section_present` checks the local judge can score, OR a + free-text rubric scored by the backend's judge() when no programmatic check fits + * a preference signal (was the user satisfied?) to weight failures + +It is deliberately conservative: it only emits a task when it can name a +concrete, checkable success criterion, so the gate has real signal. Tasks it +can't make checkable are dropped (logged), not faked. +""" +from __future__ import annotations + +import json +import re +from typing import Any, Callable, Dict, List + +from skillopt_sleep.backend import Backend, _extract_json +from skillopt_sleep.types import SessionDigest, TaskRecord + + +_MINER_PROMPT = """You are mining a user's past AI-assistant sessions to find RECURRING tasks +worth optimizing a skill for. From the session below, extract 0-3 reusable tasks. + +A good task is something the user asks for repeatedly or had to correct, where a +GENERAL rule would help next time (formatting, structure, tool-use, conventions). +Skip one-off or purely exploratory requests. + +For each task return: + - "intent": the reusable request, generalized (no one-off specifics) + - "checks": a list of programmatic success checks a grader can run on a future + answer. Each check is one of: + {"op":"section_present","arg":""} + {"op":"regex","arg":""} + {"op":"contains","arg":""} + {"op":"max_chars","arg":} + Only include checks you are confident a GOOD answer must satisfy. + - "rubric": a one-sentence description of what a good answer looks like + - "satisfied": true/false — did the user seem satisfied with the assistant's answer? + +Return ONLY a JSON array (possibly empty). No prose. + +# Session +project: __PROJECT__ +user prompts: +__PROMPTS__ +assistant final (last): +__FINAL__ +feedback signals: __FEEDBACK__ +""" + + +def _digest_to_prompt(d: SessionDigest) -> str: + prompts = "\n".join(f" - {p[:240]}" for p in d.user_prompts[:6]) or " (none)" + final = (d.assistant_finals[-1][:400] if d.assistant_finals else "(none)") + return ( + _MINER_PROMPT + .replace("__PROJECT__", d.project or "(unknown)") + .replace("__PROMPTS__", prompts) + .replace("__FINAL__", final) + .replace("__FEEDBACK__", ", ".join(d.feedback_signals[:6]) or "(none)") + ) + + +def _mk_task(d: SessionDigest, obj: Dict[str, Any], idx: int) -> TaskRecord | None: + intent = str(obj.get("intent", "")).strip() + if len(intent) < 8: + return None + checks = obj.get("checks") or [] + rubric = str(obj.get("rubric", "")).strip() + satisfied = bool(obj.get("satisfied", False)) + + # keep only well-formed checks + clean_checks = [] + for c in checks: + if isinstance(c, dict) and c.get("op") in { + "section_present", "regex", "contains", "max_chars", "min_chars", + }: + clean_checks.append({"op": c["op"], "arg": c.get("arg")}) + + import hashlib + tid = "llm_" + hashlib.sha256((d.project + intent).encode()).hexdigest()[:12] + + if clean_checks: + return TaskRecord( + id=tid, project=d.project, intent=intent, + reference_kind="rule", judge={"kind": "rule", "checks": clean_checks}, + outcome="success" if satisfied else "fail", + tags=["mined:llm"], source_sessions=[d.session_id], + ) + if rubric: + return TaskRecord( + id=tid, project=d.project, intent=intent, + reference_kind="rubric", reference=rubric, + outcome="success" if satisfied else "fail", + tags=["mined:llm"], source_sessions=[d.session_id], + ) + return None # not checkable -> drop + + +def make_llm_miner( + backend: Backend, + *, + max_sessions: int = 20, + max_tasks: int = 40, +) -> Callable[[List[SessionDigest]], List[TaskRecord]]: + """Return an llm_miner(digests) -> list[TaskRecord] bound to a backend.""" + + def _miner(digests: List[SessionDigest]) -> List[TaskRecord]: + out: List[TaskRecord] = [] + for d in digests[:max_sessions]: + if not d.user_prompts: + continue + raw = backend._call(_digest_to_prompt(d), max_tokens=800) # type: ignore[attr-defined] + arr = _extract_json(raw, "array") + if not isinstance(arr, list): + continue + for i, obj in enumerate(arr[:3]): + if isinstance(obj, dict): + t = _mk_task(d, obj, i) + if t is not None: + out.append(t) + if len(out) >= max_tasks: + return out + return out + + return _miner diff --git a/skillopt_sleep/memory.py b/skillopt_sleep/memory.py new file mode 100644 index 0000000..ef67f36 --- /dev/null +++ b/skillopt_sleep/memory.py @@ -0,0 +1,129 @@ +"""SkillOpt-Sleep — skill/memory document manipulation. + +Applies bounded EditRecords to a skill (SKILL.md body) or memory (CLAUDE.md) +document, and provides Dream-style consolidation helpers (dedup near-identical +lines, drop contradictions). All edits live inside a protected, clearly-marked +region so the sleep cycle never clobbers the user's hand-written content. +""" +from __future__ import annotations + +import re +from typing import List, Tuple + +from skillopt_sleep.types import EditRecord + +LEARNED_START = "" +LEARNED_END = "" +_BANNER = ( + "_This block is maintained by SkillOpt-Sleep. Edits here are proposed " + "offline, validated against your past tasks, and adopted only after you " + "approve them. Hand-edits outside this block are never touched._" +) + + +def extract_learned(doc: str) -> str: + s = doc.find(LEARNED_START) + e = doc.find(LEARNED_END) + if s == -1 or e == -1: + return "" + return doc[s + len(LEARNED_START):e].strip() + + +def _strip_learned(doc: str) -> str: + while True: + s = doc.find(LEARNED_START) + if s == -1: + break + e = doc.find(LEARNED_END, s) + if e == -1: + doc = doc[:s] + break + doc = doc[:s] + doc[e + len(LEARNED_END):] + while "\n\n\n" in doc: + doc = doc.replace("\n\n\n", "\n\n") + return doc.rstrip() + + +def set_learned(doc: str, learned_lines: List[str]) -> str: + """Replace the protected learned region with the given bullet lines.""" + base = _strip_learned(doc) + body = "\n".join(f"- {ln.strip().lstrip('- ').strip()}" for ln in learned_lines if ln.strip()) + block = ( + f"\n\n{LEARNED_START}\n" + f"## Learned preferences & procedures\n\n{_BANNER}\n\n{body}\n" + f"{LEARNED_END}\n" + ) + return (base + block).lstrip("\n") + + +def current_learned_lines(doc: str) -> List[str]: + inner = extract_learned(doc) + lines: List[str] = [] + for ln in inner.splitlines(): + ln = ln.strip() + if ln.startswith("- "): + lines.append(ln[2:].strip()) + return lines + + +def _norm(s: str) -> str: + return re.sub(r"\s+", " ", (s or "").lower()).strip() + + +def apply_edits(doc: str, edits: List[EditRecord]) -> Tuple[str, List[EditRecord]]: + """Apply add/delete/replace edits to the protected learned region. + + Returns (new_doc, applied_edits). Dedups: an `add` whose content already + exists (normalized) is skipped. `delete`/`replace` match on normalized + anchor substring. + """ + lines = current_learned_lines(doc) + norm_set = {_norm(line) for line in lines} + applied: List[EditRecord] = [] + + for e in edits: + op = (e.op or "add").lower() + if op == "add": + if _norm(e.content) in norm_set or not e.content.strip(): + continue + lines.append(e.content.strip()) + norm_set.add(_norm(e.content)) + applied.append(e) + elif op == "delete": + anchor = _norm(e.anchor or e.content) + keep = [line for line in lines if anchor not in _norm(line)] + if len(keep) != len(lines): + lines = keep + norm_set = {_norm(line) for line in lines} + applied.append(e) + elif op == "replace": + anchor = _norm(e.anchor) + new_lines = [] + changed = False + for line in lines: + if anchor and anchor in _norm(line): + new_lines.append(e.content.strip()) + changed = True + else: + new_lines.append(line) + if changed: + lines = new_lines + norm_set = {_norm(line) for line in lines} + applied.append(e) + + return set_learned(doc, lines), applied + + +def ensure_skill_scaffold(doc: str, *, name: str, description: str) -> str: + """Ensure a SKILL.md has YAML frontmatter so local agents load it.""" + if doc.lstrip().startswith("---"): + return doc + fm = ( + "---\n" + f"name: {name}\n" + f"description: {description}\n" + "---\n\n" + f"# {name}\n\n" + "Preferences and procedures learned from your past local agent sessions.\n" + ) + return fm + doc diff --git a/skillopt_sleep/mine.py b/skillopt_sleep/mine.py new file mode 100644 index 0000000..4483057 --- /dev/null +++ b/skillopt_sleep/mine.py @@ -0,0 +1,312 @@ +"""SkillOpt-Sleep — Stage 2: mine. + +Turn :class:`SessionDigest` objects into :class:`TaskRecord` training units. + +Two miners: + * heuristic_mine — deterministic, no API. Detects retry chains (a prompt + re-asked after negative feedback => the early attempt failed), extracts + the user's recurring intents, and labels outcomes from feedback signals. + * llm_mine — optional; uses an optimizer backend to produce richer + TaskRecords with checkable references. Falls back to heuristic on error. + +The heuristic miner is what makes the whole cycle runnable offline and is the +basis of the deterministic experiment. +""" +from __future__ import annotations + +import hashlib +import os +import re +from collections import Counter +from typing import Any, Callable, List, Optional, Set, Tuple + +from skillopt_sleep.types import SessionDigest, TaskRecord + + +def _tid(project: str, intent: str) -> str: + h = hashlib.sha256((project + "::" + intent).encode("utf-8")).hexdigest()[:12] + return "task_" + h + + +def _short(text: str, n: int = 600) -> str: + text = (text or "").strip() + return text if len(text) <= n else text[:n] + " …" + + +def _looks_negative(signals: List[str]) -> bool: + return any(s.startswith("neg:") for s in signals) + + +def _looks_positive(signals: List[str]) -> bool: + return any(s.startswith("pos:") for s in signals) + + +_TARGET_STOPWORDS = { + "about", "after", "again", "agent", "agents", "all", "also", "always", + "and", "any", "are", "before", "being", "but", "can", "codex", + "current", "default", "docs", "does", "done", "each", "file", "files", + "for", "from", "have", "into", "keep", "must", "not", "only", "path", + "paths", "project", "read", "repo", "request", "requests", "rule", + "rules", "same", "should", "skill", "skills", "source", "start", + "task", "tasks", "that", "the", "their", "then", "this", "unless", + "update", "user", "users", "when", "with", "work", "workflow", +} + + +def _target_tokens(text: str) -> List[str]: + tokens: List[str] = [] + for raw in re.findall(r"[\w][\w.-]*", (text or "").lower(), flags=re.UNICODE): + parts = [raw] + re.split(r"[\W_]+", raw, flags=re.UNICODE) + for part in parts: + if len(part) < 3 or part.isdigit() or part in _TARGET_STOPWORDS: + continue + tokens.append(part) + return tokens + + +def _expand_target_keywords(keywords: Set[str]) -> None: + if "mcp" in keywords: + keywords.update({ + "configure", "configuration", "connect", "connected", "enable", + "enabled", "install", "installed", "server", "servers", + "настрой", "настроить", "подключи", "подключить", + }) + if {"conflict", "conflicts"} & keywords: + keywords.update({ + "cherry", "conflict", "conflicts", "git", "merge", "rebase", + "unmerged", "конфликт", "конфликты", + }) + + +def target_task_keywords( + target_skill_text: str, + target_skill_path: str = "", + *, + limit: int = 180, +) -> Tuple[Set[str], Set[str]]: + """Return (strong, weak) keywords that describe a target skill.""" + path_text = (target_skill_path or "").replace(os.sep, " ") + headings = "\n".join(re.findall(r"(?m)^#+\s+(.+)$", target_skill_text or "")) + strong = set(_target_tokens(path_text + "\n" + headings)) + weak = set(strong) + counts = Counter(_target_tokens(target_skill_text or "")) + for token, _count in counts.most_common(limit): + weak.add(token) + _expand_target_keywords(strong) + _expand_target_keywords(weak) + return strong, weak + + +def _task_search_text(task: TaskRecord) -> str: + return "\n".join([ + task.intent or "", + task.context_excerpt or "", + " ".join(task.tags or []), + ]) + + +def filter_tasks_for_target( + tasks: List[TaskRecord], + target_skill_text: str, + target_skill_path: str = "", +) -> List[TaskRecord]: + """Prefer tasks whose language overlaps the explicit target skill. + + If nothing matches, return the original list. This keeps a target run useful + even when transcripts are too sparse or the skill is too generic. + """ + strong, weak = target_task_keywords(target_skill_text, target_skill_path) + if not tasks or not (strong or weak): + return tasks + + ranked = [] + for idx, task in enumerate(tasks): + tokens = set(_target_tokens(_task_search_text(task))) + strong_hits = tokens & strong + weak_hits = tokens & weak + if not strong_hits and len(weak_hits) < 2: + continue + score = len(strong_hits) * 3 + len(weak_hits) + ranked.append((score, idx, task)) + if not ranked: + return tasks + ranked.sort(key=lambda item: (-item[0], item[1])) + return [task for _score, _idx, task in ranked] + + +def heuristic_mine( + digests: List[SessionDigest], + *, + max_tasks: int = 40, +) -> List[TaskRecord]: + """Deterministic miner — no API calls. + + Strategy: + * Each session with >=1 real user prompt yields one TaskRecord whose + intent is the FIRST substantive prompt (the original ask). + * Outcome is inferred: + - negative feedback present and no later positive -> "fail" + - positive feedback present -> "success" + - re-asks (multiple user turns) without resolution -> "mixed" + - otherwise -> "unknown" + * attempted_solution = the last assistant final (what was produced). + * reference_kind defaults to "none"; the consolidation step will use a + rubric judge for these. (Exact refs are added by the experiment data + or by the LLM miner when it can derive a checkable answer.) + """ + tasks: List[TaskRecord] = [] + for d in digests: + if not d.user_prompts: + continue + intent = d.user_prompts[0] + if len(intent.strip()) < 8: + continue + if _looks_positive(d.feedback_signals) and not _looks_negative(d.feedback_signals): + outcome = "success" + elif _looks_negative(d.feedback_signals): + outcome = "fail" + elif d.n_user_turns >= 3: + outcome = "mixed" + else: + outcome = "unknown" + + attempted = d.assistant_finals[-1] if d.assistant_finals else "" + context = "" + if len(d.user_prompts) > 1: + # later prompts often carry the corrective detail / real constraints + context = "Follow-up constraints from the same session:\n- " + "\n- ".join( + _short(p, 200) for p in d.user_prompts[1:4] + ) + tags = [] + if d.tools_used: + tags.append("tools:" + "+".join(d.tools_used[:4])) + if d.git_branch: + tags.append("branch:" + d.git_branch) + + tasks.append( + TaskRecord( + id=_tid(d.project, intent), + project=d.project, + intent=_short(intent, 800), + context_excerpt=_short(context, 600), + attempted_solution=_short(attempted, 600), + outcome=outcome, + reference_kind="none", + reference="", + tags=tags, + source_sessions=[d.session_id], + ) + ) + if len(tasks) >= max_tasks: + break + return tasks + + +def dedup_tasks(tasks: List[TaskRecord]) -> List[TaskRecord]: + """Merge tasks sharing an id (same project+intent across sessions).""" + by_id: dict = {} + for t in tasks: + if t.id in by_id: + ex = by_id[t.id] + ex.source_sessions = list(dict.fromkeys(ex.source_sessions + t.source_sessions)) + # prefer a resolved outcome if either session resolved it + order = {"success": 3, "fail": 2, "mixed": 1, "unknown": 0} + if order.get(t.outcome, 0) > order.get(ex.outcome, 0): + ex.outcome = t.outcome + else: + by_id[t.id] = t + return list(by_id.values()) + + +def assign_splits( + tasks: List[TaskRecord], + *, + val_fraction: float = 0.34, + test_fraction: float = 0.0, + holdout_fraction: float | None = None, # legacy alias for val_fraction + seed: int = 42, +) -> List[TaskRecord]: + """Deterministically split tasks into train / val / test. + + Anti-overfitting contract (the user's design): + * ``val`` and ``test`` are drawn ONLY from REAL mined tasks (origin=='real') + and never overlap. val gates updates; test is the final held-out measure. + * ``train`` may include DREAM-augmented tasks (origin=='dream'); those are + NEVER placed in val/test. + + A stable hash of the task id keeps the same real task in the same split across + nights (a fixed held-out gate, like SkillOpt's D_sel/D_test). + + Back-compat: if ``test_fraction`` is 0 (default), this behaves like the old + two-way replay/holdout split — real tasks divide into train + val, no test. + ``holdout_fraction`` is accepted as an alias for ``val_fraction``. + """ + if holdout_fraction is not None: + val_fraction = holdout_fraction + + dream = [t for t in tasks if t.origin == "dream"] + real = [t for t in tasks if t.origin != "dream"] + + # all dream tasks go to train, unconditionally + for t in dream: + t.split = "train" + + val_cut = int(round(val_fraction * 100)) + test_cut = val_cut + int(round(test_fraction * 100)) + for t in real: + bucket = int(hashlib.sha256((str(seed) + t.id).encode()).hexdigest(), 16) % 100 + if bucket < val_cut: + t.split = "val" + elif bucket < test_cut: + t.split = "test" + else: + t.split = "train" + + # guarantee val (the gate) is non-empty when we have >=2 real tasks + real_splits = {t.split for t in real} + if len(real) >= 2 and "val" not in real_splits: + real[-1].split = "val" + # guarantee a train pool exists (dream or real) when possible + if not any(t.split == "train" for t in tasks) and len(real) >= 2: + real[0].split = "train" + # if test was requested but ended up empty with >=3 real tasks, carve one + if test_fraction > 0 and len(real) >= 3 and not any(t.split == "test" for t in real): + for t in real: + if t.split == "train": + t.split = "test" + break + return tasks + + +def normalize_legacy_split(value: str) -> str: + """Map old split names to the new vocabulary.""" + return {"replay": "train", "holdout": "val"}.get(value, value) + + +def mine( + digests: List[SessionDigest], + *, + max_tasks: int = 40, + candidate_limit: int = 0, + holdout_fraction: float = 0.34, + seed: int = 42, + llm_miner: Optional[Callable[[List[SessionDigest]], List[TaskRecord]]] = None, + target_skill_text: str = "", + target_skill_path: str = "", +) -> List[TaskRecord]: + """Top-level miner. Uses ``llm_miner`` if provided, else heuristic.""" + candidate_limit = candidate_limit or max_tasks + tasks: List[TaskRecord] = [] + if llm_miner is not None: + try: + tasks = llm_miner(digests) or [] + except Exception: + tasks = [] + if not tasks: + tasks = heuristic_mine(digests, max_tasks=candidate_limit) + tasks = dedup_tasks(tasks) + if target_skill_text or target_skill_path: + tasks = filter_tasks_for_target(tasks, target_skill_text, target_skill_path) + tasks = tasks[:max_tasks] + tasks = assign_splits(tasks, holdout_fraction=holdout_fraction, seed=seed) + return tasks diff --git a/skillopt_sleep/replay.py b/skillopt_sleep/replay.py new file mode 100644 index 0000000..e15f3df --- /dev/null +++ b/skillopt_sleep/replay.py @@ -0,0 +1,146 @@ +"""SkillOpt-Sleep — Stage 3: replay. + +Re-run mined TaskRecords offline under a given (skill, memory) and score +them, producing the (hard, soft) signal SkillOpt's gate consumes. + +Single-shot text replay by default. Tasks whose rule judge requires a tool +call (gbrain's `tool_called`) are run through the backend's real tool loop +(attempt_with_tools), so tool use is verified honestly rather than self-reported. +""" +from __future__ import annotations + +from typing import List, Tuple + +from skillopt_sleep.backend import Backend +from skillopt_sleep.types import ReplayResult, TaskRecord + + +def _required_tools(task: TaskRecord) -> List[str]: + """Tool names a rule judge requires (op == 'tool_called').""" + if task.reference_kind != "rule" or not task.judge: + return [] + tools = [] + for c in task.judge.get("checks", []) or []: + if isinstance(c, dict) and c.get("op") == "tool_called" and c.get("arg"): + tools.append(str(c["arg"])) + return tools + + +def replay_one(backend: Backend, task: TaskRecord, skill: str, memory: str, + sample_id: int = 0) -> ReplayResult: + """``sample_id`` distinguishes repeated dream rollouts of the same + (task, skill, memory) in the attempt cache — without it all K rollouts + collapse to one cached response and the contrastive signal is always 0.""" + import time + tools = _required_tools(task) + tools_called: List[str] = [] + t0 = time.time() + tok_before = backend.tokens_used() + if tools: + response, tools_called = backend.attempt_with_tools(task, skill, memory, tools) + else: + response = backend.attempt(task, skill, memory, sample_id=sample_id) + latency_ms = (time.time() - t0) * 1000.0 + tokens = max(0, backend.tokens_used() - tok_before) + # if the backend doesn't track tokens (e.g. mock), approximate from text length + if tokens == 0: + tokens = (len(skill) + len(memory) + len(task.intent) + len(response)) // 4 + + # rule judges may need the detected tool calls; score locally when possible + if task.reference_kind == "rule" and task.judge: + from skillopt_sleep.judges import score_rule_judge + hard, soft, rationale = score_rule_judge(task.judge, response, tools_called) + else: + hard, soft, rationale = backend.judge(task, response) + + return ReplayResult( + id=task.id, + hard=float(hard), + soft=float(soft), + response=response, + fail_reason="" if hard >= 1.0 else (rationale or "below threshold"), + task_type=(task.tags[0] if task.tags else "task"), + judge_rationale=rationale, + tools_called=tools_called, + tokens=int(tokens), + latency_ms=round(latency_ms, 1), + ) + + +import os +from concurrent.futures import ThreadPoolExecutor + + +def replay_batch( + backend: Backend, + tasks: List[TaskRecord], + skill: str, + memory: str, + *, + workers: int = 0, +) -> List[Tuple[TaskRecord, ReplayResult]]: + """Replay tasks, optionally in parallel. + + Real backends are network-bound, so a thread pool gives a large speedup on + big test sets (like the research harness's --workers). ``workers`` defaults + to env SKILLOPT_SLEEP_WORKERS or 1 (sequential). Mock stays sequential + (deterministic) unless asked otherwise. + """ + if workers <= 0: + workers = int(os.environ.get("SKILLOPT_SLEEP_WORKERS", "1") or "1") + if workers <= 1 or len(tasks) <= 1: + return [(t, replay_one(backend, t, skill, memory)) for t in tasks] + results: List = [None] * len(tasks) + with ThreadPoolExecutor(max_workers=min(workers, len(tasks))) as ex: + futs = {ex.submit(replay_one, backend, t, skill, memory): i + for i, t in enumerate(tasks)} + for fut in futs: + i = futs[fut] + results[i] = (tasks[i], fut.result()) + return results + + +def aggregate_scores(pairs: List[Tuple[TaskRecord, ReplayResult]]) -> Tuple[float, float]: + if not pairs: + return 0.0, 0.0 + hard = sum(r.hard for _t, r in pairs) / len(pairs) + soft = sum(r.soft for _t, r in pairs) / len(pairs) + return hard, soft + + +def aggregate_cost(pairs: List[Tuple[TaskRecord, ReplayResult]]) -> Tuple[float, float]: + """Mean (tokens, latency_ms) per task — the cost objectives.""" + if not pairs: + return 0.0, 0.0 + tok = sum(r.tokens for _t, r in pairs) / len(pairs) + lat = sum(r.latency_ms for _t, r in pairs) / len(pairs) + return tok, lat + + +def multi_objective_reward( + pairs: List[Tuple[TaskRecord, ReplayResult]], + *, + w_acc: float = 1.0, + w_tokens: float = 0.0, + w_latency: float = 0.0, + token_ref: float = 2000.0, + latency_ref_ms: float = 15000.0, +) -> float: + """Weighted reward = accuracy↑, tokens↓, latency↓. + + Cost terms are normalized against a reference and clamped to [0,1], so a + response at/under the reference cost contributes ~1.0 and an expensive one + less. Weights let the user trade off (default = accuracy only, backward + compatible). + """ + if not pairs: + return 0.0 + acc, _soft = aggregate_scores(pairs) + tok, lat = aggregate_cost(pairs) + tok_score = max(0.0, 1.0 - tok / max(1.0, token_ref)) if token_ref else 0.0 + lat_score = max(0.0, 1.0 - lat / max(1.0, latency_ref_ms)) if latency_ref_ms else 0.0 + total_w = w_acc + w_tokens + w_latency + if total_w <= 0: + return acc + return (w_acc * acc + w_tokens * tok_score + w_latency * lat_score) / total_w + diff --git a/skillopt_sleep/rollout.py b/skillopt_sleep/rollout.py new file mode 100644 index 0000000..8dc2c95 --- /dev/null +++ b/skillopt_sleep/rollout.py @@ -0,0 +1,153 @@ +"""SkillOpt-Sleep — multi-rollout + contrastive reflection (the imagination core). + +The core idea: let the agent re-run the SAME task many times, then look at +which rollouts went well vs badly and distill a rule from the *contrast*. This +is a much stronger learning signal than a single failure, and it is the essence +of the offline "dream/imagination" process — train-time rollouts are synthetic, +so doing many is fine. + +Pieces: + * multi_rollout — run one task K times under (skill, memory), return scored attempts + * contrastive_reflect — given good vs bad attempts of the same tasks, ask the + optimizer what distinguishes them and propose a general rule + +Driven through the Backend abstraction (mock/claude/codex), import-light. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import List, Optional, Tuple + +from skillopt_sleep.backend import Backend, _extract_json +from skillopt_sleep.replay import replay_one +from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord + + +@dataclass +class RolloutSet: + """K scored attempts at one task under a fixed (skill, memory).""" + task: TaskRecord + attempts: List[ReplayResult] = field(default_factory=list) + + @property + def best(self) -> Optional[ReplayResult]: + return max(self.attempts, key=lambda r: r.hard, default=None) + + @property + def worst(self) -> Optional[ReplayResult]: + return min(self.attempts, key=lambda r: r.hard, default=None) + + @property + def spread(self) -> float: + if not self.attempts: + return 0.0 + hs = [r.hard for r in self.attempts] + return max(hs) - min(hs) + + @property + def pass_rate(self) -> float: + if not self.attempts: + return 0.0 + return sum(1 for r in self.attempts if r.hard >= 1.0) / len(self.attempts) + + +def multi_rollout( + backend: Backend, + task: TaskRecord, + skill: str, + memory: str, + *, + k: int = 3, + workers: int = 0, +) -> RolloutSet: + """Run ``task`` K times. replay_one is deterministic for mock; for real + backends the model's own sampling yields variation across attempts. + + The K attempts are independent, so they run concurrently (this is the dream + phase's dominant cost). ``workers`` defaults to the SKILLOPT_SLEEP_WORKERS + env (capped at k); set to 1 to force serial (used by the mock tests). + """ + import os + rs = RolloutSet(task=task) + k = max(1, k) + if workers <= 0: + try: + workers = int(os.environ.get("SKILLOPT_SLEEP_WORKERS", "1")) + except ValueError: + workers = 1 + workers = max(1, min(workers, k)) + if workers == 1: + for i in range(k): + rs.attempts.append(replay_one(backend, task, skill, memory, sample_id=i)) + return rs + from concurrent.futures import ThreadPoolExecutor + with ThreadPoolExecutor(max_workers=workers) as ex: + futs = [ex.submit(replay_one, backend, task, skill, memory, sample_id=i) + for i in range(k)] + for f in futs: + rs.attempts.append(f.result()) + return rs + + +def contrastive_reflect( + backend: Backend, + rollout_sets: List[RolloutSet], + skill: str, + memory: str, + *, + edit_budget: int = 4, + target: str = "skill", +) -> List[EditRecord]: + """Distill a rule from the contrast between good and bad attempts. + + We pick tasks with the highest score *spread* (some attempts passed, some + failed) — those are the most informative — and show the optimizer a + high-scoring vs a low-scoring attempt of each, asking what general rule makes + the good behavior reliable. + """ + informative = [rs for rs in rollout_sets if rs.spread > 0 and rs.best and rs.worst] + informative.sort(key=lambda rs: rs.spread, reverse=True) + informative = informative[:6] + if not informative: + return [] + + blocks = [] + for rs in informative: + blocks.append( + f"## Task: {rs.task.intent[:160]}\n" + f"- GOOD attempt (score {rs.best.hard:.1f}): {rs.best.response[:200]}\n" + f"- BAD attempt (score {rs.worst.hard:.1f}): {rs.worst.response[:200]}\n" + f" (bad failed: {rs.worst.fail_reason[:100]})" + ) + # the output contract the proposed rules must not violate (same guardrail the + # single-shot reflect uses — prevents harness-violating rules like "return VBA" + # or "ask the user for the range" on SpreadsheetBench). + from skillopt_sleep.backend import _task_guardrail + guard = _task_guardrail([(rs.task, rs.best) for rs in informative]) + prompt = ( + "You are SkillOpt's optimizer doing CONTRASTIVE reflection. For each task " + "below the agent was run multiple times; some attempts succeeded and some " + "failed. Identify what the GOOD attempts did that the BAD ones did not, " + f"and propose at most {edit_budget} SHORT, GENERAL, reusable rules for the " + f"{target} that would make the good behavior reliable every time. Quote " + "concrete thresholds/formats verbatim; do not paraphrase vaguely. " + "Every rule MUST obey the task output contract (if shown) — never propose " + "a rule that changes the required output format/language or tells the agent " + "to ask the user a question; such a rule scores ZERO.\n" + f"{guard}" + 'Return ONLY a JSON array: ' + '[{"op":"add","content":"","rationale":""}].\n\n' + + "\n\n".join(blocks) + ) + raw = backend._call(prompt, max_tokens=1024) # type: ignore[attr-defined] + arr = _extract_json(raw, "array") + edits: List[EditRecord] = [] + if isinstance(arr, list): + for e in arr[:edit_budget]: + if isinstance(e, dict) and str(e.get("content", "")).strip(): + edits.append(EditRecord( + target=target, op=str(e.get("op", "add")).strip().lower(), + content=str(e["content"]).strip(), + rationale=str(e.get("rationale", "")).strip(), + )) + return edits diff --git a/skillopt_sleep/scheduler.py b/skillopt_sleep/scheduler.py new file mode 100644 index 0000000..3b32cb4 --- /dev/null +++ b/skillopt_sleep/scheduler.py @@ -0,0 +1,138 @@ +"""SkillOpt-Sleep — built-in nightly scheduler. + +Installs/removes a crontab entry that runs the sleep cycle automatically, so the +user doesn't have to wire cron themselves. Idempotent: a managed block delimited +by marker comments is added/replaced/removed in the user's crontab. + +Design choices: + * Off-:00 minute (3:17 local by default) so many users don't all hit the API + at the same instant. + * The entry runs `python -m skillopt_sleep run` for a specific project and + appends to /.skillopt-sleep/cron.log. + * `schedule` is additive per project (keyed by project path); `unschedule` + removes the project's line (or the whole managed block with --all). + +cron is the portable mechanism on Linux/macOS. On systems without `crontab`, +`schedule` prints the line and instructions instead of failing. +""" +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from typing import List, Optional, Tuple + +_BEGIN = "# >>> skillopt-sleep (managed) >>>" +_END = "# <<< skillopt-sleep (managed) <<<" + + +def _have_crontab() -> bool: + return shutil.which("crontab") is not None + + +def _read_crontab() -> str: + try: + proc = subprocess.run(["crontab", "-l"], capture_output=True, text=True) + return proc.stdout if proc.returncode == 0 else "" + except Exception: + return "" + + +def _write_crontab(content: str) -> bool: + try: + proc = subprocess.run(["crontab", "-"], input=content, text=True, + capture_output=True) + return proc.returncode == 0 + except Exception: + return False + + +def _split_managed(crontab: str) -> Tuple[str, List[str]]: + """Return (text_outside_block, managed_lines_inside_block).""" + lines = crontab.splitlines() + outside: List[str] = [] + managed: List[str] = [] + in_block = False + for ln in lines: + if ln.strip() == _BEGIN: + in_block = True + continue + if ln.strip() == _END: + in_block = False + continue + (managed if in_block else outside).append(ln) + return "\n".join(outside).rstrip(), managed + + +def _runner_cmd(project: str, backend: str, extra: str, python: str) -> str: + logdir = os.path.join(project, ".skillopt-sleep") + log = os.path.join(logdir, "cron.log") + # use absolute python + -m so cron's minimal env still works + cmd = (f'{python} -m skillopt_sleep run --project "{project}" ' + f'--scope invoked --backend {backend} {extra}'.rstrip()) + return f'mkdir -p "{logdir}"; cd "{_repo_root()}" && {cmd} >> "{log}" 2>&1' + + +def _repo_root() -> str: + # the package lives at /skillopt_sleep/; repo root is its parent + return os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + + +def _project_marker(project: str) -> str: + return f"# project={os.path.abspath(project)}" + + +def schedule(project: str, *, backend: str = "mock", hour: int = 3, minute: int = 17, + extra: str = "", python: Optional[str] = None) -> Tuple[bool, str]: + """Install (or replace) the nightly entry for ``project``. + + Returns (installed, message). If crontab is unavailable, installed=False and + the message contains the line to add manually. + """ + project = os.path.abspath(project) + python = python or sys.executable or "python3" + cron_line = f"{minute} {hour} * * * {_runner_cmd(project, backend, extra, python)} {_project_marker(project)}" + + if not _have_crontab(): + return False, ("crontab not found on this system. Add this line to your " + "scheduler manually:\n" + cron_line) + + outside, managed = _split_managed(_read_crontab()) + # drop any existing line for this project, then add the new one + marker = _project_marker(project) + managed = [ln for ln in managed if marker not in ln and ln.strip()] + managed.append(cron_line) + + block = _BEGIN + "\n" + "\n".join(managed) + "\n" + _END + new_crontab = (outside + "\n\n" + block + "\n").lstrip("\n") + ok = _write_crontab(new_crontab) + if ok: + return True, (f"Scheduled nightly at {hour:02d}:{minute:02d} for {project} " + f"(backend={backend}). Logs -> {project}/.skillopt-sleep/cron.log\n" + f"Runs `skillopt_sleep run`; it only STAGES a proposal — adopt is still manual.") + return False, "Failed to write crontab. Line to add manually:\n" + cron_line + + +def unschedule(project: Optional[str] = None, *, all_projects: bool = False) -> Tuple[bool, str]: + """Remove the entry for ``project`` (or the whole managed block with all_projects).""" + if not _have_crontab(): + return False, "crontab not found; nothing to remove." + outside, managed = _split_managed(_read_crontab()) + if all_projects: + managed = [] + elif project: + marker = _project_marker(project) + managed = [ln for ln in managed if marker not in ln and ln.strip()] + if managed: + block = _BEGIN + "\n" + "\n".join(managed) + "\n" + _END + new_crontab = (outside + "\n\n" + block + "\n").lstrip("\n") + else: + new_crontab = outside.rstrip() + "\n" + ok = _write_crontab(new_crontab) + return ok, ("Removed." if ok else "Failed to update crontab.") + + +def list_scheduled() -> List[str]: + _outside, managed = _split_managed(_read_crontab()) + return [ln for ln in managed if ln.strip()] diff --git a/skillopt_sleep/slow_update.py b/skillopt_sleep/slow_update.py new file mode 100644 index 0000000..7262785 --- /dev/null +++ b/skillopt_sleep/slow_update.py @@ -0,0 +1,142 @@ +"""SkillOpt-Sleep — slow update (cross-night long-term memory). + +This is the deployment-time analogue of SkillOpt's epoch-wise slow/meta update +(paper §3.6). Step-level edits (consolidate) learn from one night's batch; the +slow update learns across nights and writes a durable "longitudinal guidance" +block into a PROTECTED field of the skill that step-level edits never touch. + +It reuses the exact protected-field marker convention from the main repo +(``skillopt/optimizer/slow_update.py``) so the artifact is compatible: + + ... + +Why it matters: even when the user turns the validation gate OFF (greedy mode), +the slow update still runs at the end of the run, so short-term nightly +experience is consolidated into long-term memory rather than lost. The cross-night +content is carried in ``state.slow_memory``. + +Driven through the Backend abstraction (mock/claude/codex), so it stays +import-light — no `openai` dependency. +""" +from __future__ import annotations + +import re +from typing import List, Optional, Tuple + +from skillopt_sleep.backend import Backend, _extract_json +from skillopt_sleep.types import ReplayResult, TaskRecord + + +SLOW_UPDATE_START = "" +SLOW_UPDATE_END = "" + + +# ── protected-field helpers (mirror skillopt/optimizer/slow_update.py) ───────── + +def has_slow_field(skill: str) -> bool: + return SLOW_UPDATE_START in skill and SLOW_UPDATE_END in skill + + +def extract_slow_field(skill: str) -> str: + s = skill.find(SLOW_UPDATE_START) + e = skill.find(SLOW_UPDATE_END) + if s == -1 or e == -1: + return "" + return skill[s + len(SLOW_UPDATE_START):e].strip() + + +def _strip_slow_fields(skill: str) -> str: + while True: + s = skill.find(SLOW_UPDATE_START) + if s == -1: + break + e = skill.find(SLOW_UPDATE_END, s) + if e == -1: + skill = skill[:s] + break + skill = skill[:s] + skill[e + len(SLOW_UPDATE_END):] + skill = skill.replace(SLOW_UPDATE_END, "") + while "\n\n\n" in skill: + skill = skill.replace("\n\n\n", "\n\n") + return skill.rstrip() + + +def replace_slow_field(skill: str, content: str) -> str: + """Set the protected slow-update field to ``content`` (exactly one block).""" + base = _strip_slow_fields(skill) + if not content.strip(): + return base + block = f"\n\n{SLOW_UPDATE_START}\n{content.strip()}\n{SLOW_UPDATE_END}\n" + return base + block + + +# ── the slow-update synthesis ────────────────────────────────────────────────── + +def _summarize_pairs( + prev_pairs: List[Tuple[TaskRecord, ReplayResult]], + curr_pairs: List[Tuple[TaskRecord, ReplayResult]], +) -> str: + """Group adjacent-version outcomes into improved/regressed/persistent/stable.""" + prev_by = {t.id: r for t, r in prev_pairs} + lines: List[str] = [] + counts = {"improved": 0, "regressed": 0, "persistent_fail": 0, "stable_success": 0} + for t, r in curr_pairs: + p = prev_by.get(t.id) + if p is None: + continue + a, b = p.hard, r.hard + if b > a: + cat = "improved" + elif b < a: + cat = "regressed" + elif b >= 1.0: + cat = "stable_success" + else: + cat = "persistent_fail" + counts[cat] += 1 + if cat in ("regressed", "persistent_fail") and len(lines) < 8: + lines.append(f"- [{cat}] {t.intent[:120]} (why: {r.fail_reason[:80]})") + head = ", ".join(f"{k}={v}" for k, v in counts.items()) + return head + ("\n" + "\n".join(lines) if lines else ""), counts # type: ignore[return-value] + + +def run_slow_update( + backend: Backend, + *, + prev_skill: str, + curr_skill: str, + prev_pairs: List[Tuple[TaskRecord, ReplayResult]], + curr_pairs: List[Tuple[TaskRecord, ReplayResult]], + prev_slow_content: str = "", +) -> Optional[str]: + """Produce durable longitudinal guidance text (or None). + + Compares behavior under the previous vs current skill across the same tasks + and asks the optimizer to distill a short, durable guidance block — what to + keep doing, what regressions to avoid — refining any prior slow-update text. + """ + summary, counts = _summarize_pairs(prev_pairs, curr_pairs) # type: ignore[misc] + # nothing changed and no prior guidance to refine → skip + if counts["regressed"] == 0 and counts["persistent_fail"] == 0 and not prev_slow_content: + return None + + prompt = ( + "You are SkillOpt's SLOW UPDATE — the long-term memory pass that runs " + "across nights. Write a SHORT, durable guidance block (2-5 bullet " + "points) capturing the longitudinal lessons: behaviors that reliably " + "help and should be preserved, and regressions/persistent failures to " + "avoid. Keep it GENERAL and stable (not tied to one task). If prior " + "guidance is given, refine it rather than restate it.\n" + 'Return ONLY JSON: {"guidance": ""}.\n\n' + f"# Cross-night outcome summary\n{summary}\n\n" + f"# Prior long-term guidance (refine this)\n{prev_slow_content or '(none)'}" + ) + raw = backend._call(prompt, max_tokens=600) # type: ignore[attr-defined] + obj = _extract_json(raw, "object") + if isinstance(obj, dict): + g = str(obj.get("guidance", "")).strip() + if g: + return g + # fallback: if the model returned prose, keep the first ~400 chars + text = (raw or "").strip() + return text[:400] if text else None diff --git a/skillopt_sleep/staging.py b/skillopt_sleep/staging.py new file mode 100644 index 0000000..49dd859 --- /dev/null +++ b/skillopt_sleep/staging.py @@ -0,0 +1,159 @@ +"""SkillOpt-Sleep — Stage 5/6: staging and adoption. + +Implements the Dreams safety contract: the cycle never mutates the user's +live CLAUDE.md / SKILL.md. It writes proposals + a human-readable report into +a staging directory; a separate, explicit `adopt` step copies them over the +live files after taking a backup. +""" +from __future__ import annotations + +import json +import os +import re +import shutil +import time +from typing import Any, List, Optional + +from skillopt_sleep.types import SleepReport + +# Secret patterns scrubbed from any free-text we persist to the staging dir +# (diagnostics, reports). Kept here so every on-disk artifact shares one +# redaction pass; harvest_codex reuses these for session text too. +_SECRET_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = ( + (re.compile(r"sk-[A-Za-z0-9_-]{10,}"), "[REDACTED_OPENAI_KEY]"), + # Distinctive vendor token prefixes (low false-positive: these prefixes do + # not occur in normal diagnostic prose). + (re.compile(r"\bAKIA[0-9A-Z]{16}\b"), "[REDACTED_AWS_KEY]"), + (re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}\b"), "[REDACTED_GITHUB_TOKEN]"), + (re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{10,}\b"), "[REDACTED_SLACK_TOKEN]"), + (re.compile(r"\bAIza[0-9A-Za-z_-]{20,}\b"), "[REDACTED_GOOGLE_KEY]"), + # Bare JWT (three base64url segments) — e.g. a leaked bearer body without + # the "Authorization:" prefix. + (re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b"), + "[REDACTED_JWT]"), + (re.compile(r"(?i)(Authorization:\s*Bearer\s+)[^\s\"']+"), r"\1[REDACTED]"), + (re.compile(r"(?i)(Authorization:\s*Basic\s+)[^\s\"']+"), r"\1[REDACTED]"), + ( + re.compile(r"(?i)\b(api[_-]?key|token|password|secret)\b(\s*[:=]\s*)[^\s\"']+"), + r"\1\2[REDACTED]", + ), + ( + re.compile(r"(?i)\b(api[_-]?key|token|password|secret)\b(\s+)[^\s\"']+"), + r"\1\2[REDACTED]", + ), + ( + re.compile( + r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----", + re.DOTALL, + ), + "[REDACTED_PRIVATE_KEY]", + ), +) + + +def redact_secrets(value: Any) -> Any: + """Scrub secret-looking substrings (API keys, bearer tokens, private keys) + from a string, or recursively from the string leaves of a list/dict. + + Used before writing backend stderr / optimizer replies / task responses to + on-disk diagnostics: those are surfaced for debugging, but the underlying + text (e.g. a codex 401 stderr dump) can carry credentials. Non-string + scalars pass through unchanged. + """ + if isinstance(value, str): + out = value + for pattern, replacement in _SECRET_PATTERNS: + out = pattern.sub(replacement, out) + return out + if isinstance(value, list): + return [redact_secrets(v) for v in value] + if isinstance(value, dict): + return {k: redact_secrets(v) for k, v in value.items()} + return value + + +def _ts_dir() -> str: + return time.strftime("%Y%m%d-%H%M%S", time.localtime()) + + +def staging_root(project: str) -> str: + return os.path.join(project, ".skillopt-sleep", "staging") + + +def latest_staging(project: str) -> Optional[str]: + root = staging_root(project) + if not os.path.isdir(root): + return None + subs = sorted( + (os.path.join(root, d) for d in os.listdir(root)), + key=lambda p: os.path.getmtime(p), + reverse=True, + ) + return subs[0] if subs else None + + +def write_staging( + project: str, + *, + report: SleepReport, + proposed_skill: Optional[str], + proposed_memory: Optional[str], + live_skill_path: str, + live_memory_path: str, + report_md: str, +) -> str: + """Write proposals + report into staging// and return that path.""" + out = os.path.join(staging_root(project), _ts_dir()) + os.makedirs(out, exist_ok=True) + + manifest = { + "live_skill_path": live_skill_path, + "live_memory_path": live_memory_path, + "has_skill": proposed_skill is not None, + "has_memory": proposed_memory is not None, + "accepted": report.accepted, + } + if proposed_skill is not None: + with open(os.path.join(out, "proposed_SKILL.md"), "w", encoding="utf-8") as f: + f.write(proposed_skill) + if proposed_memory is not None: + with open(os.path.join(out, "proposed_CLAUDE.md"), "w", encoding="utf-8") as f: + f.write(proposed_memory) + with open(os.path.join(out, "report.json"), "w", encoding="utf-8") as f: + json.dump(report.to_dict(), f, ensure_ascii=False, indent=2) + with open(os.path.join(out, "report.md"), "w", encoding="utf-8") as f: + f.write(report_md) + with open(os.path.join(out, "manifest.json"), "w", encoding="utf-8") as f: + json.dump(manifest, f, ensure_ascii=False, indent=2) + return out + + +def _backup(path: str, backup_dir: str) -> None: + if os.path.exists(path): + os.makedirs(backup_dir, exist_ok=True) + shutil.copy2(path, os.path.join(backup_dir, os.path.basename(path))) + + +def adopt(staging_dir: str) -> List[str]: + """Copy staged proposals over the live files, backing up first. + + Returns the list of live paths that were updated. + """ + with open(os.path.join(staging_dir, "manifest.json")) as f: + manifest = json.load(f) + backup_dir = os.path.join(staging_dir, "backup") + updated: List[str] = [] + + if manifest.get("has_skill"): + live = manifest["live_skill_path"] + os.makedirs(os.path.dirname(live), exist_ok=True) + _backup(live, backup_dir) + shutil.copy2(os.path.join(staging_dir, "proposed_SKILL.md"), live) + updated.append(live) + if manifest.get("has_memory"): + live = manifest["live_memory_path"] + os.makedirs(os.path.dirname(live), exist_ok=True) + _backup(live, backup_dir) + shutil.copy2(os.path.join(staging_dir, "proposed_CLAUDE.md"), live) + updated.append(live) + return updated diff --git a/skillopt_sleep/state.py b/skillopt_sleep/state.py new file mode 100644 index 0000000..1e16157 --- /dev/null +++ b/skillopt_sleep/state.py @@ -0,0 +1,96 @@ +"""SkillOpt-Sleep — persistent cross-night state. + +state.json lives in ~/.skillopt-sleep and is the "long-term" store that +turns nightly episodes into durable competence (the Agent-Sleep paper's +short-term -> long-term transfer). It records: + + - night counter + - last harvest timestamp per project (so each night only sees new data) + - cross-night "slow/meta" memory (lessons that persisted across nights) + - per-night history (scores, accept/reject) for trend reporting +""" +from __future__ import annotations + +import json +import os +from typing import Any, Dict, List, Optional + + +def _now_iso(clock: Optional[float] = None) -> str: + # caller passes a timestamp; we avoid importing time at module import + import time as _t + return _t.strftime("%Y-%m-%dT%H:%M:%S", _t.localtime(clock if clock is not None else _t.time())) + + +DEFAULT_STATE: Dict[str, Any] = { + "version": 1, + "night": 0, + "last_harvest": {}, # project -> iso timestamp of last harvested record + "slow_memory": "", # cross-night consolidated lessons (meta-skill analogue) + "history": [], # list of per-night summaries + "task_archive": [], # capped list of past mined tasks (for associative recall) +} + + +class SleepState: + def __init__(self, path: str, data: Optional[Dict[str, Any]] = None) -> None: + self.path = path + self.data = data if data is not None else dict(DEFAULT_STATE) + + # io --------------------------------------------------------------------- + @classmethod + def load(cls, path: str) -> "SleepState": + if os.path.exists(path): + try: + with open(path) as f: + data = json.load(f) + merged = dict(DEFAULT_STATE) + merged.update(data if isinstance(data, dict) else {}) + return cls(path, merged) + except Exception: + pass + return cls(path, dict(DEFAULT_STATE)) + + def save(self) -> None: + os.makedirs(os.path.dirname(self.path), exist_ok=True) + tmp = self.path + ".tmp" + with open(tmp, "w") as f: + json.dump(self.data, f, ensure_ascii=False, indent=2) + os.replace(tmp, self.path) + + # accessors -------------------------------------------------------------- + @property + def night(self) -> int: + return int(self.data.get("night", 0)) + + def last_harvest_for(self, project: str) -> Optional[str]: + return self.data.get("last_harvest", {}).get(project) + + def set_last_harvest(self, project: str, iso_ts: str) -> None: + self.data.setdefault("last_harvest", {})[project] = iso_ts + + @property + def slow_memory(self) -> str: + return str(self.data.get("slow_memory", "")) + + def set_slow_memory(self, content: str) -> None: + self.data["slow_memory"] = content + + def begin_night(self, clock: Optional[float] = None) -> int: + self.data["night"] = self.night + 1 + return self.night + + def record_night(self, summary: Dict[str, Any]) -> None: + self.data.setdefault("history", []).append(summary) + + # ── task archive (associative-recall memory) ────────────────────────── + def task_archive(self) -> list: + """Past mined tasks as plain dicts (newest last).""" + return list(self.data.get("task_archive", [])) + + def add_to_archive(self, task_dicts: list, cap: int = 300) -> None: + """Append tonight's tasks; keep only the most recent ``cap``.""" + arc = self.data.setdefault("task_archive", []) + arc.extend(task_dicts) + if len(arc) > cap: + self.data["task_archive"] = arc[-cap:] diff --git a/skillopt_sleep/tasks_file.py b/skillopt_sleep/tasks_file.py new file mode 100644 index 0000000..d89166b --- /dev/null +++ b/skillopt_sleep/tasks_file.py @@ -0,0 +1,81 @@ +"""Reviewed task-file helpers for privacy-safe SkillOpt-Sleep runs.""" +from __future__ import annotations + +import json +import os +from typing import Any, Dict, List, Tuple + +from skillopt_sleep.mine import assign_splits, normalize_legacy_split +from skillopt_sleep.types import TaskRecord + + +def make_tasks_payload( + tasks: List[TaskRecord], + *, + project: str, + transcript_source: str = "", + n_sessions: int = 0, + target_skill_path: str = "", +) -> Dict[str, Any]: + return { + "format": "skillopt_sleep.tasks.v1", + "project": project, + "transcript_source": transcript_source, + "n_sessions": n_sessions, + "target_skill_path": target_skill_path, + "reviewed": False, + "tasks": [t.to_dict() for t in tasks], + } + + +def write_tasks_file(path: str, payload: Dict[str, Any]) -> str: + out = os.path.abspath(os.path.expanduser(path)) + parent = os.path.dirname(out) + if parent: + os.makedirs(parent, exist_ok=True) + with open(out, "w", encoding="utf-8") as f: + json.dump(payload, f, ensure_ascii=False, indent=2) + f.write("\n") + return out + + +def _normalize_tasks( + tasks: List[TaskRecord], + *, + holdout_fraction: float, + seed: int, +) -> List[TaskRecord]: + for task in tasks: + task.split = normalize_legacy_split(task.split or "train") + if len(tasks) >= 2 and not any(task.split in {"val", "test"} for task in tasks): + tasks = assign_splits(tasks, holdout_fraction=holdout_fraction, seed=seed) + return tasks + + +def load_tasks_file( + path: str, + *, + holdout_fraction: float = 0.34, + seed: int = 42, +) -> Tuple[List[TaskRecord], Dict[str, Any]]: + source = os.path.abspath(os.path.expanduser(path)) + with open(source, encoding="utf-8") as f: + payload = json.load(f) + if isinstance(payload, list): + meta: Dict[str, Any] = {"format": "skillopt_sleep.tasks.v1", "tasks_file": source} + raw_tasks = payload + elif isinstance(payload, dict): + meta = {k: v for k, v in payload.items() if k != "tasks"} + meta["tasks_file"] = source + raw_tasks = payload.get("tasks", []) + else: + raise ValueError("tasks file must contain a JSON object with tasks or a JSON task array") + if not isinstance(raw_tasks, list): + raise ValueError("tasks file field 'tasks' must be an array") + + tasks: List[TaskRecord] = [] + for item in raw_tasks: + if not isinstance(item, dict): + raise ValueError("each task entry must be a JSON object") + tasks.append(TaskRecord.from_dict(item)) + return _normalize_tasks(tasks, holdout_fraction=holdout_fraction, seed=seed), meta diff --git a/skillopt_sleep/types.py b/skillopt_sleep/types.py new file mode 100644 index 0000000..6cfa623 --- /dev/null +++ b/skillopt_sleep/types.py @@ -0,0 +1,146 @@ +"""SkillOpt-Sleep — core data types. + +These dataclasses are the interfaces between the sleep-cycle stages +(harvest -> mine -> replay -> consolidate -> stage). They are intentionally +plain (no slots, no heavy deps) so the package imports cleanly on any +Python 3.8+ interpreter and the deterministic experiment runs with zero +external dependencies. +""" +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import Any, Dict, List + +# ── Stage 1: harvest ────────────────────────────────────────────────────────── + +@dataclass +class SessionDigest: + """A normalized summary of one local agent session transcript. + + Produced by source-specific harvesters from Claude Code transcripts or + Codex Desktop archived sessions. + """ + + session_id: str + project: str + git_branch: str = "" + started_at: str = "" + ended_at: str = "" + user_prompts: List[str] = field(default_factory=list) + assistant_finals: List[str] = field(default_factory=list) + tools_used: List[str] = field(default_factory=list) + files_touched: List[str] = field(default_factory=list) + feedback_signals: List[str] = field(default_factory=list) # "still broken", "perfect", ... + n_user_turns: int = 0 + n_assistant_turns: int = 0 + raw_path: str = "" + + def to_dict(self) -> Dict[str, Any]: + return asdict(self) + + +# ── Stage 2: mine ───────────────────────────────────────────────────────────── + +@dataclass +class TaskRecord: + """A self-contained recurring task mined from one or more sessions. + + This is the *training unit* of the sleep cycle — the analogue of a + SkillOpt benchmark item. + """ + + id: str + project: str + intent: str # what the user wanted (the "question") + context_excerpt: str = "" # minimal context needed to attempt it + # Optional system framing for the rollout. When set (e.g. real benchmarks + # carrying the research repo's exact rollout_system), the backend uses THIS + # verbatim instead of its generic instruction wrapper — this keeps scoring + # faithful to the source task and avoids re-deriving framing the benchmark + # already bakes in. + system: str = "" + attempted_solution: str = "" # what the agent produced before + outcome: str = "unknown" # success | fail | mixed | unknown + reference_kind: str = "none" # exact | rubric | rule | none + reference: str = "" # exact answer, or rubric text + judge: Dict[str, Any] = field(default_factory=dict) # gbrain-style rule judge + tags: List[str] = field(default_factory=list) + source_sessions: List[str] = field(default_factory=list) + # split ∈ {train, val, test}. val + test come ONLY from real mined tasks and + # never overlap (val gates updates, test is the final held-out measure). train + # may be dream-augmented (see origin). Legacy values replay->train, + # holdout->val are normalized on load. + split: str = "train" + # origin ∈ {real, dream}. 'real' = mined from the user's actual sessions; + # 'dream' = synthetic/augmented for the training pool. Dream tasks are NEVER + # allowed into val/test, which is the anti-overfitting guarantee. + origin: str = "real" + derived_from: str = "" # for dream tasks: the real task id it varies + + def to_dict(self) -> Dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, d: Dict[str, Any]) -> "TaskRecord": + known = {f for f in cls.__dataclass_fields__} # type: ignore[attr-defined] + return cls(**{k: v for k, v in d.items() if k in known}) + + +# ── Stage 3: replay ─────────────────────────────────────────────────────────── + +@dataclass +class ReplayResult: + """Outcome of re-running one TaskRecord offline under a given skill+memory.""" + + id: str + hard: float = 0.0 # 0/1 exact, or continuous reward + soft: float = 0.0 # partial credit / judge score 0..1 + response: str = "" + fail_reason: str = "" + task_type: str = "task" + judge_rationale: str = "" + tools_called: List[str] = field(default_factory=list) + tokens: int = 0 # approx tokens this rollout cost (for token objective) + latency_ms: float = 0.0 # wall-clock for this rollout (for latency objective) + + def to_dict(self) -> Dict[str, Any]: + return asdict(self) + + +# ── Stage 4/5: consolidation report ─────────────────────────────────────────── + +@dataclass +class EditRecord: + """One bounded edit proposed/applied to skill or memory.""" + + target: str # "skill" | "memory" + op: str # add | delete | replace + content: str = "" + anchor: str = "" # for replace/delete: text being changed + rationale: str = "" + + +@dataclass +class SleepReport: + """Everything one night produced — written to staging for review.""" + + night: int + project: str + started_at: str = "" + ended_at: str = "" + n_sessions: int = 0 + n_tasks: int = 0 + n_replayed: int = 0 + baseline_score: float = 0.0 + candidate_score: float = 0.0 + accepted: bool = False + gate_action: str = "" + no_edits_reason: str = "" + edits: List[EditRecord] = field(default_factory=list) + rejected_edits: List[EditRecord] = field(default_factory=list) + tokens_used: int = 0 + notes: List[str] = field(default_factory=list) + + def to_dict(self) -> Dict[str, Any]: + d = asdict(self) + return d diff --git a/skillopt_webui/__init__.py b/skillopt_webui/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/skillopt_webui/__main__.py b/skillopt_webui/__main__.py new file mode 100644 index 0000000..6014cd1 --- /dev/null +++ b/skillopt_webui/__main__.py @@ -0,0 +1,3 @@ +# SkillOpt WebUI — `__main__` entry point +from skillopt_webui.app import main +main() diff --git a/skillopt_webui/app.py b/skillopt_webui/app.py new file mode 100644 index 0000000..e4978c5 --- /dev/null +++ b/skillopt_webui/app.py @@ -0,0 +1,660 @@ +""" +SkillOpt WebUI — Configure, launch, and monitor training from your browser. + +Usage: + python -m skillopt_webui.app [--port PORT] [--share] +""" +import argparse +import glob +import json +import os +import signal +import socket +import subprocess +import sys +import threading +from pathlib import Path +from urllib.parse import urlparse + +import gradio as gr +import yaml + +from skillopt.config import flatten_config +from skillopt.config import load_config as load_merged_config + +PROJECT_ROOT = Path(__file__).resolve().parent.parent + + +# ─── Config helpers ────────────────────────────────────────────────────────── + +def discover_configs() -> list[str]: + """Find all YAML configs under configs/.""" + pattern = str(PROJECT_ROOT / "configs" / "**" / "*.yaml") + paths = sorted(glob.glob(pattern, recursive=True)) + return [os.path.relpath(p, PROJECT_ROOT) for p in paths + if "_base_" not in p] + + +def load_config(path: str) -> dict: + """Load a YAML config file.""" + with open(PROJECT_ROOT / path) as f: + return yaml.safe_load(f) + + +def config_to_display(cfg: dict) -> str: + """Pretty-print config for display.""" + return yaml.dump(cfg, default_flow_style=False, sort_keys=False) + + +def _can_connect_to_url(url: str, timeout: float = 0.5) -> bool: + parsed = urlparse(url) + host = parsed.hostname + if not host: + return False + port = parsed.port or (443 if parsed.scheme == "https" else 80) + try: + with socket.create_connection((host, port), timeout=timeout): + return True + except OSError: + return False + + +def _load_env_file(path: Path, env: dict[str, str]) -> None: + for line in path.read_text().splitlines(): + line = line.strip() + if line.startswith("export "): + line = line[len("export "):].strip() + if line and not line.startswith("#") and "=" in line: + key, value = line.split("=", 1) + env[key.strip()] = value.strip().strip("\"'") + + +def build_training_env() -> dict[str, str]: + """Build the environment shared by preflight and the training subprocess.""" + env = os.environ.copy() + env["PYTHONUNBUFFERED"] = "1" + + dot_env = PROJECT_ROOT / ".env" + if dot_env.is_file(): + _load_env_file(dot_env, env) + + secrets_dir = PROJECT_ROOT / ".secrets" + if secrets_dir.is_dir(): + for env_file in sorted(secrets_dir.glob("*.env")): + _load_env_file(env_file, env) + + # Propagate OPTIMIZER_* to base AZURE_OPENAI_* when base is missing, + # so target/default endpoints inherit from optimizer config. + for suffix in ( + "ENDPOINT", "API_VERSION", "AUTH_MODE", "MANAGED_IDENTITY_CLIENT_ID", + "AD_SCOPE", "API_KEY", + ): + base_key = f"AZURE_OPENAI_{suffix}" + optimizer_key = f"OPTIMIZER_AZURE_OPENAI_{suffix}" + if not env.get(base_key) and env.get(optimizer_key): + env[base_key] = env[optimizer_key] + return env + + +def validate_training_config( + config_path: str, + overrides: dict, + env: dict[str, str] | None = None, +) -> str | None: + """Return an actionable preflight error, or None when training can start.""" + env = env or os.environ + cfg_options = [ + f"{key}={value}" for key, value in overrides.items() + if value is not None and value != "" + ] + try: + cfg = flatten_config(load_merged_config(str(PROJECT_ROOT / config_path), cfg_options)) + except Exception as exc: + return f"❌ Invalid config: {exc}" + + shared_endpoint = ( + cfg.get("azure_openai_endpoint") + or cfg.get("azure_endpoint") + or env.get("AZURE_OPENAI_ENDPOINT") + ) + missing_openai_roles = [] + for role in ("optimizer", "target"): + if cfg.get(f"{role}_backend") != "openai_chat": + continue + role_endpoint = ( + cfg.get(f"{role}_azure_openai_endpoint") + or env.get(f"{role.upper()}_AZURE_OPENAI_ENDPOINT") + or shared_endpoint + ) + if not role_endpoint: + missing_openai_roles.append(role) + if missing_openai_roles: + configured_backend = cfg.get("model_backend") + detail = "" + if configured_backend in {"qwen", "qwen_chat"}: + detail = ( + "\nNote: model.backend is qwen, but explicit optimizer_backend/" + "target_backend values are still openai_chat." + ) + return ( + "❌ Model backend is not ready: missing Azure/OpenAI-compatible endpoint " + f"for {', '.join(missing_openai_roles)}.\n" + "Set model.azure_openai_endpoint (or AZURE_OPENAI_ENDPOINT), or change " + "the role backends to the backend you intend to use." + f"{detail}" + ) + + qwen_failures = [] + qwen_shared = ( + cfg.get("qwen_chat_base_url") + or env.get("QWEN_CHAT_BASE_URL") + or "http://localhost:8000/v1" + ) + for role in ("optimizer", "target"): + if cfg.get(f"{role}_backend") != "qwen_chat": + continue + base_url = ( + cfg.get(f"{role}_qwen_chat_base_url") + or env.get(f"{role.upper()}_QWEN_CHAT_BASE_URL") + or qwen_shared + ) + if not _can_connect_to_url(str(base_url)): + qwen_failures.append(f"{role}={base_url}") + if qwen_failures: + return ( + "❌ Model backend is not ready: cannot connect to qwen_chat endpoint " + f"for {', '.join(qwen_failures)}.\n" + "Start your OpenAI-compatible Qwen/vLLM server, or set " + "model.qwen_chat_base_url / OPTIMIZER_QWEN_CHAT_BASE_URL / " + "TARGET_QWEN_CHAT_BASE_URL to the correct URL." + ) + return None + + +# ─── Training process management ──────────────────────────────────────────── + +class TrainingManager: + """Manages a single training subprocess.""" + + def __init__(self): + self._lock = threading.Lock() + self.process = None + self.log_lines: list[str] = [] + self.stage = "Idle" + self.step = 0 + self.total_steps = 0 + self.epoch = 0 + self.total_epochs = 0 + self.running = False + + def start(self, config_path: str, overrides: dict) -> str: + with self._lock: + if self.running: + return "⚠️ Training already running. Stop it first." + + env = build_training_env() + preflight_error = validate_training_config(config_path, overrides, env) + if preflight_error: + return preflight_error + + cmd = [ + sys.executable, "scripts/train.py", + "--config", config_path, + ] + cfg_options = [] + for k, v in overrides.items(): + if v is not None and v != "": + cfg_options.append(f"{k}={v}") + if cfg_options: + cmd.append("--cfg-options") + cmd.extend(cfg_options) + + try: + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + cwd=str(PROJECT_ROOT), + bufsize=1, + env=env, + start_new_session=True, # create process group for clean kill + ) + except Exception as e: + return f"❌ Failed to start training: {e}" + + with self._lock: + self.process = proc + self.log_lines = [f"$ {' '.join(cmd)}\n"] + self.stage = "Starting" + self.step = 0 + self.total_steps = 0 + self.epoch = 0 + self.total_epochs = 0 + self.running = True + + thread = threading.Thread(target=self._read_output, daemon=True) + thread.start() + + return "✅ Training started!" + + def _read_output(self): + for line in self.process.stdout: + with self._lock: + self.log_lines.append(line) + self._parse_stage(line) + if len(self.log_lines) > 5000: + self.log_lines = self.log_lines[-4000:] + self.process.wait() + with self._lock: + self.running = False + self.stage = f"Finished (exit={self.process.returncode})" + + def _parse_stage(self, line: str): + line_lower = line.lower() + if "1/6 rollout" in line_lower or ("rollout" in line_lower and "worker" in line_lower): + self.stage = "🎯 Rollout" + elif "2/6 reflect" in line_lower or ("reflect" in line_lower and "patch" in line_lower): + self.stage = "🔍 Reflect" + elif "3/6 aggregate" in line_lower or "merge" in line_lower: + self.stage = "🔗 Aggregate" + elif "4/6 select" in line_lower: + self.stage = "✂️ Select" + elif "5/6 update" in line_lower: + self.stage = "📝 Update" + elif "6/6" in line_lower or ("gate" in line_lower and "score" in line_lower): + self.stage = "🚦 Gate" + elif "slow update" in line_lower: + self.stage = "🔄 Slow Update" + elif "meta skill" in line_lower: + self.stage = "🧠 Meta Skill" + elif "baseline" in line_lower and "evaluate" in line_lower: + self.stage = "📊 Baseline" + if "[step" in line_lower: + try: + parts = line.split("[STEP")[1].split("]")[0].split("/") + self.step = int(parts[0].strip()) + self.total_steps = int(parts[1].strip()) + except (IndexError, ValueError): + pass + if "[epoch" in line_lower: + try: + parts = line.split("[EPOCH")[1].split("]")[0].split("/") + self.epoch = int(parts[0].strip()) + self.total_epochs = int(parts[1].strip()) + except (IndexError, ValueError): + pass + + def stop(self) -> str: + with self._lock: + if self.process and self.running: + try: + # Kill entire process group (children included) + os.killpg(os.getpgid(self.process.pid), signal.SIGTERM) + except (ProcessLookupError, OSError): + self.process.terminate() + self.process.wait(timeout=5) + self.running = False + self.stage = "Stopped" + return "🛑 Training stopped." + return "No training running." + + def get_logs(self) -> str: + with self._lock: + return "".join(self.log_lines[-500:]) + + def get_colored_logs_html(self) -> str: + """Render last 300 log lines with color-coded stages.""" + import html as html_mod + with self._lock: + lines = list(self.log_lines[-300:]) + parts = [] + for line in lines: + # Rebrand: display "skillopt" instead of "reflact" in logs + line_display = line.replace("reflact", "skillopt").replace("ReflACT", "SkillOpt").replace("Reflact", "Skillopt").replace("REFLACT", "SKILLOPT") + escaped = html_mod.escape(line_display.rstrip("\n")) + low = line.lower() + if "[epoch" in low: + color = "#f59e0b" # amber + weight = "700" + elif "[step" in low: + color = "#8b5cf6" # purple + weight = "700" + elif "rollout]" in low or "1/6" in low: + color = "#3b82f6" # blue + elif "reflect" in low or "2/6" in low: + color = "#f97316" # orange + elif "aggregate" in low or "3/6" in low or "merge" in low: + color = "#06b6d4" # cyan + elif "select" in low or "4/6" in low: + color = "#ec4899" # pink + elif "update" in low or "5/6" in low: + color = "#10b981" # green + elif "gate" in low or "6/6" in low: + color = "#ef4444" # red + elif "slow update" in low: + color = "#f59e0b" # amber + weight = "700" + elif "meta skill" in low: + color = "#a855f7" # violet + weight = "700" + elif "baseline" in low: + color = "#6366f1" # indigo + weight = "700" + elif "[rollout]" in low: + # per-item rollout progress + if "hard=1" in line: + color = "#22c55e" # green for correct + elif "hard=0" in line: + color = "#f87171" # red for wrong + elif "timeout" in low: + color = "#fbbf24" # yellow for timeout + else: + color = "#94a3b8" # gray + weight = "400" + elif "error" in low or "fail" in low: + color = "#ef4444" + weight = "700" + elif "========" in line: + color = "#64748b" # separator + weight = "400" + else: + color = "#e2e8f0" # default light gray + weight = "400" + if "weight" not in dir(): + weight = "400" + parts.append(f'{escaped}') + weight = "400" # reset + + log_html = "
".join(parts) if parts else 'No logs yet. Click Refresh after launching training.' + return f'''
{log_html}
''' + + def get_progress_html(self) -> str: + """Render a visual progress bar.""" + s = self.get_status() + step = s["step"] + total = s["total_steps"] + epoch = self.epoch + total_epochs = self.total_epochs + pct = s["progress"] * 100 + + if not self.running and step == 0: + return '
Waiting for training to start...
' + + # Color based on progress + if pct < 25: + bar_color = "linear-gradient(90deg, #3b82f6, #6366f1)" + elif pct < 50: + bar_color = "linear-gradient(90deg, #6366f1, #8b5cf6)" + elif pct < 75: + bar_color = "linear-gradient(90deg, #8b5cf6, #a855f7)" + else: + bar_color = "linear-gradient(90deg, #a855f7, #22c55e)" + + stage_icon = self.stage if self.stage != "Idle" else "⏳" + status_dot = "🟢" if self.running else ("✅" if "Finished" in self.stage else "⚪") + + epoch_str = f"Epoch {epoch}/{total_epochs}" if total_epochs > 0 else "" + step_str = f"Step {step}/{total}" if total > 0 else "" + + return f''' +
+
+ {status_dot} {stage_icon} + {epoch_str}   {step_str} + {pct:.1f}% +
+
+
+
+
''' + + def get_status(self) -> dict: + with self._lock: + progress = 0 + if self.total_steps > 0: + progress = self.step / self.total_steps + return { + "running": self.running, + "stage": self.stage, + "step": self.step, + "total_steps": self.total_steps, + "progress": progress, + } + + +manager = TrainingManager() + + +# ─── Pipeline Stage HTML ──────────────────────────────────────────────────── + +STAGES = ["Rollout", "Reflect", "Aggregate", "Select", "Update", "Gate"] +STAGE_ICONS = ["🎯", "🔍", "🔗", "✂️", "📝", "🚦"] + + +def render_pipeline_html(active_stage: str = "") -> str: + """Render animated pipeline HTML.""" + html = '
' + for i, (name, icon) in enumerate(zip(STAGES, STAGE_ICONS)): + is_active = name.lower() in active_stage.lower() if active_stage else False + bg = "#6366f1" if is_active else "#f3f4f6" + color = "white" if is_active else "#374151" + border = "3px solid #4f46e5" if is_active else "2px solid #d1d5db" + shadow = "0 0 20px rgba(99,102,241,0.4)" if is_active else "none" + pulse = "animation: pulse 1.5s ease-in-out infinite;" if is_active else "" + html += f''' +
+ {icon} + {name} +
''' + if i < len(STAGES) - 1: + arrow_color = "#6366f1" if is_active else "#d1d5db" + html += f'
→
' + html += '
' + html += '' + return html + + +# ─── Gradio UI ────────────────────────────────────────────────────────────── + +def build_ui(): + configs = discover_configs() + + with gr.Blocks( + title="SkillOpt WebUI", + ) as app: + gr.Markdown("# 🧠 SkillOpt Training Dashboard") + gr.Markdown("*SKILLOPT: Executive Strategy for Self-Evolving Agent Skills — Configure, launch, and monitor training.*") + + with gr.Tabs(): + # ── Tab 1: Configure & Launch ──────────────────────────── + with gr.Tab("⚙️ Configure & Launch"): + with gr.Row(): + with gr.Column(scale=1): + config_dropdown = gr.Dropdown( + choices=configs, + label="Config File", + value=configs[0] if configs else None, + ) + config_preview = gr.Code( + label="Config Preview", + language="yaml", + interactive=False, + ) + + with gr.Column(scale=1): + gr.Markdown("### Hyperparameters (DL Analogy)") + lr = gr.Slider(1, 32, value=4, step=1, + label="Learning Rate (max edits/step)") + scheduler = gr.Dropdown( + ["cosine", "linear", "constant", "autonomous"], + value="cosine", + label="LR Scheduler", + ) + num_epochs = gr.Slider(1, 8, value=4, step=1, + label="Epochs") + batch_size = gr.Slider(10, 100, value=40, step=5, + label="Batch Size (tasks per step)") + analyst_workers = gr.Slider(1, 32, value=16, step=1, + label="Analyst Workers (parallel reflection)") + use_slow_update = gr.Checkbox(value=True, + label="Slow Update (epoch-boundary momentum)") + use_meta_skill = gr.Checkbox(value=True, + label="Meta Skill (cross-epoch optimizer memory)") + use_gate = gr.Checkbox(value=True, + label="Gate (validation-based accept/reject)") + + with gr.Row(): + launch_btn = gr.Button("🚀 Launch Training", + variant="primary", size="lg") + stop_btn = gr.Button("🛑 Stop", variant="stop") + + status_text = gr.Textbox(label="Status", interactive=False) + + def on_config_change(path): + if path: + try: + return config_to_display(load_config(path)) + except Exception as e: + return f"Error: {e}" + return "" + + config_dropdown.change(on_config_change, config_dropdown, config_preview) + + def on_launch(cfg_path, lr_val, sched, epochs, batch, workers, + slow_update, meta_skill, gate): + overrides = { + "optimizer.learning_rate": lr_val, + "optimizer.lr_scheduler": sched, + "train.num_epochs": epochs, + "train.batch_size": batch, + "gradient.analyst_workers": workers, + "optimizer.use_slow_update": slow_update, + "optimizer.use_meta_skill": meta_skill, + "evaluation.use_gate": gate, + } + return manager.start(cfg_path, overrides) + + launch_btn.click( + on_launch, + [config_dropdown, lr, scheduler, num_epochs, batch_size, + analyst_workers, use_slow_update, use_meta_skill, use_gate], + status_text, + ) + stop_btn.click(lambda: manager.stop(), outputs=status_text) + + # ── Tab 2: Monitor ─────────────────────────────────────── + with gr.Tab("📊 Monitor"): + pipeline_html = gr.HTML( + value=render_pipeline_html(), + label="Pipeline Stage", + ) + + progress_html = gr.HTML( + value=manager.get_progress_html(), + label="Progress", + ) + + log_html = gr.HTML( + value=manager.get_colored_logs_html(), + label="Training Logs", + ) + + refresh_btn = gr.Button("🔄 Refresh Logs", variant="primary", size="lg") + + def on_refresh(): + s = manager.get_status() + pipeline = render_pipeline_html(s["stage"]) + progress = manager.get_progress_html() + logs = manager.get_colored_logs_html() + return pipeline, progress, logs + + refresh_btn.click( + on_refresh, + outputs=[pipeline_html, progress_html, log_html], + ) + + # ── Tab 3: Results ─────────────────────────────────────── + with gr.Tab("📈 Results"): + gr.Markdown("### Output Explorer") + output_dir = gr.Textbox( + label="Output Directory", + value="outputs/", + interactive=True, + ) + scan_btn = gr.Button("🔍 Scan Results") + results_table = gr.Dataframe( + headers=["Experiment", "Benchmark", "Best Score", "Steps"], + label="Experiments", + ) + + def scan_outputs(out_dir): + rows = [] + base = PROJECT_ROOT / out_dir + if not base.exists(): + return rows + for bench_dir in sorted(base.iterdir()): + if not bench_dir.is_dir(): + continue + for run_dir in sorted(bench_dir.iterdir()): + if not run_dir.is_dir(): + continue + cfg_file = run_dir / "config.yaml" + score = "—" + steps = "—" + if cfg_file.exists(): + try: + c = yaml.safe_load(cfg_file.read_text()) + steps = str(c.get("train", {}).get("num_steps", "—")) + except Exception: + pass + # Try to find best score from logs + for log_f in run_dir.glob("**/*.jsonl"): + try: + with open(log_f) as f: + for line in f: + d = json.loads(line) + if "score" in d: + score = f"{d['score']:.4f}" + except Exception: + pass + rows.append([ + run_dir.name, + bench_dir.name, + score, + steps, + ]) + return rows + + scan_btn.click(scan_outputs, output_dir, results_table) + + return app + + +def main(): + parser = argparse.ArgumentParser(description="SkillOpt WebUI") + parser.add_argument("--port", type=int, default=7860) + parser.add_argument("--share", action="store_true") + parser.add_argument("--host", type=str, default="0.0.0.0", + help="Server host. Use 0.0.0.0 for public access.") + args = parser.parse_args() + + app = build_ui() + app.launch( + server_name=args.host, + server_port=args.port, + share=args.share, + theme=gr.themes.Soft(primary_hue="indigo"), + ) + + +if __name__ == "__main__": + main() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_alfworld_paths.py b/tests/test_alfworld_paths.py new file mode 100644 index 0000000..eb0229b --- /dev/null +++ b/tests/test_alfworld_paths.py @@ -0,0 +1,33 @@ +import os + +from skillopt.envs.alfworld.rollout import _resolve_alfworld_gamefile, _resolve_alfworld_gamefiles + + +def test_resolve_alfworld_gamefile_uses_alfworld_data_for_relative_paths(monkeypatch, tmp_path): + data_root = tmp_path / "alfworld_data" + monkeypatch.setenv("ALFWORLD_DATA", str(data_root)) + + resolved = _resolve_alfworld_gamefile("json_2.1.1/valid_seen/task/game.tw-pddl") + + assert resolved == os.path.join(str(data_root), "json_2.1.1/valid_seen/task/game.tw-pddl") + + +def test_resolve_alfworld_gamefile_keeps_absolute_paths(monkeypatch, tmp_path): + monkeypatch.setenv("ALFWORLD_DATA", str(tmp_path / "alfworld_data")) + absolute = tmp_path / "elsewhere" / "game.tw-pddl" + + assert _resolve_alfworld_gamefile(str(absolute)) == str(absolute) + + +def test_resolve_alfworld_gamefile_keeps_relative_path_without_alfworld_data(monkeypatch): + monkeypatch.delenv("ALFWORLD_DATA", raising=False) + + assert _resolve_alfworld_gamefile("json_2.1.1/train/task/game.tw-pddl") == ( + "json_2.1.1/train/task/game.tw-pddl" + ) + + +def test_resolve_alfworld_gamefiles_handles_none(monkeypatch): + monkeypatch.setenv("ALFWORLD_DATA", "/tmp/alfworld_data") + + assert _resolve_alfworld_gamefiles(None) is None diff --git a/tests/test_devin_plugin.py b/tests/test_devin_plugin.py new file mode 100644 index 0000000..fb276b9 --- /dev/null +++ b/tests/test_devin_plugin.py @@ -0,0 +1,98 @@ +"""Tests for the Devin MCP plugin: tool schema, ATIF-v1.7 harvest, path expansion.""" +import importlib +import json +import os +import sys +import tempfile +import unittest + +# Allow importing from the plugin directory (mirrors tests/test_mcp_schema.py) +PLUGIN = os.path.join(os.path.dirname(__file__), "..", "plugins", "devin") +sys.path.insert(0, PLUGIN) + +import mcp_server # noqa: E402 +import harvest_devin as hw # noqa: E402 + +FIXTURES = os.path.join(PLUGIN, "fixtures") + + +def _read_jsonl(path): + with open(path, encoding="utf-8") as f: + return [json.loads(line) for line in f if line.strip()] + + +def _find_session_jsonl(out_dir): + for root, _dirs, files in os.walk(os.path.join(out_dir, "projects")): + for name in files: + if name.endswith(".jsonl"): + return _read_jsonl(os.path.join(root, name)) + raise AssertionError("no session jsonl written") + + +class TestDevinMcpSchema(unittest.TestCase): + def test_tools_are_the_sleep_interface(self): + names = {t["name"] for t in mcp_server.TOOLS} + self.assertEqual(names, {"sleep_status", "sleep_dry_run", "sleep_run", + "sleep_adopt", "sleep_harvest", + "sleep_schedule", "sleep_unschedule"}) + + def test_actions_map_to_engine_subcommands(self): + expected = {"sleep_status": "status", "sleep_dry_run": "dry-run", + "sleep_run": "run", "sleep_adopt": "adopt", + "sleep_harvest": "harvest", "sleep_schedule": "schedule", + "sleep_unschedule": "unschedule"} + for t in mcp_server.TOOLS: + self.assertEqual(t["action"], expected[t["name"]]) + + def test_backends_in_enum(self): + backends = mcp_server._TOOL_SCHEMA["properties"]["backend"]["enum"] + for b in ["mock", "claude", "codex", "copilot"]: + self.assertIn(b, backends) + + def test_schema_has_key_engine_params(self): + # parity with plugins/copilot's schema (tests/test_plugin_sync.py) + props = set(mcp_server._TOOL_SCHEMA["properties"].keys()) + for param in {"project", "backend", "scope", "source", "model", + "tasks_file", "target_skill_path", "max_sessions", + "max_tasks", "lookback_hours", "auto_adopt", "json", + "edit_budget", "hour", "minute"}: + self.assertIn(param, props) + + +class TestClaudeHomeExpansion(unittest.TestCase): + """Regression: ~ must be expanded even when CLAUDE_HOME comes from the env + (the documented mcp-config sets SKILLOPT_DEVIN_CLAUDE_HOME="~/...").""" + + def test_env_tilde_is_expanded(self): + os.environ["SKILLOPT_DEVIN_CLAUDE_HOME"] = "~/.skillopt-sleep-devin" + try: + importlib.reload(mcp_server) + self.assertFalse(mcp_server.CLAUDE_HOME.startswith("~")) + self.assertEqual(mcp_server.CLAUDE_HOME, + os.path.expanduser("~/.skillopt-sleep-devin")) + finally: + del os.environ["SKILLOPT_DEVIN_CLAUDE_HOME"] + importlib.reload(mcp_server) + + +class TestDevinHarvest(unittest.TestCase): + def test_atif_fixture_yields_gradeable_task(self): + with tempfile.TemporaryDirectory() as out: + n = hw.harvest_devin_transcripts(FIXTURES, out, ["/tmp/proj"]) + self.assertEqual(n, 1) + + outcomes = _read_jsonl(os.path.join(out, "outcomes.jsonl")) + self.assertEqual(len(outcomes), 1) + o = outcomes[0] + self.assertEqual(o["verifier"], "tests") + self.assertTrue(o["success"]) + self.assertIn("repro", o["reference"]) + + # the converted transcript carries the grouping key on the user turn + session = _find_session_jsonl(out) + user_turn = next(r for r in session if r["type"] == "user") + self.assertIn("taskKey", user_turn) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_gate.py b/tests/test_gate.py new file mode 100644 index 0000000..7a54234 --- /dev/null +++ b/tests/test_gate.py @@ -0,0 +1,286 @@ +"""Tests for skillopt.evaluation.gate — the validation gate decision function. + +The gate is the optimizer's model-selection / early-stopping core: given a +candidate skill's score, it decides whether to accept it as the new current +skill and whether it becomes the new best-so-far. These are pure functions, +so they can be exercised directly without any LLM or rollout. +""" +from __future__ import annotations + +import dataclasses + +import pytest + +from skillopt.evaluation.gate import ( + GateResult, + evaluate_gate, + select_gate_score, +) + + +class TestSelectGateScore: + """select_gate_score — project (hard, soft) onto a single comparison metric.""" + + def test_hard_metric_returns_hard(self) -> None: + assert select_gate_score(0.8, 0.3, "hard") == 0.8 + + def test_soft_metric_returns_soft(self) -> None: + assert select_gate_score(0.8, 0.3, "soft") == 0.3 + + def test_default_metric_is_hard(self) -> None: + assert select_gate_score(0.42, 0.99) == 0.42 + + def test_mixed_metric_default_weight(self) -> None: + # (1 - 0.5) * 1.0 + 0.5 * 0.0 == 0.5 + assert select_gate_score(1.0, 0.0, "mixed") == pytest.approx(0.5) + + def test_mixed_metric_custom_weight(self) -> None: + # (1 - 0.25) * 0.8 + 0.25 * 0.4 == 0.7 + assert select_gate_score(0.8, 0.4, "mixed", 0.25) == pytest.approx(0.7) + + def test_mixed_weight_zero_equals_hard(self) -> None: + assert select_gate_score(0.8, 0.3, "mixed", 0.0) == pytest.approx(0.8) + + def test_mixed_weight_one_equals_soft(self) -> None: + assert select_gate_score(0.8, 0.3, "mixed", 1.0) == pytest.approx(0.3) + + def test_mixed_weight_clamped_above_one(self) -> None: + """Out-of-range weight is clamped to 1.0 (→ pure soft).""" + assert select_gate_score(0.8, 0.3, "mixed", 5.0) == pytest.approx(0.3) + + def test_mixed_weight_clamped_below_zero(self) -> None: + """Negative weight is clamped to 0.0 (→ pure hard).""" + assert select_gate_score(0.8, 0.3, "mixed", -2.0) == pytest.approx(0.8) + + def test_returns_float(self) -> None: + assert isinstance(select_gate_score(1, 0, "hard"), float) + + def test_unknown_metric_raises(self) -> None: + with pytest.raises(ValueError, match="unknown gate metric"): + select_gate_score(0.5, 0.5, "rouge") # type: ignore[arg-type] + + +class TestEvaluateGateAcceptNewBest: + """evaluate_gate — candidate beats both current and best.""" + + def test_accept_new_best_action_and_state(self) -> None: + result = evaluate_gate( + candidate_skill="CAND", + cand_hard=0.9, + current_skill="CURR", + current_score=0.5, + best_skill="BEST", + best_score=0.5, + best_step=3, + global_step=7, + ) + assert result.action == "accept_new_best" + assert result.current_skill == "CAND" + assert result.current_score == pytest.approx(0.9) + assert result.best_skill == "CAND" + assert result.best_score == pytest.approx(0.9) + assert result.best_step == 7 # updated to the accepting step + + +class TestEvaluateGateAccept: + """evaluate_gate — candidate beats current but not best. + + This branch is only reachable when ``current_score < best_score``; it + advances the current skill without disturbing the best-so-far checkpoint. + """ + + def test_accept_updates_current_only(self) -> None: + result = evaluate_gate( + candidate_skill="CAND", + cand_hard=0.6, + current_skill="CURR", + current_score=0.4, + best_skill="BEST", + best_score=0.8, + best_step=2, + global_step=9, + ) + assert result.action == "accept" + assert result.current_skill == "CAND" + assert result.current_score == pytest.approx(0.6) + # best-so-far is preserved, including its step + assert result.best_skill == "BEST" + assert result.best_score == pytest.approx(0.8) + assert result.best_step == 2 + + def test_tie_with_best_but_above_current_accepts(self) -> None: + """cand == best (not strictly greater) but > current → accept, not new best.""" + result = evaluate_gate( + candidate_skill="CAND", + cand_hard=0.8, + current_skill="CURR", + current_score=0.5, + best_skill="BEST", + best_score=0.8, + best_step=1, + global_step=4, + ) + assert result.action == "accept" + assert result.current_skill == "CAND" + assert result.best_skill == "BEST" + assert result.best_score == pytest.approx(0.8) + assert result.best_step == 1 + + +class TestEvaluateGateReject: + """evaluate_gate — candidate does not beat current.""" + + def test_reject_below_current(self) -> None: + result = evaluate_gate( + candidate_skill="CAND", + cand_hard=0.3, + current_skill="CURR", + current_score=0.5, + best_skill="BEST", + best_score=0.8, + best_step=2, + global_step=6, + ) + assert result.action == "reject" + assert result.current_skill == "CURR" + assert result.current_score == pytest.approx(0.5) + assert result.best_skill == "BEST" + assert result.best_score == pytest.approx(0.8) + assert result.best_step == 2 + + def test_tie_with_current_rejects(self) -> None: + """Strict inequality: cand == current is rejected (no lateral moves).""" + result = evaluate_gate( + candidate_skill="CAND", + cand_hard=0.5, + current_skill="CURR", + current_score=0.5, + best_skill="BEST", + best_score=0.5, + best_step=0, + global_step=3, + ) + assert result.action == "reject" + assert result.current_skill == "CURR" + assert result.best_skill == "BEST" + + +class TestEvaluateGateMetrics: + """evaluate_gate — non-hard metrics drive the comparison via cand_soft.""" + + def test_soft_metric_uses_cand_soft(self) -> None: + # High hard, low soft: under 'soft' the candidate must be rejected. + result = evaluate_gate( + candidate_skill="CAND", + cand_hard=0.95, + current_skill="CURR", + current_score=0.5, + best_skill="BEST", + best_score=0.5, + best_step=0, + global_step=1, + cand_soft=0.2, + metric="soft", + ) + assert result.action == "reject" + + def test_mixed_metric_uses_weighted_score(self) -> None: + # mixed w=0.5: (0.5 * 1.0) + (0.5 * 0.6) == 0.8 > current 0.5 and best 0.5 + result = evaluate_gate( + candidate_skill="CAND", + cand_hard=1.0, + current_skill="CURR", + current_score=0.5, + best_skill="BEST", + best_score=0.5, + best_step=0, + global_step=2, + cand_soft=0.6, + metric="mixed", + mixed_weight=0.5, + ) + assert result.action == "accept_new_best" + assert result.current_score == pytest.approx(0.8) + assert result.best_score == pytest.approx(0.8) + + def test_default_metric_ignores_soft(self) -> None: + """Default metric is 'hard'; cand_soft must not affect the decision.""" + result = evaluate_gate( + candidate_skill="CAND", + cand_hard=0.9, + current_skill="CURR", + current_score=0.5, + best_skill="BEST", + best_score=0.5, + best_step=0, + global_step=1, + cand_soft=0.0, + ) + assert result.action == "accept_new_best" + assert result.current_score == pytest.approx(0.9) + + +class TestGateResult: + """GateResult — immutable outcome dataclass.""" + + def test_fields(self) -> None: + result = GateResult( + action="accept", + current_skill="c", + current_score=0.5, + best_skill="b", + best_score=0.9, + best_step=4, + ) + assert result.action == "accept" + assert result.current_skill == "c" + assert result.current_score == 0.5 + assert result.best_skill == "b" + assert result.best_score == 0.9 + assert result.best_step == 4 + + def test_is_frozen(self) -> None: + result = GateResult( + action="reject", + current_skill="c", + current_score=0.0, + best_skill="b", + best_score=0.0, + best_step=0, + ) + with pytest.raises(dataclasses.FrozenInstanceError): + result.current_score = 1.0 # type: ignore[misc] + + +class TestGateInvariants: + """Behavioral invariants of the gate over a sequence of steps.""" + + def test_current_tracks_best_from_equal_start(self) -> None: + """When current == best at the start, every acceptance is a new best, so + the two stay locked together and the 'accept' branch is never taken. + + This documents the trainer's ``s_cur``/``s_best`` usage: they are + initialized equal and updated only through this gate. + """ + current_skill, current_score = "S0", 0.2 + best_skill, best_score, best_step = "S0", 0.2, 0 + for step, cand in enumerate([0.1, 0.5, 0.4, 0.7], start=1): + result = evaluate_gate( + candidate_skill=f"S{step}", + cand_hard=cand, + current_skill=current_skill, + current_score=current_score, + best_skill=best_skill, + best_score=best_score, + best_step=best_step, + global_step=step, + ) + current_skill, current_score = result.current_skill, result.current_score + best_skill = result.best_skill + best_score = result.best_score + best_step = result.best_step + assert result.action in {"accept_new_best", "reject"} + assert current_score == best_score + assert current_skill == best_skill + assert best_score == pytest.approx(0.7) + assert best_step == 4 diff --git a/tests/test_handoff_backend.py b/tests/test_handoff_backend.py new file mode 100644 index 0000000..0ecfae1 --- /dev/null +++ b/tests/test_handoff_backend.py @@ -0,0 +1,220 @@ +"""Tests for the handoff backend: session-executed model calls via +prompt/answer files, resumed across engine runs.""" +from __future__ import annotations + +import json +import os +import re +import tempfile +import unittest + +from skillopt_sleep.backend import get_backend +from skillopt_sleep.config import load_config +from skillopt_sleep.cycle import run_sleep_cycle +from skillopt_sleep.handoff_backend import ( + PENDING_SENTINEL_PREFIX, + HandoffBackend, + PendingCalls, +) +from skillopt_sleep.mine import assign_splits +from skillopt_sleep.types import TaskRecord + +# The rule the simulated executor "learns"; once present in the skill the +# executor answers arithmetic tasks correctly, mirroring MockBackend's +# rule-gated model of reality. +RULE = "Always answer with just the number." + + +def _tasks(): + ts = [ + TaskRecord( + id=f"t{i}", project="/p", + intent=f"What is {i} + {i}?", + reference_kind="exact", reference=str(i + i), + ) + for i in range(1, 7) + ] + return assign_splits(ts, holdout_fraction=0.34, seed=42) + + +def _answer_pending(backend: HandoffBackend) -> None: + """Deterministic stand-in for the interactive session answering PROMPTS.md.""" + for key, item in list(backend.pending.items()): + prompt = str(item["prompt"]) + if "You are SkillOpt's optimizer" in prompt: + ans = json.dumps([{ + "op": "add", "content": RULE, + "rationale": "outputs failed exact match; answer bare numbers", + }]) + else: + m = re.search(r"What is (\d+) \+ (\d+)\?", prompt) + if m and RULE in prompt: + ans = str(int(m.group(1)) + int(m.group(2))) + else: + ans = "cannot say" + with open(backend.answer_path(key), "w", encoding="utf-8") as f: + f.write(ans) + + +class TestHandoffBackendUnit(unittest.TestCase): + def test_miss_records_pending_and_returns_sentinel(self): + with tempfile.TemporaryDirectory() as hdir: + be = HandoffBackend(handoff_dir=hdir) + out = be._call("some prompt") + self.assertTrue(out.startswith(PENDING_SENTINEL_PREFIX)) + self.assertEqual(len(be.pending), 1) + + def test_answer_file_resolves_call(self): + with tempfile.TemporaryDirectory() as hdir: + be = HandoffBackend(handoff_dir=hdir) + key = be._call("what is 2+2?").split(":")[-1].rstrip("]") + with open(be.answer_path(key), "w", encoding="utf-8") as f: + f.write("4\n") + fresh = HandoffBackend(handoff_dir=hdir) + self.assertEqual(fresh._call("what is 2+2?"), "4") + self.assertEqual(len(fresh.pending), 0) + + def test_dependent_prompt_raises(self): + with tempfile.TemporaryDirectory() as hdir: + be = HandoffBackend(handoff_dir=hdir) + placeholder = be._call("first question") + with self.assertRaises(PendingCalls) as ctx: + be._call(f"judge this response: {placeholder}") + self.assertEqual(len(ctx.exception.pending), 1) + + def test_reflect_retry_of_pending_reply_raises(self): + with tempfile.TemporaryDirectory() as hdir: + be = HandoffBackend(handoff_dir=hdir) + be._call("reflect on failures") + with self.assertRaises(PendingCalls): + be._call("reflect on failures\n\nIMPORTANT: " + "your previous reply was not valid JSON. Reply with " + "ONLY the JSON array, no prose, no markdown fences.") + + def test_flush_writes_prompts_and_pending_json(self): + with tempfile.TemporaryDirectory() as hdir: + be = HandoffBackend(handoff_dir=hdir) + be._call("prompt A") + be._call("prompt B") + prompts_path = be.flush_pending() + self.assertTrue(os.path.exists(prompts_path)) + with open(os.path.join(hdir, "pending.json"), encoding="utf-8") as f: + payload = json.load(f) + self.assertEqual(payload["format"], "skillopt_sleep.handoff.v1") + self.assertEqual(len(payload["pending"]), 2) + self.assertEqual(payload["pending"][0]["prompt"], "prompt A") + with open(prompts_path, encoding="utf-8") as f: + md = f.read() + self.assertIn("BEGIN PROMPT", md) + self.assertIn("prompt B", md) + + def test_get_backend_registration(self): + with tempfile.TemporaryDirectory() as proj: + be = get_backend("handoff", project_dir=proj) + self.assertIsInstance(be, HandoffBackend) + self.assertTrue(be.handoff_dir.startswith(os.path.realpath(proj)) + or be.handoff_dir.startswith(proj)) + + +class TestHandoffCycle(unittest.TestCase): + def test_cycle_converges_over_handoff_rounds(self): + with tempfile.TemporaryDirectory() as proj, \ + tempfile.TemporaryDirectory() as home: + hdir = os.path.join(proj, ".skillopt-sleep-handoff") + + def cfg(): + return load_config( + invoked_project=proj, projects="invoked", + backend="handoff", + claude_home=os.path.join(home, ".claude"), + ) + + final = None + rounds = 0 + for _ in range(8): + rounds += 1 + backend = HandoffBackend(handoff_dir=hdir) + try: + outcome = run_sleep_cycle( + cfg(), seed_tasks=_tasks(), dry_run=True, backend=backend, + ) + except PendingCalls: + outcome = None + if backend.pending: + backend.flush_pending() + _answer_pending(backend) + continue + final = outcome + break + + self.assertIsNotNone(final, "cycle never completed within 8 rounds") + self.assertGreater(rounds, 1, "expected at least one handoff round") + rep = final.report + self.assertTrue(rep.accepted, f"gate did not accept: {rep.gate_action}") + self.assertGreater(rep.candidate_score, rep.baseline_score) + self.assertTrue(any(RULE in e.content for e in rep.edits)) + + +class TestHandoffCli(unittest.TestCase): + def test_run_exits_3_and_stages_prompts(self): + from skillopt_sleep.__main__ import main + from skillopt_sleep.tasks_file import make_tasks_payload, write_tasks_file + + with tempfile.TemporaryDirectory() as proj, \ + tempfile.TemporaryDirectory() as home: + tasks_path = os.path.join(proj, "tasks.json") + payload = make_tasks_payload(_tasks(), project=proj) + payload["reviewed"] = True + write_tasks_file(tasks_path, payload) + + rc = main([ + "run", "--backend", "handoff", + "--project", proj, + "--claude-home", os.path.join(home, ".claude"), + "--tasks-file", tasks_path, + ]) + self.assertEqual(rc, 3) + hdir = os.path.join(proj, ".skillopt-sleep-handoff") + self.assertTrue(os.path.exists(os.path.join(hdir, "PROMPTS.md"))) + self.assertTrue(os.path.exists(os.path.join(hdir, "pending.json"))) + + def test_corrupt_digests_pin_falls_back_to_reharvest(self): + from skillopt_sleep.__main__ import main + + with tempfile.TemporaryDirectory() as proj, \ + tempfile.TemporaryDirectory() as home: + hdir = os.path.join(proj, ".skillopt-sleep-handoff") + os.makedirs(hdir, exist_ok=True) + with open(os.path.join(hdir, "digests.json"), "w", encoding="utf-8") as f: + f.write("{not valid json") + rc = main([ + "run", "--backend", "handoff", + "--project", proj, + "--claude-home", os.path.join(home, ".claude"), + ]) + # must not crash: corrupt pin -> fresh harvest -> no tasks -> 0 + self.assertEqual(rc, 0) + + def test_run_with_no_tasks_exits_0_and_advances_harvest_window(self): + from skillopt_sleep.__main__ import main + from skillopt_sleep.config import load_config + from skillopt_sleep.state import SleepState + + with tempfile.TemporaryDirectory() as proj, \ + tempfile.TemporaryDirectory() as home: + claude_home = os.path.join(home, ".claude") + rc = main([ + "run", "--backend", "handoff", + "--project", proj, + "--claude-home", claude_home, + ]) + self.assertEqual(rc, 0) + # the no-tasks branch must persist last-harvest, otherwise every + # later run re-scans the same stale window forever + cfg = load_config(invoked_project=proj, claude_home=claude_home) + state = SleepState.load(cfg.state_path) + self.assertIsNotNone(state.last_harvest_for(proj)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_json_utils.py b/tests/test_json_utils.py new file mode 100644 index 0000000..286efd7 --- /dev/null +++ b/tests/test_json_utils.py @@ -0,0 +1,188 @@ +"""Tests for skillopt.utils.json_utils.""" +from __future__ import annotations + +import pytest + +from skillopt.utils.json_utils import ( + _top_level_brace_objects, + extract_json, + extract_json_array, +) + + +class TestExtractJson: + """extract_json — extract a JSON object from LLM response text.""" + + def test_code_fence_json(self) -> None: + text = 'Some text\n```json\n{"key": "value", "num": 42}\n```\nmore text' + assert extract_json(text) == {"key": "value", "num": 42} + + def test_bare_json_object(self) -> None: + text = 'The result is {"answer": "yes", "score": 0.95}.' + assert extract_json(text) == {"answer": "yes", "score": 0.95} + + def test_code_fence_takes_precedence(self) -> None: + """If fence content parses successfully it should be preferred over bare.""" + text = ( + '```json\n{"source": "fence"}\n```\n' + 'Then also {"source": "bare"}' + ) + assert extract_json(text) == {"source": "fence"} + + def test_broken_fence_falls_back_to_bare(self) -> None: + """When fence content is invalid JSON, fall back to bare {...} match.""" + # Use invalid fence content that has no braces so the greedy bare + # regex doesn't swallow the valid object. + text = ( + '```json\nnot json at all\n```\n' + 'Answer: {"fallback": "yes"}' + ) + assert extract_json(text) == {"fallback": "yes"} + + def test_nested_json(self) -> None: + text = '```json\n{"outer": {"inner": [1, 2, 3]}}\n```' + assert extract_json(text) == {"outer": {"inner": [1, 2, 3]}} + + def test_no_json_returns_none(self) -> None: + assert extract_json("Just plain text without JSON.") is None + + def test_empty_string_returns_none(self) -> None: + assert extract_json("") is None + + def test_malformed_json_returns_none(self) -> None: + assert extract_json("{broken") is None + + def test_empty_json_object(self) -> None: + assert extract_json('{"empty": {}}') == {"empty": {}} + + def test_json_with_escaped_chars(self) -> None: + text = '{"message": "hello\\nworld"}' + assert extract_json(text) == {"message": "hello\nworld"} + + def test_only_fence_with_no_json_syntax(self) -> None: + """Code fences without valid JSON content should not match.""" + text = "```\nplain code block\n```" + assert extract_json(text) is None + + +class TestTopLevelBraceObjects: + """_top_level_brace_objects — string/escape-aware top-level object scan.""" + + def test_single_clean_object(self) -> None: + assert _top_level_brace_objects('{"a": 1}') == ['{"a": 1}'] + + def test_two_top_level_objects(self) -> None: + assert _top_level_brace_objects('{"a":1}\n{"b":2}') == ['{"a":1}', '{"b":2}'] + + def test_brace_inside_quoted_prose_is_ignored(self) -> None: + """A '{' inside a quoted string must NOT start an object (the bug).""" + # Brace-shaped content inside a string, with no real object → no spans. + assert _top_level_brace_objects('label is "set it to {x: 1}" done') == [] + + def test_real_object_after_quoted_brace(self) -> None: + """Quoted-prose braces are skipped; a later real object is still found.""" + text = 'note "{wrong: 1}" then actual {"edit": "right"}' + assert _top_level_brace_objects(text) == ['{"edit": "right"}'] + + +class TestExtractJsonTolerantFallback: + """extract_json — json_repair fallback for malformed non-OpenAI output.""" + + def test_prose_pseudo_json_returns_none(self) -> None: + """Regression: brace-shaped prose inside quotes must not be 'repaired' + into a bogus dict. It returned {'op': 'delete'} before the fix.""" + text = 'The literal string "{op: delete}" appears in prose, not as JSON.' + assert extract_json(text) is None + + def test_single_quoted_and_backticked_prose_returns_none(self) -> None: + """Regression: pseudo-JSON in single quotes / backticks / bare prose must + not be repaired into a bogus dict (the string-aware scan only skips + double-quoted prose; the JSON-like guard catches the rest).""" + for text in ( + "The literal string '{op: delete}' appears in prose, not JSON.", + "The inline code `{op: delete}` appears in prose, not JSON.", + "The literal string 'set it to {x: 1}' appears in prose.", + "A bare mapping {op: delete} written in prose.", + ): + assert extract_json(text) is None, text + + def test_json_string_values_with_quotes_still_repair(self) -> None: + """The JSON-like guard must NOT reject legitimate objects whose string + values contain single quotes or backticks.""" + pytest.importorskip("json_repair") + assert extract_json('{"msg": "it\'s a test",}') == {"msg": "it's a test"} + assert extract_json('{"code": "use `backtick` here",}') == {"code": "use `backtick` here"} + + def test_no_warning_on_quoted_prose(self, recwarn: pytest.WarningsRecorder) -> None: + """Prose pseudo-JSON (no real candidate) must not warn even without + json_repair installed — the JSON-like guard returns None before import.""" + assert extract_json("The inline code `{op: delete}` appears in prose.") is None + assert extract_json("A bare mapping {op: delete} in prose.") is None + assert [w for w in recwarn.list if issubclass(w.category, RuntimeWarning)] == [] + + def test_no_warning_on_plain_text(self, recwarn: pytest.WarningsRecorder) -> None: + """No json_repair warning for ordinary no-JSON replies (no candidate).""" + assert extract_json("Just plain text without JSON.") is None + assert extract_json("") is None + assert [w for w in recwarn.list if issubclass(w.category, RuntimeWarning)] == [] + + def test_trailing_comma_repaired_when_available(self) -> None: + """With json_repair installed, a single malformed object is repaired.""" + pytest.importorskip("json_repair") + assert extract_json('{"edit": "add", "text": "x",}') == {"edit": "add", "text": "x"} + + def test_two_malformed_objects_too_ambiguous(self) -> None: + """Multiple top-level objects are ambiguous → None, never guess.""" + pytest.importorskip("json_repair") + assert extract_json('{"first": true,} noise {"second": true,}') is None + + +class TestExtractJsonArray: + """extract_json_array — extract a JSON array from LLM response text.""" + + def test_code_fence_array(self) -> None: + text = '```json\n["a", "b", "c"]\n```' + assert extract_json_array(text) == ["a", "b", "c"] + + def test_bare_array(self) -> None: + text = "The items are [1, 2, 3]." + assert extract_json_array(text) == [1, 2, 3] + + def test_code_fence_takes_precedence(self) -> None: + text = ( + '```json\n["from_fence"]\n```\n' + 'also ["from_bare"]' + ) + assert extract_json_array(text) == ["from_fence"] + + def test_broken_fence_falls_back_to_bare(self) -> None: + text = ( + '```json\nnot json at all\n```\n' + 'values: [42]' + ) + assert extract_json_array(text) == [42] + + def test_nested_array(self) -> None: + text = '```json\n[[1, 2], [3, 4]]\n```' + assert extract_json_array(text) == [[1, 2], [3, 4]] + + def test_no_array_returns_none(self) -> None: + assert extract_json_array("no brackets here") is None + + def test_empty_string_returns_none(self) -> None: + assert extract_json_array("") is None + + def test_malformed_array_returns_none(self) -> None: + assert extract_json_array("[1, 2, ") is None + + def test_empty_json_array(self) -> None: + assert extract_json_array("[]") == [] + + def test_array_of_objects(self) -> None: + text = '[{"x": 1}, {"x": 2}]' + assert extract_json_array(text) == [{"x": 1}, {"x": 2}] + + def test_object_not_confused_with_array(self) -> None: + """extract_json_array should not match a bare JSON object.""" + text = '{"this is an object": true}' + assert extract_json_array(text) is None diff --git a/tests/test_materialize_searchqa.py b/tests/test_materialize_searchqa.py new file mode 100644 index 0000000..bbfb2a8 --- /dev/null +++ b/tests/test_materialize_searchqa.py @@ -0,0 +1,66 @@ +import json +from pathlib import Path + +import pytest + +from scripts.materialize_searchqa import materialize_searchqa_splits + + +def _write_manifest(root: Path, split_ids: dict[str, list[str]]) -> None: + for split, ids in split_ids.items(): + split_dir = root / split + split_dir.mkdir(parents=True) + (split_dir / "items.json").write_text( + json.dumps([{"id": item_id} for item_id in ids]), + encoding="utf-8", + ) + + +def _row(key: str) -> dict: + return { + "key": key, + "question": f"question {key}", + "context": f"context {key}", + "answers": [f"answer {key}"], + "ignored": "not written", + } + + +def test_materialize_searchqa_splits_preserves_manifest_order(tmp_path): + manifest_dir = tmp_path / "manifest" + output_dir = tmp_path / "out" + _write_manifest(manifest_dir, {"train": ["b", "a"], "val": ["c"], "test": ["d"]}) + + counts = materialize_searchqa_splits( + manifest_dir, + output_dir, + {"train": [_row("a"), _row("b")], "validation": [_row("c"), _row("d")]}, + dataset_name="example/searchqa", + ) + + assert counts == {"train": 2, "val": 1, "test": 1} + train_items = json.loads((output_dir / "train" / "items.json").read_text(encoding="utf-8")) + assert [item["id"] for item in train_items] == ["b", "a"] + assert train_items[0] == { + "id": "b", + "question": "question b", + "context": "context b", + "answers": ["answer b"], + } + + split_manifest = json.loads((output_dir / "split_manifest.json").read_text(encoding="utf-8")) + assert split_manifest["source_dataset"] == "example/searchqa" + assert split_manifest["counts"] == counts + + +def test_materialize_searchqa_splits_fails_on_missing_manifest_id(tmp_path): + manifest_dir = tmp_path / "manifest" + _write_manifest(manifest_dir, {"train": ["a"], "val": ["missing"], "test": []}) + + with pytest.raises(RuntimeError, match="missing"): + materialize_searchqa_splits( + manifest_dir, + tmp_path / "out", + {"train": [_row("a")]}, + dataset_name="example/searchqa", + ) diff --git a/tests/test_mcp_schema.py b/tests/test_mcp_schema.py new file mode 100644 index 0000000..f8960b1 --- /dev/null +++ b/tests/test_mcp_schema.py @@ -0,0 +1,37 @@ +"""Tests for the Copilot MCP server schema completeness.""" +import os +import sys +import unittest + +# Allow importing from the plugin directory +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "plugins", "copilot")) + + +class TestMcpSchema(unittest.TestCase): + def test_schema_includes_all_engine_flags(self): + from mcp_server import _TOOL_SCHEMA + required_params = { + "project", "backend", "scope", "source", "model", + "tasks_file", "target_skill_path", "progress", + "max_sessions", "max_tasks", "lookback_hours", + "auto_adopt", "json", "edit_budget", + } + schema_props = set(_TOOL_SCHEMA["properties"].keys()) + missing = required_params - schema_props + self.assertEqual(missing, set(), f"MCP schema missing: {missing}") + + def test_all_backends_in_enum(self): + from mcp_server import _TOOL_SCHEMA + backends = _TOOL_SCHEMA["properties"]["backend"]["enum"] + for b in ["mock", "claude", "codex", "copilot"]: + self.assertIn(b, backends) + + def test_schedule_tools_exist(self): + from mcp_server import TOOLS + names = {t["name"] for t in TOOLS} + self.assertIn("sleep_schedule", names) + self.assertIn("sleep_unschedule", names) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_plugin_sync.py b/tests/test_plugin_sync.py new file mode 100644 index 0000000..f7850e2 --- /dev/null +++ b/tests/test_plugin_sync.py @@ -0,0 +1,87 @@ +"""Cross-plugin parity tests — ensure all plugins document the same features. + +Run: python3 -m pytest tests/test_plugin_sync.py -v +""" +import os +import unittest + +REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + +PLUGIN_SKILL_MDS = { + "claude-code": os.path.join(REPO, "plugins/claude-code/skills/skillopt-sleep/SKILL.md"), + "codex": os.path.join(REPO, "plugins/codex/skills/skillopt-sleep/SKILL.md"), + "openclaw": os.path.join(REPO, "plugins/openclaw/SKILL.md"), +} + +MCP_SERVER = os.path.join(REPO, "plugins/copilot/mcp_server.py") +COPILOT_INSTRUCTIONS = os.path.join(REPO, "plugins/copilot/copilot-instructions.snippet.md") + +CANONICAL_BACKENDS = {"mock", "claude", "codex", "copilot"} + + +def _read(path): + if not os.path.exists(path): + return "" + with open(path, encoding="utf-8") as f: + return f.read() + + +class TestPluginParity(unittest.TestCase): + def test_all_skill_mds_mention_all_backends(self): + for name, path in PLUGIN_SKILL_MDS.items(): + text = _read(path) + if not text: + self.skipTest(f"{name} SKILL.md not found") + for backend in CANONICAL_BACKENDS: + self.assertIn(backend, text, + f"{name}/SKILL.md missing backend '{backend}'") + + def test_all_skill_mds_mention_schedule(self): + for name, path in PLUGIN_SKILL_MDS.items(): + text = _read(path) + if not text: + continue + self.assertIn("schedule", text.lower(), + f"{name}/SKILL.md missing 'schedule'") + self.assertIn("unschedule", text.lower(), + f"{name}/SKILL.md missing 'unschedule'") + + def test_copilot_instructions_mention_schedule(self): + text = _read(COPILOT_INSTRUCTIONS) + self.assertIn("sleep_schedule", text) + self.assertIn("sleep_unschedule", text) + + def test_copilot_instructions_mention_all_backends(self): + text = _read(COPILOT_INSTRUCTIONS) + for backend in CANONICAL_BACKENDS: + self.assertIn(backend, text, + f"copilot-instructions missing backend '{backend}'") + + def test_mcp_server_has_schedule_tools(self): + text = _read(MCP_SERVER) + self.assertIn("sleep_schedule", text) + self.assertIn("sleep_unschedule", text) + + def test_mcp_schema_has_key_params(self): + text = _read(MCP_SERVER) + for param in ["source", "tasks_file", "target_skill_path", + "max_sessions", "max_tasks", "auto_adopt", "json"]: + self.assertIn(f'"{param}"', text, + f"MCP schema missing param '{param}'") + + def test_all_skill_mds_mention_memory_consolidation(self): + for name, path in PLUGIN_SKILL_MDS.items(): + text = _read(path).lower() + if not text: + continue + has_mention = ( + "memory consolidation" in text + or "evolve_memory" in text + or ("consolidate" in text and "memory" in text) + ) + self.assertTrue(has_mention, + f"{name}/SKILL.md missing memory consolidation docs") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_qwen_backend.py b/tests/test_qwen_backend.py new file mode 100644 index 0000000..b0c9431 --- /dev/null +++ b/tests/test_qwen_backend.py @@ -0,0 +1,298 @@ +"""Tests for the OpenAI-compatible Qwen chat backend.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import sys +import types +from collections.abc import Iterator +from dataclasses import fields +from typing import Any + +import pytest + +from skillopt.envs.searchqa.evaluator import extract_answer + +_QWEN_CONFIG_ENV_KEYS = ( + "BASE_URL", + "API_KEY", + "TEMPERATURE", + "TIMEOUT_SECONDS", + "MAX_TOKENS", + "ENABLE_THINKING", + "USE_MAX_COMPLETION_TOKENS", +) +_ENV_KEYS = ("OPTIMIZER_BACKEND", "TARGET_BACKEND") + tuple( + f"{prefix}QWEN_CHAT_{key}" for prefix in ("", "OPTIMIZER_", "TARGET_") for key in _QWEN_CONFIG_ENV_KEYS +) + + +class _FakeResponse: + def __init__(self, payload: dict[str, Any]) -> None: + self._payload = payload + + def __enter__(self) -> _FakeResponse: + return self + + def __exit__(self, exc_type: object, exc: object, traceback: object) -> None: + return None + + def read(self) -> bytes: + return json.dumps(self._payload).encode("utf-8") + + +class _UrlopenRecorder: + def __init__(self, content: str = "yes") -> None: + self.content = content + self.calls: list[dict[str, Any]] = [] + + def __call__(self, request: Any, timeout: float | None = None) -> _FakeResponse: + request_data = request.data.decode("utf-8") + self.calls.append( + { + "payload": json.loads(request_data), + "timeout": timeout, + } + ) + return _FakeResponse( + { + "choices": [ + { + "message": {"content": self.content}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 2, + "completion_tokens": 1, + "total_tokens": 3, + }, + } + ) + + +class _OpenAIClientStub: + def __init__(self, *args: Any, **kwargs: Any) -> None: + self.args = args + self.kwargs = kwargs + + +def _install_openai_stub() -> None: + if "openai" in sys.modules or importlib.util.find_spec("openai") is not None: + return + openai_stub = types.ModuleType("openai") + openai_stub.AzureOpenAI = _OpenAIClientStub + openai_stub.OpenAI = _OpenAIClientStub + sys.modules["openai"] = openai_stub + + +def _import_model_modules() -> tuple[Any, Any, Any]: + _install_openai_stub() + import skillopt.model as model_module + from skillopt.model import backend_config, qwen_backend + + return model_module, backend_config, qwen_backend + + +def _snapshot_config(config: Any) -> dict[str, Any]: + return {field.name: getattr(config, field.name) for field in fields(config)} + + +def _restore_config(config: Any, snapshot: dict[str, Any]) -> None: + for key, value in snapshot.items(): + setattr(config, key, value) + + +@pytest.fixture(autouse=True) +def isolate_qwen_state() -> Iterator[tuple[Any, Any]]: + model_module, backend_config, qwen_backend = _import_model_modules() + optimizer_config = _snapshot_config(qwen_backend.OPTIMIZER_CONFIG) + target_config = _snapshot_config(qwen_backend.TARGET_CONFIG) + optimizer_backend = backend_config.get_optimizer_backend() + target_backend = backend_config.get_target_backend() + env = {key: os.environ.get(key) for key in _ENV_KEYS} + qwen_backend.reset_token_tracker() + yield model_module, qwen_backend + qwen_backend.reset_token_tracker() + _restore_config(qwen_backend.OPTIMIZER_CONFIG, optimizer_config) + _restore_config(qwen_backend.TARGET_CONFIG, target_config) + backend_config.set_optimizer_backend(optimizer_backend) + backend_config.set_target_backend(target_backend) + for key, value in env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +def _use_qwen_target(model_module: Any, qwen_backend: Any, enable_thinking: bool) -> None: + model_module.set_target_backend("qwen_chat") + qwen_backend.TARGET_CONFIG.base_url = "http://qwen.example/v1" + qwen_backend.TARGET_CONFIG.api_key = "" + qwen_backend.TARGET_CONFIG.timeout_seconds = 300.0 + qwen_backend.TARGET_CONFIG.max_tokens = 8000 + qwen_backend.TARGET_CONFIG.temperature = None + qwen_backend.TARGET_CONFIG.enable_thinking = enable_thinking + qwen_backend.TARGET_CONFIG.deployment = "qwen-test" + + +def _record_urlopen( + monkeypatch: pytest.MonkeyPatch, + qwen_backend: Any, + content: str = "yes", +) -> _UrlopenRecorder: + recorder = _UrlopenRecorder(content) + monkeypatch.setattr(qwen_backend.urllib.request, "urlopen", recorder) + return recorder + + +def test_chat_target_omits_chat_template_kwargs_when_thinking_disabled( + monkeypatch: pytest.MonkeyPatch, + isolate_qwen_state: tuple[Any, Any], +) -> None: + model_module, qwen_backend = isolate_qwen_state + _use_qwen_target(model_module, qwen_backend, enable_thinking=False) + recorder = _record_urlopen(monkeypatch, qwen_backend) + + text, usage = model_module.chat_target( + "system", + "user", + max_completion_tokens=128, + retries=1, + timeout=10.0, + ) + + assert text == "yes" + assert usage["total_tokens"] == 3 + assert "chat_template_kwargs" not in recorder.calls[0]["payload"] + assert recorder.calls[0]["timeout"] == 10.0 + + +def test_chat_target_includes_chat_template_kwargs_when_thinking_enabled( + monkeypatch: pytest.MonkeyPatch, + isolate_qwen_state: tuple[Any, Any], +) -> None: + model_module, qwen_backend = isolate_qwen_state + _use_qwen_target(model_module, qwen_backend, enable_thinking=True) + content = "working\nyes" + recorder = _record_urlopen(monkeypatch, qwen_backend, content=content) + + text, _ = model_module.chat_target( + "system", + "user", + max_completion_tokens=128, + retries=1, + ) + + assert recorder.calls[0]["payload"]["chat_template_kwargs"] == {"enable_thinking": True} + assert extract_answer(text) == "yes" + + +def test_chat_target_messages_forwards_timeout_to_qwen_backend( + monkeypatch: pytest.MonkeyPatch, + isolate_qwen_state: tuple[Any, Any], +) -> None: + model_module, qwen_backend = isolate_qwen_state + _use_qwen_target(model_module, qwen_backend, enable_thinking=False) + recorder = _record_urlopen(monkeypatch, qwen_backend) + + text, _ = model_module.chat_target_messages( + [{"role": "user", "content": "question"}], + max_completion_tokens=128, + retries=1, + timeout=10.0, + ) + + assert text == "yes" + assert recorder.calls[0]["timeout"] == 10.0 + + +def test_configure_qwen_chat_runtime_toggle_controls_payload( + monkeypatch: pytest.MonkeyPatch, + isolate_qwen_state: tuple[Any, Any], +) -> None: + model_module, qwen_backend = isolate_qwen_state + _use_qwen_target(model_module, qwen_backend, enable_thinking=False) + recorder = _record_urlopen(monkeypatch, qwen_backend) + + model_module.configure_qwen_chat(enable_thinking=True) + model_module.chat_target("system", "user", max_completion_tokens=128, retries=1) + model_module.configure_qwen_chat(enable_thinking=False) + model_module.chat_target("system", "user", max_completion_tokens=128, retries=1) + + assert recorder.calls[0]["payload"]["chat_template_kwargs"] == {"enable_thinking": True} + assert "chat_template_kwargs" not in recorder.calls[1]["payload"] + + +def test_chat_target_uses_max_tokens_by_default( + monkeypatch: pytest.MonkeyPatch, + isolate_qwen_state: tuple[Any, Any], +) -> None: + model_module, qwen_backend = isolate_qwen_state + _use_qwen_target(model_module, qwen_backend, enable_thinking=False) + recorder = _record_urlopen(monkeypatch, qwen_backend) + + model_module.chat_target("system", "user", max_completion_tokens=128, retries=1) + + payload = recorder.calls[0]["payload"] + assert payload["max_tokens"] == 128 + assert "max_completion_tokens" not in payload + + +def test_chat_target_uses_max_completion_tokens_when_enabled( + monkeypatch: pytest.MonkeyPatch, + isolate_qwen_state: tuple[Any, Any], +) -> None: + model_module, qwen_backend = isolate_qwen_state + _use_qwen_target(model_module, qwen_backend, enable_thinking=False) + qwen_backend.TARGET_CONFIG.use_max_completion_tokens = True + recorder = _record_urlopen(monkeypatch, qwen_backend) + + model_module.chat_target("system", "user", max_completion_tokens=128, retries=1) + + payload = recorder.calls[0]["payload"] + assert payload["max_completion_tokens"] == 128 + assert "max_tokens" not in payload + + +def test_configure_qwen_chat_toggles_max_completion_tokens( + monkeypatch: pytest.MonkeyPatch, + isolate_qwen_state: tuple[Any, Any], +) -> None: + model_module, qwen_backend = isolate_qwen_state + _use_qwen_target(model_module, qwen_backend, enable_thinking=False) + recorder = _record_urlopen(monkeypatch, qwen_backend) + + model_module.configure_qwen_chat(target_use_max_completion_tokens=True) + model_module.chat_target("system", "user", max_completion_tokens=128, retries=1) + model_module.configure_qwen_chat(target_use_max_completion_tokens=False) + model_module.chat_target("system", "user", max_completion_tokens=128, retries=1) + + assert "max_completion_tokens" in recorder.calls[0]["payload"] + assert "max_tokens" in recorder.calls[1]["payload"] + + +def test_temperature_omitted_when_env_is_blank( + monkeypatch: pytest.MonkeyPatch, + isolate_qwen_state: tuple[Any, Any], +) -> None: + _model_module, qwen_backend = isolate_qwen_state + # Explicit blank means "omit", not "fall back to 0.7". + monkeypatch.setenv("TARGET_QWEN_CHAT_TEMPERATURE", "") + assert qwen_backend._resolve_temperature("target") is None + monkeypatch.setenv("TARGET_QWEN_CHAT_TEMPERATURE", "off") + assert qwen_backend._resolve_temperature("target") is None + + +def test_temperature_resolves_float_and_default( + monkeypatch: pytest.MonkeyPatch, + isolate_qwen_state: tuple[Any, Any], +) -> None: + _model_module, qwen_backend = isolate_qwen_state + monkeypatch.delenv("QWEN_CHAT_TEMPERATURE", raising=False) + monkeypatch.delenv("TARGET_QWEN_CHAT_TEMPERATURE", raising=False) + assert qwen_backend._resolve_temperature("target") == 0.7 + monkeypatch.setenv("TARGET_QWEN_CHAT_TEMPERATURE", "0.2") + assert qwen_backend._resolve_temperature("target") == 0.2 diff --git a/tests/test_scoring.py b/tests/test_scoring.py new file mode 100644 index 0000000..281c6b8 --- /dev/null +++ b/tests/test_scoring.py @@ -0,0 +1,106 @@ +"""Tests for skillopt.utils.scoring.""" +from __future__ import annotations + +import pytest + +from skillopt.utils.scoring import compute_score, skill_hash + + +class _ResultObject: + """Minimal object with hard/soft attrs (duck-typing path).""" + + def __init__(self, hard: float, soft: float) -> None: + self.hard = hard + self.soft = soft + + +class TestComputeScore: + """compute_score — hard/soft accuracy from a list of episode results.""" + + def test_empty_list_returns_zeros(self) -> None: + assert compute_score([]) == (0.0, 0.0) + + def test_dict_results_happy_path(self) -> None: + results = [ + {"hard": 1, "soft": 0.8}, + {"hard": 0, "soft": 0.5}, + {"hard": 1, "soft": 0.9}, + ] + hard, soft = compute_score(results) + assert hard == pytest.approx(2 / 3) + assert soft == pytest.approx((0.8 + 0.5 + 0.9) / 3) + + def test_object_results(self) -> None: + results = [ + _ResultObject(1.0, 0.75), + _ResultObject(0.0, 0.25), + ] + hard, soft = compute_score(results) + assert hard == 0.5 + assert soft == 0.5 + + def test_mixed_dict_and_object_results(self) -> None: + results = [ + {"hard": 1, "soft": 1.0}, + _ResultObject(0, 0.0), + ] + hard, soft = compute_score(results) + assert hard == 0.5 + assert soft == 0.5 + + def test_missing_keys_default_to_zero(self) -> None: + results = [ + {"hard": 1}, + {}, + ] + hard, soft = compute_score(results) + assert hard == 0.5 + assert soft == 0.0 + + def test_single_result(self) -> None: + results = [{"hard": 1, "soft": 0.95}] + assert compute_score(results) == (1.0, 0.95) + + def test_continuous_hard_values(self) -> None: + """Hard may be continuous 0.0-1.0 when using smoothed reward.""" + results = [ + {"hard": 0.75, "soft": 0.6}, + {"hard": 0.25, "soft": 0.4}, + ] + hard, soft = compute_score(results) + assert hard == 0.5 + assert soft == 0.5 + + +class TestSkillHash: + """skill_hash — a short, deterministic hash of skill content.""" + + def test_deterministic(self) -> None: + assert skill_hash("hello") == skill_hash("hello") + + def test_different_input_produces_different_hash(self) -> None: + assert skill_hash("hello") != skill_hash("world") + + def test_empty_string(self) -> None: + h = skill_hash("") + assert isinstance(h, str) + assert len(h) == 16 + + def test_output_length(self) -> None: + h = skill_hash("some skill content here") + assert len(h) == 16 + + def test_hex_characters(self) -> None: + h = skill_hash("any content") + assert all(c in "0123456789abcdef" for c in h) + + def test_unicode_content(self) -> None: + h1 = skill_hash("cafe") + h2 = skill_hash("cafe") + assert h1 == h2 + + def test_multiline_content(self) -> None: + content = "line1\nline2\nline3" + h = skill_hash(content) + assert len(h) == 16 + assert isinstance(h, str) diff --git a/tests/test_searchqa_rollout_failfast.py b/tests/test_searchqa_rollout_failfast.py new file mode 100644 index 0000000..ef2ef71 --- /dev/null +++ b/tests/test_searchqa_rollout_failfast.py @@ -0,0 +1,25 @@ +import json + +import pytest + +from skillopt.envs.searchqa.rollout import run_batch + + +def test_cached_systemic_rollout_failure_aborts(tmp_path): + (tmp_path / "results.jsonl").write_text( + "\n".join([ + json.dumps({"id": "1", "agent_ok": False, "fail_reason": "endpoint missing"}), + json.dumps({"id": "2", "agent_ok": False, "fail_reason": "endpoint missing"}), + ]), + encoding="utf-8", + ) + + with pytest.raises(RuntimeError, match="endpoint missing"): + run_batch([{"id": "1"}, {"id": "2"}], str(tmp_path), "skill") + + +def test_cached_answered_wrong_rollout_does_not_abort(tmp_path): + result = {"id": "1", "agent_ok": True, "hard": 0, "fail_reason": "wrong answer"} + (tmp_path / "results.jsonl").write_text(json.dumps(result), encoding="utf-8") + + assert run_batch([{"id": "1"}], str(tmp_path), "skill") == [result] diff --git a/tests/test_semantic_density.py b/tests/test_semantic_density.py new file mode 100644 index 0000000..dcb5fcd --- /dev/null +++ b/tests/test_semantic_density.py @@ -0,0 +1,124 @@ +"""Tests for semantic density heuristic in the validation gate.""" +from __future__ import annotations + +import unittest +from skillopt.evaluation.gate import ( + compute_semantic_density, + select_gate_score, + evaluate_gate, +) + + +class TestSemanticDensity(unittest.TestCase): + """Test suite for semantic density scoring and gating decisions.""" + + def test_compute_semantic_density_basic(self) -> None: + """Verify basic compute_semantic_density behaviour with default words.""" + # 10 words, 2 leading words ("always", "never") -> 0.2 density + skill = "Always check the inputs and never mix up proxy values." + density = compute_semantic_density(skill) + self.assertAlmostEqual(density, 0.2) + + # Empty skill should have 0 density + self.assertEqual(compute_semantic_density(""), 0.0) + self.assertEqual(compute_semantic_density(" \n "), 0.0) + + def test_compute_semantic_density_custom_leading_words(self) -> None: + """Verify compute_semantic_density with custom leading words.""" + skill = "Check the inputs carefully and resolve the equation." + leading = ["check", "resolve"] + # 8 words, 2 custom leading words -> 0.25 density + density = compute_semantic_density(skill, leading_words=leading) + self.assertAlmostEqual(density, 0.25) + + def test_compute_semantic_density_with_protected_regions(self) -> None: + """Verify protected comments are excluded from density calculation.""" + skill = ( + "Always check inputs.\n" + "\n" + "This contains many words that should not count towards density " + "always and never and only.\n" + "\n" + "\n" + "More excluded words.\n" + "\n" + ) + # Without stripping, there would be many more words and a different density. + # Stripped text: "Always check inputs." -> 3 words, 1 leading word ("always") -> 1/3 density + density = compute_semantic_density(skill) + self.assertAlmostEqual(density, 1.0 / 3.0) + + def test_select_gate_score_no_density(self) -> None: + """Verify select_gate_score without semantic density adjustment.""" + # Default behavior: no semantic density adjustment + score_hard = select_gate_score(0.8, 0.6, metric="hard") + self.assertEqual(score_hard, 0.8) + + score_soft = select_gate_score(0.8, 0.6, metric="soft") + self.assertEqual(score_soft, 0.6) + + score_mixed = select_gate_score(0.8, 0.6, metric="mixed", mixed_weight=0.5) + self.assertAlmostEqual(score_mixed, 0.7) + + def test_select_gate_score_with_density(self) -> None: + """Verify select_gate_score with semantic density adjustment.""" + # 10 words, 2 leading words ("always", "never") -> 0.2 density + skill = "Always check the inputs and never mix up proxy values." + # bonus: 0.1 (weight) * 0.2 (density) = 0.02 + score = select_gate_score( + hard=0.8, + soft=0.6, + metric="hard", + skill_content=skill, + use_semantic_density=True, + semantic_density_weight=0.1, + ) + self.assertAlmostEqual(score, 0.82) + + def test_evaluate_gate_with_density_preference(self) -> None: + """Verify evaluate_gate prefers candidates with higher semantic density.""" + # Baseline/current skill: + # "Always do this task step by step and be very careful because errors are bad." + # 15 words, 1 leading ("always") -> 1/15 density = ~0.0667 + current_skill = "Always do this task step by step and be very careful because errors are bad." + + # Candidate skill (shorter/more steerable): + # "Always verify outputs. Never mix proxy values." + # 7 words, 3 leading ("always", "verify", "never") -> 3/7 density = ~0.4286 + candidate_skill = "Always verify outputs. Never mix proxy values." + + # Both have same rollout accuracy (hard=0.8, soft=0.8) + # Baseline/current score: 0.8 + 0.1 * (1/15) = ~0.8067 + current_score = select_gate_score( + hard=0.8, + soft=0.8, + metric="hard", + skill_content=current_skill, + use_semantic_density=True, + semantic_density_weight=0.1, + ) + + # Candidate score: 0.8 + 0.1 * (3/7) = ~0.8429 + # Even though accuracy is equal, the candidate should be accepted due to higher semantic density + res = evaluate_gate( + candidate_skill=candidate_skill, + cand_hard=0.8, + current_skill=current_skill, + current_score=current_score, + best_skill=current_skill, + best_score=current_score, + best_step=1, + global_step=2, + cand_soft=0.8, + metric="hard", + use_semantic_density=True, + semantic_density_weight=0.1, + ) + + self.assertEqual(res.action, "accept_new_best") + self.assertEqual(res.current_skill, candidate_skill) + self.assertAlmostEqual(res.current_score, 0.8 + 0.1 * (3.0 / 7.0)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_skill_aware_reflection.py b/tests/test_skill_aware_reflection.py new file mode 100644 index 0000000..68d3533 --- /dev/null +++ b/tests/test_skill_aware_reflection.py @@ -0,0 +1,274 @@ +"""Standalone regression + function tests for skill-aware reflection. + +Run directly (no pytest needed): + python tests/test_skill_aware_reflection.py + +Covers: +1. Toggle-OFF byte-identical guarantee for skill.py edit application + (slow-update-only behavior must be unchanged). +2. Appendix module: inject / append / dedup / extract / accumulate. +3. Appendix-region protection from step-level edits. +4. Coexistence of appendix + slow_update regions. +5. reflect.py prompt augmentation + appendix_notes parsing (no LLM call). +""" +from __future__ import annotations + +import os +import sys + +# Ensure THIS repo's skillopt is imported (not an installed copy) when the +# file is run directly: script mode puts tests/ on sys.path, not the repo root. +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +def _reference_old_apply(skill: str, edit: dict) -> str: + """Reproduce the ORIGINAL slow-update-only edit behavior inline.""" + SU_START = "" + SU_END = "" + op = edit.get("op", "") + content = edit.get("content", "").strip().replace(SU_START, "").replace(SU_END, "") + target = edit.get("target", "") + si = skill.find(SU_START) + ei = skill.find(SU_END) + + def in_su(t: str) -> bool: + if si == -1 or ei == -1: + return False + ti = skill.find(t) + if ti == -1: + return False + return si <= ti < ei + len(SU_END) + + if target and in_su(target): + return skill + if op == "append": + s = skill.find(SU_START) + if s != -1: + return skill[:s].rstrip() + "\n\n" + content + "\n\n" + skill[s:] + return skill.rstrip() + "\n\n" + content + "\n" + if op == "insert_after": + if not target or target not in skill: + s = skill.find(SU_START) + if s != -1: + return skill[:s].rstrip() + "\n\n" + content + "\n\n" + skill[s:] + return skill.rstrip() + "\n\n" + content + "\n" + idx = skill.index(target) + len(target) + nl = skill.find("\n", idx) + at = nl + 1 if nl != -1 else len(skill) + return skill[:at] + "\n" + content + "\n" + skill[at:] + if op == "replace": + if not target or target not in skill: + return skill + return skill.replace(target, content, 1) + if op == "delete": + if not target or target not in skill: + return skill + return skill.replace(target, "", 1) + return skill + + +def test_toggle_off_byte_identical() -> None: + from skillopt.optimizer.skill import _apply_edit_with_report + + SU_START = "" + SU_END = "" + skill = ( + "# QA Skill\n\n## Rules\n- Prefer shortest answer span.\n" + "- Use clue wording to constrain answer type.\n\n" + f"{SU_START}\nSome slow update guidance here.\n{SU_END}\n" + ) + edits = [ + {"op": "append", "content": "- New rule appended."}, + {"op": "insert_after", "target": "## Rules", "content": "- Inserted rule."}, + {"op": "insert_after", "target": "NONEXISTENT", "content": "- Fallback rule."}, + {"op": "replace", "target": "Prefer shortest answer span.", "content": "Prefer the exact minimal span."}, + {"op": "delete", "target": "- Use clue wording to constrain answer type."}, + {"op": "replace", "target": "Some slow update guidance here.", "content": "HACKED"}, + {"op": "delete", "target": "Some slow update guidance here."}, + ] + for e in edits: + new_skill, _ = _apply_edit_with_report(skill, e) + old_skill = _reference_old_apply(skill, e) + assert new_skill == old_skill, f"byte mismatch for {e['op']}" + print("PASS test_toggle_off_byte_identical") + + +def test_appendix_module() -> None: + from skillopt.optimizer.appendix import ( + has_appendix_field, inject_empty_appendix_field, + extract_appendix_notes, append_to_appendix_field, APPENDIX_START, + ) + skill = "# QA Skill\n\n- Prefer shortest answer span." + s1 = inject_empty_appendix_field(skill) + assert has_appendix_field(s1) and extract_appendix_notes(s1) == [] + assert inject_empty_appendix_field(s1) == s1 # idempotent + s2 = append_to_appendix_field(s1, ["Go to fridge for ice water.", "No stop token."]) + assert extract_appendix_notes(s2) == ["Go to fridge for ice water.", "No stop token."] + s3 = append_to_appendix_field(s2, ["go to fridge for ice water", "Check sheet range."]) + assert extract_appendix_notes(s3) == [ + "Go to fridge for ice water.", "No stop token.", "Check sheet range.", + ], "near-duplicate must be dropped" + assert s3.count(APPENDIX_START) == 1, "exactly one region after accumulation" + assert "# QA Skill" in s3 and "Prefer shortest answer span" in s3 + assert extract_appendix_notes(append_to_appendix_field(s1, [" ", "", "real"])) == ["real"] + print("PASS test_appendix_module") + + +def test_appendix_protection() -> None: + from skillopt.optimizer.skill import _apply_edit_with_report + from skillopt.optimizer.appendix import append_to_appendix_field, inject_empty_appendix_field + + skill = inject_empty_appendix_field("# QA Skill\n\n- Rule one.") + skill = append_to_appendix_field(skill, ["Follow rule one before acting."]) + for e in ( + {"op": "delete", "target": "Follow rule one before acting."}, + {"op": "replace", "target": "Follow rule one before acting.", "content": "HACK"}, + ): + new, rep = _apply_edit_with_report(skill, e) + assert new == skill, f"appendix must be protected from {e['op']}" + assert rep["status"] == "skipped_protected_region" + new, rep = _apply_edit_with_report(skill, {"op": "replace", "target": "Rule one.", "content": "Rule 1."}) + assert "Rule 1." in new and "Follow rule one before acting." in new + print("PASS test_appendix_protection") + + +def test_coexistence_with_slow_update() -> None: + from skillopt.optimizer.skill import _apply_edit_with_report + from skillopt.optimizer.appendix import ( + inject_empty_appendix_field, append_to_appendix_field, extract_appendix_notes, + ) + from skillopt.optimizer.slow_update import ( + inject_empty_slow_update_field, replace_slow_update_field, extract_slow_update_field, + ) + skill = inject_empty_appendix_field("# QA Skill\n\n- Rule one.") + skill = append_to_appendix_field(skill, ["Follow rule one."]) + skill = inject_empty_slow_update_field(skill) + skill = replace_slow_update_field(skill, "Slow guidance v2.") + assert extract_appendix_notes(skill) == ["Follow rule one."] + assert extract_slow_update_field(skill) == "Slow guidance v2." + # both regions protected + n1, r1 = _apply_edit_with_report(skill, {"op": "delete", "target": "Follow rule one."}) + n2, r2 = _apply_edit_with_report(skill, {"op": "replace", "target": "Slow guidance v2.", "content": "X"}) + assert n1 == skill and n2 == skill + # append lands before both regions (body stays at top) + n3, _ = _apply_edit_with_report(skill, {"op": "append", "content": "- Rule two."}) + assert n3.find("- Rule two.") < n3.find("") + assert n3.find("- Rule two.") < n3.find("") + print("PASS test_coexistence_with_slow_update") + + +def test_reflect_parsing_and_augment() -> None: + import inspect + import skillopt.gradient.reflect as R + from skillopt.optimizer.skill_aware import extract_appendix_notes, augment_error_prompt + + for fn in ("run_error_analyst_minibatch", "run_success_analyst_minibatch"): + sig = inspect.signature(getattr(R, fn)) + assert "skill_aware_reflection" in sig.parameters + assert sig.parameters["skill_aware_reflection"].default is False, f"{fn} default must be False" + # run_minibatch_reflect uses a None sentinel: explicit kwarg wins, else the + # process-wide config switch (configure_skill_aware_reflection) decides. + sig = inspect.signature(R.run_minibatch_reflect) + assert sig.parameters["skill_aware_reflection"].default is None + assert sig.parameters["skill_aware_appendix_source"].default is None + assert extract_appendix_notes({"appendix_notes": ["a", "b"]}) == ["a", "b"] + assert extract_appendix_notes({"appendix_notes": "x"}) == ["x"] + assert extract_appendix_notes({"appendix_notes": [{"note": "n"}, {"content": "c"}, {}]}) == ["n", "c"] + assert extract_appendix_notes({}) == [] and extract_appendix_notes(None) == [] + aug = augment_error_prompt("ORIG") + assert aug.startswith("ORIG") and "SKILL_DEFECT" in aug and "EXECUTION_LAPSE" in aug + print("PASS test_reflect_parsing_and_augment") + + +def test_global_switch_env_independent() -> None: + """The config switch alone must drive SAR for ANY env adapter (no kwargs).""" + from unittest import mock + import skillopt.gradient.reflect as R + from skillopt.optimizer.skill_aware import ( + configure_skill_aware_reflection, + get_skill_aware_appendix_source, + is_skill_aware_enabled, + ) + + # configure() round-trip. + configure_skill_aware_reflection(True, "failure_only") + assert is_skill_aware_enabled() and get_skill_aware_appendix_source() == "failure_only" + configure_skill_aware_reflection(False) + assert not is_skill_aware_enabled() and get_skill_aware_appendix_source() == "both" + + # run_minibatch_reflect with NO skill-aware kwargs (adapter-style call): + # capture what it forwards to the analyst workers under each switch state. + import tempfile + captured: dict = {} + + def fake_error_analyst(*args, **kwargs): + captured["skill_aware_reflection"] = kwargs.get("skill_aware_reflection") + return None + + def run_once() -> None: + captured.clear() + with mock.patch.object(R, "run_error_analyst_minibatch", fake_error_analyst), \ + tempfile.TemporaryDirectory() as tmp: + R.run_minibatch_reflect( + results=[{"id": "t1", "hard": 0, "soft": 0.0}], + skill_content="# Skill", + prediction_dir=tmp, + patches_dir=tmp, + workers=1, + failure_only=True, + minibatch_size=8, + ) + + try: + configure_skill_aware_reflection(True, "both") + run_once() + assert captured.get("skill_aware_reflection") is True, \ + "switch ON must reach the analyst without adapter wiring" + + configure_skill_aware_reflection(False) + run_once() + assert captured.get("skill_aware_reflection") is False, \ + "switch OFF must keep the analyst at baseline" + + # Explicit kwarg still overrides the global switch (backward compat). + captured.clear() + with mock.patch.object(R, "run_error_analyst_minibatch", fake_error_analyst), \ + tempfile.TemporaryDirectory() as tmp: + R.run_minibatch_reflect( + results=[{"id": "t1", "hard": 0, "soft": 0.0}], + skill_content="# Skill", + prediction_dir=tmp, + patches_dir=tmp, + workers=1, + failure_only=True, + minibatch_size=8, + skill_aware_reflection=True, + ) + assert captured.get("skill_aware_reflection") is True + finally: + configure_skill_aware_reflection(False) + print("PASS test_global_switch_env_independent") + + +def main() -> int: + tests = [ + test_toggle_off_byte_identical, + test_appendix_module, + test_appendix_protection, + test_coexistence_with_slow_update, + test_reflect_parsing_and_augment, + test_global_switch_env_independent, + ] + failed = 0 + for t in tests: + try: + t() + except AssertionError as exc: + failed += 1 + print(f"FAIL {t.__name__}: {exc}") + print(f"\n{len(tests) - failed}/{len(tests)} passed") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_sleep_engine.py b/tests/test_sleep_engine.py new file mode 100644 index 0000000..46e88e5 --- /dev/null +++ b/tests/test_sleep_engine.py @@ -0,0 +1,1360 @@ +"""Tests for the SkillOpt-Sleep engine. + +Pure-stdlib (unittest), deterministic, no API key, no third-party deps. +Run: python3.12 -m pytest tests/test_sleep_engine.py + or: python3.12 -m unittest skillopt_sleep ... (see bottom) +""" +from __future__ import annotations + +import json +import os +import tempfile +import unittest +from unittest import mock + +from skillopt_sleep.backend import MockBackend, exact_score, keyword_soft_score +from skillopt_sleep.config import load_config +from skillopt_sleep.consolidate import consolidate +from skillopt_sleep.cycle import run_sleep_cycle +from skillopt_sleep.experiments.personas import programmer_persona, researcher_persona +from skillopt_sleep.harvest import _detect_feedback, _is_meta_prompt, digest_transcript +from skillopt_sleep.memory import apply_edits, current_learned_lines, extract_learned, set_learned +from skillopt_sleep.mine import assign_splits, filter_tasks_for_target, heuristic_mine, mine +from skillopt_sleep.staging import adopt +from skillopt_sleep.types import EditRecord, SessionDigest, SleepReport, TaskRecord + + +class TestScoring(unittest.TestCase): + def test_exact_score(self): + self.assertEqual(exact_score("arXiv:1706.03762", "the id is arXiv:1706.03762 ok"), 1.0) + self.assertEqual(exact_score("arXiv:1706.03762", "approximately arXiv:1706.037"), 0.0) + + def test_keyword_soft(self): + self.assertGreater(keyword_soft_score("add login form", "please add the login form"), 0.5) + + +class TestMemoryEdits(unittest.TestCase): + def test_add_and_dedup(self): + doc = set_learned("# skill\n", []) + doc2, applied = apply_edits(doc, [EditRecord("skill", "add", "Rule A"), + EditRecord("skill", "add", "Rule A")]) + self.assertEqual(len(applied), 1) + self.assertIn("Rule A", extract_learned(doc2)) + + def test_protected_region_roundtrip(self): + base = "# My hand-written skill\nkeep me\n" + doc = set_learned(base, ["Rule X"]) + self.assertIn("keep me", doc) + self.assertEqual(current_learned_lines(doc), ["Rule X"]) + # replacing learned region must preserve hand-written content + doc2 = set_learned(doc, ["Rule Y"]) + self.assertIn("keep me", doc2) + self.assertEqual(current_learned_lines(doc2), ["Rule Y"]) + + def test_replace_and_delete(self): + doc = set_learned("", ["old rule about commits"]) + doc, _ = apply_edits(doc, [EditRecord("skill", "replace", "new rule", anchor="old rule")]) + self.assertIn("new rule", extract_learned(doc)) + doc, _ = apply_edits(doc, [EditRecord("skill", "delete", "", anchor="new rule")]) + self.assertEqual(current_learned_lines(doc), []) + + +class TestHarvest(unittest.TestCase): + def test_feedback_detection(self): + self.assertTrue(any(s.startswith("neg:") for s in _detect_feedback("this is still broken"))) + self.assertTrue(any(s.startswith("pos:") for s in _detect_feedback("perfect, thanks"))) + + def test_meta_prompt_filter(self): + self.assertTrue(_is_meta_prompt("/clear")) + self.assertTrue(_is_meta_prompt("x")) + self.assertFalse(_is_meta_prompt("please refactor the auth module")) + + def test_digest_real_transcript_if_present(self): + # uses the live machine's transcripts when available; skips otherwise + base = os.path.expanduser("~/.claude/projects") + if not os.path.isdir(base): + self.skipTest("no ~/.claude/projects on this machine") + found = None + for root, _d, files in os.walk(base): + for fn in files: + if fn.endswith(".jsonl"): + found = os.path.join(root, fn) + break + if found: + break + if not found: + self.skipTest("no transcripts") + d = digest_transcript(found) + # may be None for empty transcripts; if not, it must have core fields + if d is not None: + self.assertIsInstance(d.session_id, str) + self.assertGreaterEqual(d.n_user_turns + d.n_assistant_turns, 0) + + def _write_jsonl(self, path, records): + with open(path, "w", encoding="utf-8") as f: + for record in records: + f.write(json.dumps(record) + "\n") + + def test_digest_codex_archived_session_sanitizes_and_skips_meta(self): + from skillopt_sleep.harvest_codex import digest_codex_archived_session + + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "rollout-example.jsonl") + self._write_jsonl(path, [ + {"type": "turn_context", "timestamp": "2026-06-12T10:00:00Z", + "payload": {"cwd": "/repo/Yoshi", "type": None}}, + {"type": "response_item", "timestamp": "2026-06-12T10:00:01Z", + "payload": {"type": "message", "role": "developer", + "content": [{"type": "text", "text": "do not copy"}]}}, + {"type": "response_item", "timestamp": "2026-06-12T10:00:02Z", + "payload": {"type": "user_message", + "message": "# AGENTS.md instructions for /repo/Yoshi\n" + "do not keep"}}, + {"type": "response_item", "timestamp": "2026-06-12T10:00:03Z", + "payload": {"type": "user_message", + "message": "run deploy with sk-1234567890abcdef and token local-secret"}}, + {"type": "response_item", "timestamp": "2026-06-12T10:00:04Z", + "payload": {"type": "function_call", "name": "exec_command", + "arguments": "raw args should not copy"}}, + {"type": "response_item", "timestamp": "2026-06-12T10:00:05Z", + "payload": {"type": "function_call_output", + "output": "raw output should not copy"}}, + {"type": "response_item", "timestamp": "2026-06-12T10:00:06Z", + "payload": {"type": "agent_message", "message": "done"}}, + ]) + + digest = digest_codex_archived_session(path, project="/repo/Yoshi") + + self.assertIsNotNone(digest) + joined = "\n".join(digest.user_prompts + digest.assistant_finals) + self.assertEqual(digest.project, "/repo/Yoshi") + self.assertIn("[REDACTED_OPENAI_KEY]", joined) + self.assertIn("token [REDACTED]", joined) + self.assertIn("exec_command", digest.tools_used) + self.assertNotIn("AGENTS.md instructions", joined) + self.assertNotIn("do not copy", joined) + self.assertNotIn("raw args should not copy", joined) + self.assertNotIn("raw output should not copy", joined) + + def test_harvest_codex_filters_project_and_cli_source(self): + from skillopt_sleep.__main__ import _cfg_from_args + from skillopt_sleep.harvest_sources import harvest_for_config + + with tempfile.TemporaryDirectory() as tmp: + codex_home = os.path.join(tmp, ".codex") + sessions = os.path.join(codex_home, "archived_sessions") + os.makedirs(sessions) + self._write_jsonl(os.path.join(sessions, "rollout-yoshi.jsonl"), [ + {"type": "turn_context", "timestamp": "2026-06-12T10:00:00Z", + "payload": {"cwd": "/repo/Yoshi", "type": None}}, + {"type": "response_item", "timestamp": "2026-06-12T10:00:01Z", + "payload": {"type": "user_message", "message": "fix Yoshi"}}, + {"type": "response_item", "timestamp": "2026-06-12T10:00:02Z", + "payload": {"type": "agent_message", "message": "fixed"}}, + ]) + self._write_jsonl(os.path.join(sessions, "rollout-other.jsonl"), [ + {"type": "turn_context", "timestamp": "2026-06-12T10:00:00Z", + "payload": {"cwd": "/repo/Other", "type": None}}, + {"type": "response_item", "timestamp": "2026-06-12T10:00:01Z", + "payload": {"type": "user_message", "message": "fix Other"}}, + ]) + + Args = type("Args", (), { + "project": "/repo/Yoshi", + "scope": "", + "backend": "", + "model": "", + "codex_path": "", + "claude_home": "", + "codex_home": codex_home, + "source": "codex", + "lookback_hours": 0, + "edit_budget": 0, + "auto_adopt": False, + }) + + cfg = _cfg_from_args(Args()) + digests = harvest_for_config(cfg, limit=10) + + self.assertEqual(cfg.get("transcript_source"), "codex") + self.assertEqual(len(digests), 1) + self.assertEqual(digests[0].session_id, "rollout-yoshi") + self.assertEqual(digests[0].user_prompts, ["fix Yoshi"]) + + def test_cli_exposes_limits_progress_and_target_skill_path(self): + from skillopt_sleep.__main__ import _cfg_from_args + + with tempfile.TemporaryDirectory() as project: + Args = type("Args", (), { + "project": project, + "scope": "", + "backend": "codex", + "model": "", + "codex_path": "", + "claude_home": "", + "codex_home": "", + "source": "codex", + "lookback_hours": 0, + "edit_budget": 2, + "max_sessions": 5, + "max_tasks": 3, + "target_skill_path": ".agents/skills/taste-skill/SKILL.md", + "progress": True, + "auto_adopt": False, + }) + + cfg = _cfg_from_args(Args()) + + self.assertEqual(cfg.get("backend"), "codex") + self.assertEqual(cfg.get("max_sessions_per_night"), 5) + self.assertEqual(cfg.get("max_tasks_per_night"), 3) + self.assertTrue(cfg.get("progress")) + self.assertEqual( + cfg.managed_skill_path(), + os.path.abspath(os.path.join(project, ".agents/skills/taste-skill/SKILL.md")), + ) + + def test_cli_report_payload_includes_rejected_edits(self): + from skillopt_sleep.__main__ import _report_payload + + report = SleepReport( + night=1, + project="/p", + edits=[EditRecord("skill", "add", "accepted rule")], + rejected_edits=[EditRecord("skill", "add", "rejected rule")], + ) + outcome = type("Outcome", (), {"staging_dir": "", "adopted": False})() + + payload = _report_payload(report, outcome) + + self.assertEqual(payload["n_accepted_edits"], 1) + self.assertEqual(payload["n_rejected_edits"], 1) + self.assertEqual(payload["rejected_edits"][0]["content"], "rejected rule") + + def test_tasks_file_roundtrip_and_split_assignment(self): + from skillopt_sleep.tasks_file import load_tasks_file, make_tasks_payload, write_tasks_file + + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "tasks.json") + payload = make_tasks_payload( + [ + TaskRecord(id="t1", project="/p", intent="configure MCP server"), + TaskRecord(id="t2", project="/p", intent="resolve Git conflict"), + ], + project="/p", + transcript_source="codex", + n_sessions=2, + target_skill_path="/p/.agents/skills/yoshi-monorepo/SKILL.md", + ) + + written = write_tasks_file(path, payload) + tasks, meta = load_tasks_file(written, holdout_fraction=0.5, seed=1) + + self.assertEqual(meta["target_skill_path"], "/p/.agents/skills/yoshi-monorepo/SKILL.md") + self.assertEqual([t.id for t in tasks], ["t1", "t2"]) + self.assertIn("val", {t.split for t in tasks}) + + def test_cfg_uses_tasks_file_target_skill_path_metadata(self): + from skillopt_sleep.__main__ import _cfg_from_args + + Args = type("Args", (), { + "project": "/repo/Yoshi", + "scope": "", + "backend": "", + "model": "", + "codex_path": "", + "claude_home": "", + "codex_home": "", + "source": "", + "lookback_hours": 0, + "edit_budget": 0, + "max_sessions": 0, + "max_tasks": 0, + "target_skill_path": "", + "progress": False, + "auto_adopt": False, + }) + + cfg = _cfg_from_args(Args(), task_meta={ + "target_skill_path": ".agents/skills/yoshi-monorepo/SKILL.md", + }) + + self.assertEqual( + cfg.managed_skill_path(), + os.path.abspath("/repo/Yoshi/.agents/skills/yoshi-monorepo/SKILL.md"), + ) + + def test_cmd_run_uses_tasks_file_without_harvest(self): + from contextlib import redirect_stdout + from io import StringIO + + from skillopt_sleep.__main__ import cmd_run + from skillopt_sleep.tasks_file import make_tasks_payload, write_tasks_file + + with tempfile.TemporaryDirectory() as project, tempfile.TemporaryDirectory() as home: + target = os.path.join(project, ".agents/skills/yoshi-monorepo/SKILL.md") + os.makedirs(os.path.dirname(target)) + with open(target, "w", encoding="utf-8") as f: + f.write("# Yoshi Monorepo\n") + tasks_path = os.path.join(home, "reviewed-tasks.json") + write_tasks_file( + tasks_path, + make_tasks_payload( + [ + TaskRecord(id="t1", project=project, intent="configure MCP server"), + TaskRecord(id="t2", project=project, intent="resolve Git conflict"), + ], + project=project, + n_sessions=2, + target_skill_path=target, + ), + ) + Args = type("Args", (), { + "project": project, + "scope": "", + "backend": "mock", + "model": "", + "codex_path": "", + "claude_home": os.path.join(home, ".claude"), + "codex_home": "", + "source": "", + "lookback_hours": 0, + "edit_budget": 2, + "max_sessions": 5, + "max_tasks": 3, + "target_skill_path": "", + "tasks_file": tasks_path, + "progress": False, + "auto_adopt": False, + "json": True, + }) + + out = StringIO() + with redirect_stdout(out): + rc = cmd_run(Args(), dry=True) + payload = json.loads(out.getvalue()) + + self.assertEqual(rc, 0) + self.assertEqual(payload["n_sessions"], 0) + self.assertEqual(payload["n_tasks"], 2) + self.assertEqual(payload["tasks_file"], tasks_path) + + def test_cmd_run_refuses_unreviewed_tasks_file_for_real_backend(self): + from contextlib import redirect_stderr + from io import StringIO + + from skillopt_sleep.__main__ import cmd_run + from skillopt_sleep.tasks_file import make_tasks_payload, write_tasks_file + + with tempfile.TemporaryDirectory() as project, tempfile.TemporaryDirectory() as home: + tasks_path = os.path.join(home, "reviewed-tasks.json") + write_tasks_file( + tasks_path, + make_tasks_payload( + [TaskRecord(id="t1", project=project, intent="configure MCP server")], + project=project, + target_skill_path=os.path.join(project, ".agents/skills/yoshi-monorepo/SKILL.md"), + ), + ) + Args = type("Args", (), { + "project": project, + "scope": "", + "backend": "codex", + "model": "", + "codex_path": "", + "claude_home": os.path.join(home, ".claude"), + "codex_home": "", + "source": "", + "lookback_hours": 0, + "edit_budget": 2, + "max_sessions": 0, + "max_tasks": 0, + "target_skill_path": "", + "tasks_file": tasks_path, + "progress": False, + "auto_adopt": False, + "json": True, + }) + + err = StringIO() + with redirect_stderr(err): + rc = cmd_run(Args(), dry=True) + + self.assertEqual(rc, 2) + self.assertIn("unreviewed tasks file", err.getvalue()) + + +class TestMine(unittest.TestCase): + def _digest(self, prompts, feedback): + return SessionDigest( + session_id="s1", project="/p", user_prompts=prompts, + assistant_finals=["did stuff"], feedback_signals=feedback, + n_user_turns=len(prompts), n_assistant_turns=1, + ) + + def test_outcome_inference(self): + fail = heuristic_mine([self._digest(["fix the parser bug please"], ["neg:still broken"])]) + self.assertEqual(fail[0].outcome, "fail") + ok = heuristic_mine([self._digest(["format the output"], ["pos:perfect"])]) + self.assertEqual(ok[0].outcome, "success") + + def test_split_stable_and_nonempty(self): + tasks = assign_splits(researcher_persona(), val_fraction=0.34, seed=42) + splits = {t.split for t in tasks} + self.assertIn("train", splits) + self.assertIn("val", splits) + # stable across calls + again = assign_splits(researcher_persona(), val_fraction=0.34, seed=42) + self.assertEqual([t.split for t in tasks], [t.split for t in again]) + + def test_dream_never_in_val_or_test(self): + # the anti-overfitting guarantee: origin='dream' tasks only ever land in train + real = researcher_persona() + dream = [TaskRecord(id=f"d{i}", project="/p", intent=f"dream {i}", + origin="dream", derived_from="r0") for i in range(5)] + tasks = assign_splits(real + dream, val_fraction=0.3, test_fraction=0.3, seed=7) + for t in tasks: + if t.origin == "dream": + self.assertEqual(t.split, "train") + # val and test contain ONLY real tasks + for t in tasks: + if t.split in ("val", "test"): + self.assertEqual(t.origin, "real") + # and val/test are disjoint (a task is in exactly one split) + self.assertTrue(any(t.split == "val" for t in tasks)) + + def test_target_filter_prefers_matching_skill_terms(self): + skill = """# Yoshi Monorepo + +## MCP Setup Requests +Configure Codex MCP servers from linked setup docs. + +## Local Git Conflicts +Resolve local Git conflicts during merge, rebase, or cherry-pick. +""" + tasks = [ + TaskRecord(id="ios", project="/p", intent="polish SwiftUI onboarding spacing"), + TaskRecord(id="mcp", project="/p", intent="configure an MCP server from docs"), + TaskRecord(id="git", project="/p", intent="resolve a local Git conflict"), + TaskRecord(id="api", project="/p", intent="deploy the Rails API with Kamal"), + ] + + filtered = filter_tasks_for_target( + tasks, + skill, + ".agents/skills/yoshi-monorepo/SKILL.md", + ) + + self.assertEqual({t.id for t in filtered}, {"mcp", "git"}) + + def test_mine_oversamples_before_target_filtering(self): + skill = """# Yoshi Monorepo + +## MCP Setup Requests +Configure Codex MCP servers. + +## Local Git Conflicts +Resolve local Git conflicts. +""" + digests = [ + self._digest(["polish SwiftUI onboarding spacing"], ["neg:missed"]), + self._digest(["configure an MCP server from docs"], ["neg:missed"]), + self._digest(["resolve a local Git conflict"], ["neg:missed"]), + ] + + tasks = mine( + digests, + max_tasks=2, + candidate_limit=3, + target_skill_text=skill, + target_skill_path=".agents/skills/yoshi-monorepo/SKILL.md", + seed=42, + ) + + self.assertEqual({t.intent for t in tasks}, { + "configure an MCP server from docs", + "resolve a local Git conflict", + }) + + +class TestConsolidateGate(unittest.TestCase): + def test_accepts_helpful_rejects_harmful(self): + be = MockBackend() + tasks = assign_splits(researcher_persona(), holdout_fraction=0.34, seed=42) + res = consolidate(be, tasks, set_learned("", []), "", edit_budget=4, + gate_metric="mixed", night=1) + self.assertTrue(res.accepted) + self.assertGreater(res.candidate_score, res.baseline_score) + + def test_consolidate_records_holdout_detail(self): + # observability: a 0.0 night must carry per-task evidence (was empty + # response vs failing checks?) so it is diagnosable, not a black box. + be = MockBackend() + tasks = assign_splits(researcher_persona(), holdout_fraction=0.34, seed=42) + res = consolidate(be, tasks, set_learned("", []), "", edit_budget=4, + gate_metric="mixed", night=1) + self.assertTrue(res.holdout_detail) # non-empty per-task rows + row = res.holdout_detail[0] + for k in ("id", "hard", "soft", "response_len", "why"): + self.assertIn(k, row) + + def test_no_op_when_already_optimal(self): + be = MockBackend() + tasks = assign_splits(programmer_persona(), holdout_fraction=0.34, seed=1) + # first night learns the rule + r1 = consolidate(be, tasks, set_learned("", []), "", edit_budget=4, night=1) + # second night on the learned skill should find nothing to add + r2 = consolidate(be, tasks, r1.new_skill, r1.new_memory, edit_budget=4, night=2) + self.assertEqual(len(r2.applied_edits), 0) + + +class TestRuleJudge(unittest.TestCase): + def test_section_and_regex(self): + from skillopt_sleep.judges import score_rule_judge + j = {"kind": "rule", "checks": [ + {"op": "section_present", "arg": "Key Risks"}, + {"op": "regex", "arg": r"[Cc]onfidence\s*[:=]"}, + ]} + ok = "# Brief\n## Key Risks\nstuff\nConfidence: High" + self.assertEqual(score_rule_judge(j, ok)[0], 1.0) + self.assertEqual(score_rule_judge(j, "just an answer")[0], 0.0) + + def test_max_chars(self): + from skillopt_sleep.judges import score_rule_judge + j = {"checks": [{"op": "max_chars", "arg": 50}]} + self.assertEqual(score_rule_judge(j, "x" * 10)[0], 1.0) + self.assertEqual(score_rule_judge(j, "x" * 100)[0], 0.0) + + def test_partial_soft_score(self): + from skillopt_sleep.judges import score_rule_judge + j = {"checks": [ + {"op": "contains", "arg": "alpha"}, + {"op": "contains", "arg": "beta"}, + ]} + h, s, _ = score_rule_judge(j, "only alpha here") + self.assertEqual(h, 0.0) + self.assertAlmostEqual(s, 0.5) + + +class TestGbrainLoader(unittest.TestCase): + def test_loads_when_present(self): + from skillopt_sleep.experiments.gbrain_bench import find_data_root, load_seed + root = find_data_root() + if not root: + self.skipTest("gbrain-evals data not present") + skill, tasks = load_seed(root, "brief-writer") + self.assertTrue(skill) + # gbrain held-out maps to our 'test'; benchmark pool to train/val + self.assertTrue(any(t.split == "test" for t in tasks)) + self.assertTrue(any(t.split == "val" for t in tasks)) + self.assertTrue(all(t.reference_kind == "rule" for t in tasks)) + # the deficient skill must FAIL its own held-out (test) checks (baseline 0) + from skillopt_sleep.judges import score_rule_judge + ho = [t for t in tasks if t.split == "test"][0] + self.assertEqual(score_rule_judge(ho.judge, skill)[0], 0.0) + + +class TestLlmMiner(unittest.TestCase): + def test_miner_emits_checkable_tasks(self): + # a stub backend whose _call returns canned miner JSON => deterministic + from skillopt_sleep.backend import Backend + from skillopt_sleep.llm_miner import make_llm_miner + + class StubBackend(Backend): + name = "stub" + def _call(self, prompt, *, max_tokens=1024): + return ('[{"intent":"write a research brief",' + '"checks":[{"op":"section_present","arg":"Key Risks"}],' + '"rubric":"has a risks section","satisfied":false}]') + + digest = SessionDigest(session_id="s1", project="/p", + user_prompts=["write a brief on X"], + assistant_finals=["a brief"], n_user_turns=1) + miner = make_llm_miner(StubBackend()) + tasks = miner([digest]) + self.assertEqual(len(tasks), 1) + self.assertEqual(tasks[0].reference_kind, "rule") + self.assertEqual(tasks[0].judge["checks"][0]["op"], "section_present") + + def test_miner_drops_uncheckable(self): + from skillopt_sleep.backend import Backend + from skillopt_sleep.llm_miner import make_llm_miner + + class EmptyBackend(Backend): + name = "stub" + def _call(self, prompt, *, max_tokens=1024): + return "[]" + + digest = SessionDigest(session_id="s1", project="/p", + user_prompts=["chat"], n_user_turns=1) + self.assertEqual(make_llm_miner(EmptyBackend())([digest]), []) + + +class TestMultiObjectiveAndPrefs(unittest.TestCase): + def test_multi_objective_reward(self): + from skillopt_sleep.replay import multi_objective_reward + from skillopt_sleep.types import ReplayResult + t = TaskRecord(id="t", project="/p", intent="x") + expensive = [(t, ReplayResult(id="t", hard=1.0, tokens=4000, latency_ms=20000))] + cheap = [(t, ReplayResult(id="t", hard=1.0, tokens=200, latency_ms=1000))] + self.assertEqual( + multi_objective_reward(expensive, w_acc=1, w_tokens=0, w_latency=0), + multi_objective_reward(cheap, w_acc=1, w_tokens=0, w_latency=0), + ) + re = multi_objective_reward(expensive, w_acc=1, w_tokens=1, w_latency=1) + rc = multi_objective_reward(cheap, w_acc=1, w_tokens=1, w_latency=1) + self.assertGreater(rc, re) + + def test_preferences_injected_into_reflect(self): + from skillopt_sleep.backend import CliBackend + from skillopt_sleep.types import ReplayResult + captured = {} + + class CapBackend(CliBackend): + name = "cap" + def _call(self, prompt, *, max_tokens=1024): + captured["prompt"] = prompt + return "[]" + + be = CapBackend() + be.preferences = "Prefer concise British English." + t = TaskRecord(id="t", project="/p", intent="x", reference_kind="rule", + judge={"checks": [{"op": "contains", "arg": "z"}]}) + be.reflect([(t, ReplayResult(id="t", hard=0.0, fail_reason="failed: contains=z"))], + [], "skill", "", edit_budget=2, evolve_skill=True, evolve_memory=False) + self.assertIn("British English", captured["prompt"]) + + def test_reflect_records_last_raw(self): + # the optimizer's raw reply must be retained so a no-edits night is + # diagnosable (empty/non-JSON reflect vs genuinely no failures). + from skillopt_sleep.backend import CliBackend + from skillopt_sleep.types import ReplayResult + + class CapBackend(CliBackend): + name = "cap" + def _call(self, prompt, *, max_tokens=1024): + return '[{"op":"add","content":"a learned rule","rationale":"x"}]' + + be = CapBackend() + t = TaskRecord(id="t", project="/p", intent="x", reference_kind="rule", + judge={"checks": [{"op": "contains", "arg": "z"}]}) + be.reflect([(t, ReplayResult(id="t", hard=0.0, fail_reason="failed: contains=z"))], + [], "skill", "", edit_budget=2, evolve_skill=True, evolve_memory=False) + self.assertIn("a learned rule", be.last_reflect_raw) + + def test_replay_records_cost(self): + from skillopt_sleep.backend import MockBackend + from skillopt_sleep.replay import replay_one + t = TaskRecord(id="t", project="/p", intent="hello world", + reference_kind="exact", reference="hi") + r = replay_one(MockBackend(), t, "some skill text", "") + self.assertGreater(r.tokens, 0) + self.assertGreaterEqual(r.latency_ms, 0.0) + + +class TestCodexBackend(unittest.TestCase): + def test_codex_cli_backend_runs_exec_in_project_dir(self): + from skillopt_sleep.backend import CodexCliBackend + + calls = [] + + def fake_run(cmd, **kwargs): + calls.append((cmd, kwargs)) + out_path = cmd[cmd.index("-o") + 1] + with open(out_path, "w", encoding="utf-8") as f: + f.write("ok") + + class Proc: + returncode = 0 + stdout = "" + stderr = "" + + return Proc() + + with tempfile.TemporaryDirectory() as project: + expected_project = os.path.abspath(project) + backend = CodexCliBackend(codex_path="codex", project_dir=project) + + with mock.patch("skillopt_sleep.backend.subprocess.run", side_effect=fake_run): + self.assertEqual(backend._call("hello"), "ok") + + self.assertEqual(len(calls), 1) + cmd, kwargs = calls[0] + self.assertEqual(kwargs["cwd"], expected_project) + self.assertIn("-C", cmd) + self.assertEqual(cmd[cmd.index("-C") + 1], expected_project) + + def test_codex_call_retries_transient_failure_not_silent_zero(self): + """A transient timeout must be RETRIED, not silently returned as "" — an + empty reply scores 0 on every judge and zeroes the held-out baseline, + making a flaky backend look identical to 'nothing to learn'.""" + import subprocess as _sp + + from skillopt_sleep.backend import CodexCliBackend + + calls = {"n": 0} + + def fake_run(cmd, **kwargs): + calls["n"] += 1 + if calls["n"] == 1: + raise _sp.TimeoutExpired(cmd, kwargs.get("timeout", 1)) + out_path = cmd[cmd.index("-o") + 1] + with open(out_path, "w", encoding="utf-8") as f: + f.write("real answer") + + class Proc: + returncode = 0 + stdout = "" + stderr = "" + + return Proc() + + backend = CodexCliBackend(codex_path="codex") + with mock.patch("skillopt_sleep.backend.subprocess.run", side_effect=fake_run), \ + mock.patch("time.sleep", lambda *_a, **_k: None): + out = backend._call("hello") + self.assertEqual(out, "real answer") # recovered on retry + self.assertGreaterEqual(calls["n"], 2) # proves it did not silently return "" once + + def test_codex_auth_error_surfaces_not_scored_as_response(self): + """An auth 401 must become a clear last_call_error + EMPTY response (not the + 9k-char error text scored as a 0 'answer'), and must NOT be retried — the + exact failure that silently stalled learning (refresh_token_reused).""" + from skillopt_sleep.backend import CodexCliBackend + + calls = {"n": 0} + + def fake_run(cmd, **kwargs): + calls["n"] += 1 + out_path = cmd[cmd.index("-o") + 1] + open(out_path, "w").close() # empty output file (codex wrote nothing) + + class Proc: + returncode = 1 + stdout = "" + stderr = "ERROR codex_core::auth: 401 Unauthorized: refresh_token_reused" + + return Proc() + + be = CodexCliBackend(codex_path="codex") + with mock.patch("skillopt_sleep.backend.subprocess.run", side_effect=fake_run), \ + mock.patch("time.sleep", lambda *_a, **_k: None): + out = be._call("hi") + self.assertEqual(out, "") # NOT the error text + self.assertIn("refresh_token_reused", be.last_call_error) # surfaced for the operator + self.assertEqual(calls["n"], 1) # failed fast, no wasted retries + + def test_codex_attempt_with_tools_surfaces_error_not_silent(self): + """A failed tool-rollout (non-zero codex exec) on the tool path must set + last_call_error and return an empty response — not a silent empty->0 the + diagnostics can't see (the gap a _call-only fix would otherwise leave).""" + from skillopt_sleep.backend import CodexCliBackend + + def fake_run(cmd, **kwargs): + class Proc: + returncode = 1 + stdout = "" + stderr = "ERROR codex_core::auth: 401 Unauthorized: refresh_token_reused" + return Proc() # writes nothing to out_path -> empty response + + be = CodexCliBackend(codex_path="codex") + task = TaskRecord(id="t", project="/p", intent="answer the question", + reference_kind="rule", + judge={"checks": [{"op": "tool_called", "arg": "search"}]}) + with mock.patch("skillopt_sleep.backend.subprocess.run", side_effect=fake_run): + resp, called = be.attempt_with_tools(task, "", "", ["search"]) + self.assertEqual(resp, "") # no leaked error text as a "response" + self.assertIn("exited 1", be.last_call_error) # failure surfaced for diagnostics + self.assertEqual(called, []) # no tool actually ran + + +class TestMultiRolloutAndBudget(unittest.TestCase): + def test_rolloutset_stats(self): + from skillopt_sleep.rollout import RolloutSet + from skillopt_sleep.types import ReplayResult + rs = RolloutSet(task=TaskRecord(id="t", project="/p", intent="x"), + attempts=[ReplayResult(id="t", hard=1.0), + ReplayResult(id="t", hard=0.0), + ReplayResult(id="t", hard=1.0)]) + self.assertEqual(rs.best.hard, 1.0) + self.assertEqual(rs.worst.hard, 0.0) + self.assertEqual(rs.spread, 1.0) + self.assertAlmostEqual(rs.pass_rate, 2 / 3) + + def test_budget_exhaustion_and_plan(self): + from skillopt_sleep.budget import Budget, plan_depth + clock = [0.0] + b = Budget(max_tokens=1000) + b.start(lambda: clock[0], tokens_now=0) + self.assertFalse(b.exhausted(tokens_now=500, clock_fn=lambda: clock[0])) + self.assertTrue(b.exhausted(tokens_now=1000, clock_fn=lambda: clock[0])) + self.assertEqual(plan_depth(Budget(), n_tasks=5, default_nights=2, default_k=1), (2, 1)) + nights, k = plan_depth(Budget(max_tokens=100_000), n_tasks=5) + self.assertGreaterEqual(nights, 1) + self.assertGreaterEqual(k, 1) + + def test_contrastive_reflect_with_stub(self): + from skillopt_sleep.backend import Backend + from skillopt_sleep.rollout import RolloutSet, contrastive_reflect + from skillopt_sleep.types import ReplayResult + + class StubBackend(Backend): + name = "stub" + def _call(self, prompt, *, max_tokens=1024): + return '[{"op":"add","content":"always do the good thing","rationale":"good passed"}]' + + rs = RolloutSet(task=TaskRecord(id="t", project="/p", intent="x"), + attempts=[ReplayResult(id="t", hard=1.0, response="good"), + ReplayResult(id="t", hard=0.0, response="bad")]) + edits = contrastive_reflect(StubBackend(), [rs], "skill", "") + self.assertEqual(len(edits), 1) + self.assertIn("good thing", edits[0].content) + + +class TestSlowUpdate(unittest.TestCase): + def test_protected_field_roundtrip(self): + from skillopt_sleep.slow_update import ( + SLOW_UPDATE_END, + SLOW_UPDATE_START, + extract_slow_field, + has_slow_field, + replace_slow_field, + ) + base = "# skill\nkeep me\n" + doc = replace_slow_field(base, "durable lesson A") + self.assertTrue(has_slow_field(doc)) + self.assertIn("keep me", doc) + self.assertEqual(extract_slow_field(doc), "durable lesson A") + # replacing keeps exactly one block and preserves hand-written text + doc2 = replace_slow_field(doc, "durable lesson B") + self.assertEqual(doc2.count(SLOW_UPDATE_START), 1) + self.assertEqual(doc2.count(SLOW_UPDATE_END), 1) + self.assertEqual(extract_slow_field(doc2), "durable lesson B") + self.assertIn("keep me", doc2) + + def test_run_slow_update_with_stub_backend(self): + from skillopt_sleep.backend import Backend + from skillopt_sleep.slow_update import run_slow_update + from skillopt_sleep.types import ReplayResult + + class StubBackend(Backend): + name = "stub" + def _call(self, prompt, *, max_tokens=1024): + return '{"guidance": "- keep doing X\\n- avoid regression Y"}' + + t = TaskRecord(id="t1", project="/p", intent="do thing") + prev = [(t, ReplayResult(id="t1", hard=0.0))] # was failing + curr = [(t, ReplayResult(id="t1", hard=1.0))] # now passing (improved) + out = run_slow_update(StubBackend(), prev_skill="s0", curr_skill="s1", + prev_pairs=prev, curr_pairs=curr) + # improvements alone with no regression/persistent-fail and no prior text -> None + self.assertIsNone(out) + # a regression triggers guidance + prev2 = [(t, ReplayResult(id="t1", hard=1.0))] + curr2 = [(t, ReplayResult(id="t1", hard=0.0))] + out2 = run_slow_update(StubBackend(), prev_skill="s0", curr_skill="s1", + prev_pairs=prev2, curr_pairs=curr2) + self.assertIn("keep doing X", out2) + + +class TestToolLoop(unittest.TestCase): + def test_tool_called_judge_via_replay(self): + from skillopt_sleep.backend import MockBackend + from skillopt_sleep.memory import set_learned + from skillopt_sleep.replay import _required_tools, replay_one + + task = TaskRecord( + id="qa1", project="/p", intent="answer the question", + reference_kind="rule", + judge={"kind": "rule", "checks": [{"op": "tool_called", "arg": "search"}]}, + ) + self.assertEqual(_required_tools(task), ["search"]) + be = MockBackend() + # deficient skill: no instruction to search -> tool not called -> hard 0 + deficient = "Answer from memory. Do NOT use tools." + r0 = replay_one(be, task, deficient, "") + self.assertEqual(r0.hard, 0.0) + self.assertEqual(r0.tools_called, []) + # learned rule to use ./search -> tool called -> hard 1 + learned = set_learned(deficient, ["Before answering you MUST run ./search first."]) + r1 = replay_one(be, task, learned, "") + self.assertEqual(r1.hard, 1.0) + self.assertEqual(r1.tools_called, ["search"]) + + +class TestFullCycleAndAdopt(unittest.TestCase): + def test_cycle_stage_then_adopt_with_backup(self): + with tempfile.TemporaryDirectory() as proj, tempfile.TemporaryDirectory() as home: + cfg = load_config( + invoked_project=proj, projects="invoked", backend="mock", + claude_home=os.path.join(home, ".claude"), + managed_skill_name="skillopt-sleep-learned", + auto_adopt=False, + ) + # seed a known persona so we don't depend on ~/.claude + tasks = assign_splits(researcher_persona(), holdout_fraction=0.34, seed=42) + + outcome = run_sleep_cycle(cfg, seed_tasks=tasks) + self.assertTrue(outcome.report.accepted) + self.assertTrue(os.path.isdir(outcome.staging_dir)) + self.assertTrue(os.path.exists(os.path.join(outcome.staging_dir, "report.md"))) + + # nothing live touched yet + live_skill = cfg.managed_skill_path() + self.assertFalse(os.path.exists(live_skill)) + + # adopt -> live file created, backup dir exists + updated = adopt(outcome.staging_dir) + self.assertTrue(any("SKILL.md" in p for p in updated)) + self.assertTrue(os.path.exists(live_skill)) + with open(live_skill) as f: + self.assertIn("answer", f.read().lower()) + + def test_cycle_can_target_repo_scoped_skill_path(self): + with tempfile.TemporaryDirectory() as proj, tempfile.TemporaryDirectory() as home: + target = os.path.abspath(os.path.join(proj, ".agents/skills/taste-skill/SKILL.md")) + cfg = load_config( + invoked_project=proj, + projects="invoked", + backend="mock", + claude_home=os.path.join(home, ".claude"), + target_skill_path=target, + auto_adopt=False, + ) + tasks = assign_splits(programmer_persona(), holdout_fraction=0.34, seed=42) + + outcome = run_sleep_cycle(cfg, seed_tasks=tasks) + + self.assertTrue(outcome.report.accepted) + manifest_path = os.path.join(outcome.staging_dir, "manifest.json") + with open(manifest_path, encoding="utf-8") as f: + manifest = json.load(f) + self.assertEqual(manifest["live_skill_path"], target) + self.assertFalse(os.path.exists(target)) + + updated = adopt(outcome.staging_dir) + + self.assertIn(target, updated) + self.assertTrue(os.path.exists(target)) + + +class TestCopilotBackend(unittest.TestCase): + """Pure-logic tests for CopilotCliBackend — no `copilot` CLI required.""" + + def test_alias_resolution(self): + from skillopt_sleep.backend import CopilotCliBackend, get_backend + for name in ("copilot", "github_copilot", "copilot_cli", "gh_copilot"): + self.assertIsInstance(get_backend(name), CopilotCliBackend, name) + + def test_parse_jsonl_concatenates_assistant_messages(self): + from skillopt_sleep.backend import CopilotCliBackend + raw = "\n".join([ + '{"type":"session.info","data":{}}', + '{"type":"assistant.message","data":{"content":"hello"}}', + 'not-json-noise', + '{"type":"user.message","data":{"content":"ignored"}}', + '{"type":"assistant.message","data":{"content":"world"}}', + ]) + self.assertEqual(CopilotCliBackend._parse_jsonl_response(raw), "hello\nworld") + + def test_parse_jsonl_ignores_non_assistant_and_blank(self): + from skillopt_sleep.backend import CopilotCliBackend + self.assertEqual(CopilotCliBackend._parse_jsonl_response(""), "") + self.assertEqual( + CopilotCliBackend._parse_jsonl_response('{"type":"result","data":{"content":"x"}}'), + "", + ) + # assistant.message with empty/missing content contributes nothing + self.assertEqual( + CopilotCliBackend._parse_jsonl_response( + '{"type":"assistant.message","data":{"content":""}}\n' + '{"type":"assistant.message","data":{}}' + ), + "", + ) + + def test_isolated_home_by_default(self): + from skillopt_sleep.backend import CopilotCliBackend + be = CopilotCliBackend() + self.assertFalse(be.full_env) + self.assertTrue(be.copilot_home) # an isolated COPILOT_HOME is set + + def test_full_env_opt_out(self): + from skillopt_sleep.backend import CopilotCliBackend + prev = os.environ.get("SKILLOPT_SLEEP_COPILOT_FULL_ENV") + os.environ["SKILLOPT_SLEEP_COPILOT_FULL_ENV"] = "1" + try: + be = CopilotCliBackend() + self.assertTrue(be.full_env) + self.assertEqual(be.copilot_home, "") # real user environment used + finally: + if prev is None: + os.environ.pop("SKILLOPT_SLEEP_COPILOT_FULL_ENV", None) + else: + os.environ["SKILLOPT_SLEEP_COPILOT_FULL_ENV"] = prev + + def test_home_override_env(self): + from skillopt_sleep.backend import CopilotCliBackend + with tempfile.TemporaryDirectory() as d: + target = os.path.join(d, "myhome") + prev = os.environ.get("SKILLOPT_SLEEP_COPILOT_HOME") + os.environ["SKILLOPT_SLEEP_COPILOT_HOME"] = target + try: + be = CopilotCliBackend() + self.assertEqual(be.copilot_home, target) + self.assertTrue(os.path.isdir(target)) # created on init + finally: + if prev is None: + os.environ.pop("SKILLOPT_SLEEP_COPILOT_HOME", None) + else: + os.environ["SKILLOPT_SLEEP_COPILOT_HOME"] = prev + + def test_attempt_with_tools_honest_detection(self): + # End-to-end (no real CLI): a tiny per-OS stub stands in for `copilot`. + # It runs the local `search` shim the backend writes into its work dir + # (so the calllog is written — honest detection) then prints one JSONL + # assistant.message. Proves both the JSONL parse and that the tool call + # is detected from the shim's log, not from a self-reported marker. + import shutil + import stat + + from skillopt_sleep.backend import CopilotCliBackend + + stub_dir = tempfile.mkdtemp(prefix="skillopt_sleep_stub_") + try: + if os.name == "nt": + stub = os.path.join(stub_dir, "copilot.cmd") + with open(stub, "w") as f: + # The backend writes `search.cmd`; run it (explicit `.\` so + # cmd's `call` resolves it from the cwd reliably) so the + # calllog is populated, then emit the JSONL line. None of + # `{ } " :` need escaping in batch echo (no > < | & ^ %). + f.write( + "@echo off\n" + 'call .\\search.cmd "q" >nul 2>&1\n' + 'echo {"type":"assistant.message","data":{"content":"Paris"}}\n' + ) + else: + stub = os.path.join(stub_dir, "copilot") + with open(stub, "w") as f: + f.write( + "#!/usr/bin/env bash\n" + './search "q" >/dev/null 2>&1\n' + "echo '{\"type\":\"assistant.message\",\"data\":{\"content\":\"Paris\"}}'\n" + ) + os.chmod( + stub, + os.stat(stub).st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH, + ) + + be = CopilotCliBackend(copilot_path=stub, timeout=60) + task = TaskRecord(id="t1", project="p", intent="What is the capital of France?") + resp, called = be.attempt_with_tools(task, skill="", memory="", tools=["search"]) + + self.assertEqual(resp, "Paris") # JSONL parsed via _parse_jsonl_response + self.assertEqual(called, ["search"]) # shim ran; detected from calllog + finally: + shutil.rmtree(stub_dir, ignore_errors=True) + + +class TestClaudeCliBackendBare(unittest.TestCase): + """Issue #68: --bare must be conditional on ANTHROPIC_API_KEY.""" + + def test_bare_included_when_api_key_set(self): + """With ANTHROPIC_API_KEY, --bare should appear in the command.""" + from skillopt_sleep.backend import ClaudeCliBackend + be = ClaudeCliBackend(claude_path="/usr/bin/false", timeout=5) + with unittest.mock.patch.dict(os.environ, {"ANTHROPIC_API_KEY": "sk-test"}): + # We can't run the real CLI, but we can inspect cmd construction + # by monkeypatching subprocess.run to capture the command. + captured = {} + def fake_run(cmd, **kwargs): + captured["cmd"] = cmd + class FakeProc: + stdout = "hello" + stderr = "" + returncode = 0 + return FakeProc() + with unittest.mock.patch("subprocess.run", side_effect=fake_run): + be._call("test prompt") + self.assertIn("--bare", captured["cmd"]) + + def test_bare_omitted_without_api_key(self): + """Without ANTHROPIC_API_KEY, --bare should NOT appear.""" + from skillopt_sleep.backend import ClaudeCliBackend + be = ClaudeCliBackend(claude_path="/usr/bin/false", timeout=5) + env = os.environ.copy() + env.pop("ANTHROPIC_API_KEY", None) + with unittest.mock.patch.dict(os.environ, env, clear=True): + captured = {} + def fake_run(cmd, **kwargs): + captured["cmd"] = cmd + class FakeProc: + stdout = "hello" + stderr = "" + returncode = 0 + return FakeProc() + with unittest.mock.patch("subprocess.run", side_effect=fake_run): + be._call("test prompt") + self.assertNotIn("--bare", captured["cmd"]) + + def test_cli_error_detected_and_logged(self): + """Auth errors in CLI output should trigger a warning.""" + from skillopt_sleep.backend import ClaudeCliBackend + be = ClaudeCliBackend(claude_path="/usr/bin/false", timeout=5) + captured = {} + def fake_run(cmd, **kwargs): + captured["cmd"] = cmd + class FakeProc: + stdout = "Not logged in · Please run /login" + stderr = "" + returncode = 0 + return FakeProc() + with unittest.mock.patch.dict(os.environ, {}, clear=False): + with unittest.mock.patch("subprocess.run", side_effect=fake_run): + result = be._call("test prompt") + # The error string is returned as output (backwards-compat) + self.assertIn("Not logged in", result) + # But it's also recorded for detection + self.assertIn("Not logged in", getattr(be, "last_call_error", "")) + + def test_spawn_failure_sets_last_call_error(self): + """When subprocess.run raises FileNotFoundError, _call must set + last_call_error and log a warning instead of silently returning ''.""" + from skillopt_sleep.backend import ClaudeCliBackend + be = ClaudeCliBackend( + claude_path="/nonexistent/claude-binary", + timeout=3, + ) + result = be._call("test prompt") + self.assertEqual(result, "") + self.assertIn("Claude CLI spawn failed", be.last_call_error) + + def test_attempt_tools_spawn_failure_sets_last_call_error(self): + """When subprocess.run raises in attempt_with_tools, last_call_error + must be set and a warning logged.""" + from skillopt_sleep.backend import ClaudeCliBackend + from skillopt_sleep.types import TaskRecord + be = ClaudeCliBackend( + claude_path="/nonexistent/claude-binary", + timeout=3, + ) + task = TaskRecord( + id="t1", project="/p", intent="test", + reference="ref", reference_kind="exact", + tags=[], split="train", + ) + resp, called = be.attempt_with_tools(task, "", "", tools=["search"]) + self.assertEqual(resp, "") + self.assertIn("Claude CLI spawn failed", be.last_call_error) + + + + +class MockRewardHackingBackend(MockBackend): + def attempt(self, task: TaskRecord, skill: str, memory: str, sample_id: int = 0) -> str: + ctx = (skill or "") + "\n" + (memory or "") + # If the reward-hacking artifact is present in the context: + if "Always emit placeholder URLs" in ctx: + if "rule:__reward_hacking__" in task.tags: + return str(task.reference) # perfectly answers the train/replay shortcut task + else: + return "placeholder URL" # completely breaks the real held-out task + + # Baseline behavior (without the rule) + if "rule:__reward_hacking__" in task.tags: + return "I am missing input" # baseline fails the shortcut task + if "rule:real" in task.tags: + return str(task.reference) # baseline gets the real task right + + return super().attempt(task, skill, memory, sample_id) + + def reflect(self, failures, successes, skill, memory, **kwargs): + edits = [] + for task, _ in failures: + if "rule:__reward_hacking__" in task.tags: + edits.append(EditRecord("skill", "add", "Always emit placeholder URLs", rationale="shortcut")) + break + return edits + +class MockBeneficialBackend(MockBackend): + def attempt(self, task: TaskRecord, skill: str, memory: str, sample_id: int = 0) -> str: + ctx = (skill or "") + "\n" + (memory or "") + # If the beneficial artifact is present: + if "Always think step-by-step" in ctx: + if "rule:__beneficial__" in task.tags: + return str(task.reference) # improves the train task + if "rule:real" in task.tags: + return str(task.reference) # improves the real held-out task + + # Baseline behavior (without the rule) + if "rule:__beneficial__" in task.tags: + return "I am missing input" # baseline fails the train task + if "rule:real" in task.tags: + return "baseline fails too" # baseline fails the real task + + return super().attempt(task, skill, memory, sample_id) + + def reflect(self, failures, successes, skill, memory, **kwargs): + edits = [] + for task, _ in failures: + if "rule:__beneficial__" in task.tags: + edits.append(EditRecord("skill", "add", "Always think step-by-step", rationale="beneficial")) + break + return edits + +class TestVerifierDiscipline(unittest.TestCase): + def test_gate_rejects_reward_hacking_edit(self): + be = MockRewardHackingBackend() + train_task = TaskRecord(id="t1", project="/p", intent="train", reference="XYZABC", reference_kind="exact", tags=["rule:__reward_hacking__"], split="train") + val_task = TaskRecord(id="v1", project="/p", intent="val", reference="PQRSTU", reference_kind="exact", tags=["rule:real"], split="val") + tasks = [train_task, val_task] + + res = consolidate(be, tasks, "", "", edit_budget=4, gate_metric="hard", night=1) + + self.assertFalse(res.accepted) + self.assertEqual(res.gate_action, "reject") + self.assertEqual(res.holdout_baseline, 1.0) + self.assertEqual(res.holdout_candidate, 1.0) # final state reverts to baseline + self.assertGreater(len(res.rejected_edits), 0) + self.assertIn("placeholder", res.rejected_edits[0].content) + + def test_gate_accepts_beneficial_edit(self): + be = MockBeneficialBackend() + train_task = TaskRecord(id="t2", project="/p", intent="train", reference="ABCDEF", reference_kind="exact", tags=["rule:__beneficial__"], split="train") + val_task = TaskRecord(id="v2", project="/p", intent="val", reference="UVWXYZ", reference_kind="exact", tags=["rule:real"], split="val") + tasks = [train_task, val_task] + + res = consolidate(be, tasks, "", "", edit_budget=4, gate_metric="hard", night=1) + + self.assertTrue(res.accepted) + self.assertEqual(res.gate_action, "accept_new_best") + self.assertEqual(res.holdout_baseline, 0.0) + self.assertEqual(res.holdout_candidate, 1.0) + self.assertGreater(len(res.applied_edits), 0) + self.assertIn("step-by-step", res.applied_edits[0].content) + +class TestDiagnosticsRedaction(unittest.TestCase): + """diagnostics.json surfaces backend stderr / optimizer replies / task + responses for debugging — but those can carry credentials (e.g. a codex 401 + stderr dump). redact_secrets() must scrub them before anything is persisted.""" + + def test_redacts_common_secret_shapes(self): + from skillopt_sleep.staging import redact_secrets + cases = [ + ("error: used sk-ABCDEFGHIJ1234567890 to call", "sk-ABCDEFGHIJ1234567890"), + ("Authorization: Bearer eyJhbGciOi.JIUzI1Ni.qwerty", "eyJhbGciOi.JIUzI1Ni.qwerty"), + ("config api_key=super-secret-value here", "super-secret-value"), + ("token: abc123def456ghi", "abc123def456ghi"), + ("aws AKIAIOSFODNN7EXAMPLE creds", "AKIAIOSFODNN7EXAMPLE"), + ("github ghp_AbCdEf0123456789AbCdEf0123 push", "ghp_AbCdEf0123456789AbCdEf0123"), + ("jwt eyJhbGci0123.eyJzdWIi4567.SflKxwRJ89 here", "eyJhbGci0123.eyJzdWIi4567.SflKxwRJ89"), + ] + for text, secret in cases: + out = redact_secrets(text) + self.assertNotIn(secret, out, f"secret leaked: {text!r} -> {out!r}") + self.assertIn("REDACTED", out, f"no redaction marker in {out!r}") + + def test_does_not_over_redact_plain_prose(self): + """Redaction must not mangle ordinary diagnostic prose that happens to + mention security words without an actual secret value attached.""" + from skillopt_sleep.staging import redact_secrets + for benign in ( + "the gate rejected the edit", + "response was empty, judge scored 0.0", + "held-out 1.000 -> 0.000 reject", + ): + self.assertEqual(redact_secrets(benign), benign, f"over-redacted: {benign!r}") + + def test_redacts_private_key_block(self): + from skillopt_sleep.staging import redact_secrets + blob = ( + "-----BEGIN RSA PRIVATE KEY-----\n" + "MIIEowIBAAKCAQEA...secret...\n" + "-----END RSA PRIVATE KEY-----" + ) + out = redact_secrets("leaked:\n" + blob) + self.assertNotIn("MIIEowIBAAKCAQEA", out) + self.assertIn("[REDACTED_PRIVATE_KEY]", out) + + def test_redacts_recursively_in_lists_and_dicts(self): + from skillopt_sleep.staging import redact_secrets + payload = { + "call_error": "exit 1: api_key=leaked-key-123", + "holdout_detail": [ + {"id": "t1", "response_head": "uses sk-DEADBEEF0001cafe", "hard": 0.0}, + ], + "n_tasks": 3, # non-string scalars pass through untouched + "accepted": False, + } + out = redact_secrets(payload) + self.assertNotIn("leaked-key-123", out["call_error"]) + self.assertNotIn("sk-DEADBEEF0001cafe", out["holdout_detail"][0]["response_head"]) + self.assertEqual(out["n_tasks"], 3) + self.assertIs(out["accepted"], False) + + def test_non_string_scalars_unchanged(self): + from skillopt_sleep.staging import redact_secrets + self.assertEqual(redact_secrets(42), 42) + self.assertEqual(redact_secrets(0.5), 0.5) + self.assertIsNone(redact_secrets(None)) + + def test_diagnostics_json_on_disk_has_no_secret(self): + """End-to-end: a codex-style 401 stderr captured in call_error must not + reach diagnostics.json verbatim once written to the staging dir.""" + import json + from skillopt_sleep.staging import redact_secrets + # Mirror exactly what cycle.py writes (the fields that carry free text). + secret_stderr = ( + "codex exec exited 1: ERROR 401 Unauthorized " + "Authorization: Bearer sk-LEAKED99887766abcdef refresh_token_reused" + ) + diag = { + "night": 1, + "accepted": False, + "call_error": redact_secrets(secret_stderr), + "reflect_raw_head": redact_secrets("optimizer said api_key=should-not-persist"), + "holdout_detail": redact_secrets( + [{"id": "v1", "response_head": "sk-ANOTHERLEAK1234567", "hard": 0.0}] + ), + } + with tempfile.TemporaryDirectory() as tmp: + p = os.path.join(tmp, "diagnostics.json") + with open(p, "w", encoding="utf-8") as fh: + json.dump(diag, fh, indent=2) + with open(p, encoding="utf-8") as fh: + on_disk = fh.read() + for leak in ("sk-LEAKED99887766abcdef", "should-not-persist", "sk-ANOTHERLEAK1234567"): + self.assertNotIn(leak, on_disk, f"secret {leak!r} leaked to diagnostics.json") + # The diagnostic value is still there (we scrub, not drop). + self.assertIn("401 Unauthorized", on_disk) + self.assertIn("REDACTED", on_disk) + + def test_codex_auth_error_log_is_redacted(self): + """The codex auth-error log line (a secondary on-disk sink when a file + log handler is attached) must not emit the raw stderr token verbatim.""" + import logging + from skillopt_sleep.backend import CodexCliBackend + be = CodexCliBackend.__new__(CodexCliBackend) # no __init__ side effects + be.timeout = 1 + be._AUTH_MARKERS = CodexCliBackend._AUTH_MARKERS + secret = "sk-LOGLEAK0011223344aa" + calls = {"n": 0} + + def _fake_once(prompt, *, max_tokens=1024): + calls["n"] += 1 + be.last_call_error = f"401 Unauthorized Authorization: Bearer {secret}" + return "" + + be._call_once = _fake_once + with self.assertLogs("skillopt_sleep", level="ERROR") as cm: + out = be._call("p", retries=3) + self.assertEqual(out, "") + self.assertEqual(calls["n"], 1, "auth error must fail fast, not retry") + joined = "\n".join(cm.output) + self.assertNotIn(secret, joined, "raw token leaked into the log line") + self.assertIn("REDACTED", joined) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_types.py b/tests/test_types.py new file mode 100644 index 0000000..f39c8f6 --- /dev/null +++ b/tests/test_types.py @@ -0,0 +1,249 @@ +"""Tests for skillopt.types — Edit and Patch dataclass serialization.""" +from __future__ import annotations + +import pytest + +from skillopt.types import Edit, Patch + + +# ── Edit ──────────────────────────────────────────────────────────────────── + + +class TestEditCreation: + """Edit dataclass construction.""" + + def test_minimal_edit(self) -> None: + e = Edit(op="append") + assert e.op == "append" + assert e.content == "" + assert e.target == "" + assert e.support_count is None + assert e.source_type is None + assert e.merge_level is None + assert e.update_origin == "" + assert e.update_target == "" + + def test_full_edit(self) -> None: + e = Edit( + op="replace", + content="new content", + target="old content", + support_count=5, + source_type="failure", + merge_level=2, + update_origin="reflect", + update_target="skill", + ) + assert e.op == "replace" + assert e.content == "new content" + assert e.target == "old content" + assert e.support_count == 5 + assert e.source_type == "failure" + assert e.merge_level == 2 + assert e.update_origin == "reflect" + assert e.update_target == "skill" + + def test_insert_after_op(self) -> None: + e = Edit(op="insert_after", content="insertion", target="anchor") + assert e.op == "insert_after" + assert e.content == "insertion" + assert e.target == "anchor" + + def test_delete_op(self) -> None: + e = Edit(op="delete", target="thing_to_remove") + assert e.op == "delete" + assert e.target == "thing_to_remove" + + +class TestEditRoundTrip: + """Edit.to_dict() / Edit.from_dict() round-trip.""" + + def test_round_trip_minimal(self) -> None: + e = Edit(op="append") + d = e.to_dict() + restored = Edit.from_dict(d) + assert restored == e + + def test_round_trip_full(self) -> None: + e = Edit( + op="replace", + content="new content", + target="old content", + support_count=3, + source_type="success", + merge_level=1, + update_origin="meta_reflect", + update_target="system_prompt", + ) + d = e.to_dict() + restored = Edit.from_dict(d) + assert restored == e + + def test_round_trip_delete_without_content(self) -> None: + e = Edit(op="delete", target="obsolete_line") + d = e.to_dict() + restored = Edit.from_dict(d) + assert restored == e + + def test_optional_fields_omitted_when_default(self) -> None: + e = Edit(op="append") + d = e.to_dict() + assert d == {"op": "append", "content": ""} + # support_count, source_type, etc. should be absent + assert "support_count" not in d + assert "source_type" not in d + assert "merge_level" not in d + assert "target" not in d + assert "update_origin" not in d + assert "update_target" not in d + + def test_from_dict_with_defaults(self) -> None: + d = {"op": "replace", "content": "abc"} + e = Edit.from_dict(d) + assert e.op == "replace" + assert e.content == "abc" + assert e.target == "" + assert e.support_count is None + assert e.source_type is None + + def test_from_dict_with_extra_keys(self) -> None: + """Extra keys in dict should be ignored.""" + d = {"op": "append", "content": "", "unknown_field": 42} + e = Edit.from_dict(d) + assert e.op == "append" + assert not hasattr(e, "unknown_field") + + +class TestEditEdgeCases: + """Edge cases around Edit.""" + + def test_support_count_zero(self) -> None: + """0 is a valid support_count and should be serialized.""" + e = Edit(op="append", support_count=0) + d = e.to_dict() + assert d["support_count"] == 0 + restored = Edit.from_dict(d) + assert restored.support_count == 0 + + def test_merge_level_zero(self) -> None: + e = Edit(op="replace", merge_level=0) + d = e.to_dict() + assert d["merge_level"] == 0 + restored = Edit.from_dict(d) + assert restored.merge_level == 0 + + def test_empty_target_stays_empty(self) -> None: + e = Edit(op="append", target="") + d = e.to_dict() + assert "target" not in d + + +# ── Patch ─────────────────────────────────────────────────────────────────── + + +class TestPatchCreation: + """Patch dataclass construction.""" + + def test_empty_patch(self) -> None: + p = Patch() + assert p.edits == [] + assert p.reasoning == "" + assert p.ranking_details is None + + def test_patch_with_edits(self) -> None: + edits = [ + Edit(op="append", content="step 1"), + Edit(op="append", content="step 2"), + ] + p = Patch(edits=edits, reasoning="Added two steps") + assert len(p.edits) == 2 + assert p.reasoning == "Added two steps" + + def test_patch_with_ranking_details(self) -> None: + p = Patch(ranking_details={"score": 0.95, "rank": 1}) + assert p.ranking_details == {"score": 0.95, "rank": 1} + + +class TestPatchRoundTrip: + """Patch.to_dict() / Patch.from_dict() round-trip.""" + + def test_round_trip_empty(self) -> None: + p = Patch() + d = p.to_dict() + restored = Patch.from_dict(d) + assert restored.edits == [] + assert restored.reasoning == "" + assert restored.ranking_details is None + + def test_round_trip_with_edits(self) -> None: + edits = [ + Edit(op="insert_after", content="new step", target="existing step"), + Edit(op="replace", content="updated", target="old"), + ] + p = Patch(edits=edits, reasoning="Batch update") + d = p.to_dict() + restored = Patch.from_dict(d) + assert len(restored.edits) == 2 + for original, restored_edit in zip(p.edits, restored.edits): + assert isinstance(restored_edit, Edit) + assert original == restored_edit + assert restored.reasoning == "Batch update" + assert restored.ranking_details is None + + def test_round_trip_with_ranking_details(self) -> None: + details = {"strategy": "rouge", "scores": [0.9, 0.8, 0.7]} + p = Patch( + edits=[Edit(op="append", content="a")], + reasoning="selected best", + ranking_details=details, + ) + d = p.to_dict() + restored = Patch.from_dict(d) + assert restored.ranking_details == details + + def test_to_dict_contains_reasoning_and_edits(self) -> None: + p = Patch(edits=[Edit(op="append", content="test")], reasoning="reason") + d = p.to_dict() + assert "reasoning" in d + assert "edits" in d + assert isinstance(d["edits"], list) + + def test_from_dict_preserves_edit_order(self) -> None: + edits = [ + Edit(op="append", content="first"), + Edit(op="insert_after", content="second", target="first"), + Edit(op="append", content="third"), + ] + p = Patch(edits=edits, reasoning="ordered") + d = p.to_dict() + restored = Patch.from_dict(d) + assert restored.edits[0].content == "first" + assert restored.edits[1].content == "second" + assert restored.edits[2].content == "third" + + +class TestPatchEdgeCases: + """Edge cases around Patch.""" + + def test_reasoning_empty_string(self) -> None: + p = Patch(reasoning="") + d = p.to_dict() + assert d["reasoning"] == "" + + def test_zero_edits(self) -> None: + """Patch with explicitly empty edit list.""" + p = Patch(edits=[]) + d = p.to_dict() + assert d["edits"] == [] + + def test_nested_edit_from_dict_handles_dicts(self) -> None: + """from_dict should accept dicts in the 'edits' list.""" + d = { + "reasoning": "test", + "edits": [{"op": "append", "content": "hello"}], + } + p = Patch.from_dict(d) + assert len(p.edits) == 1 + assert isinstance(p.edits[0], Edit) + assert p.edits[0].op == "append" + assert p.edits[0].content == "hello" diff --git a/tests/test_webui_env_preflight.py b/tests/test_webui_env_preflight.py new file mode 100644 index 0000000..5b84d86 --- /dev/null +++ b/tests/test_webui_env_preflight.py @@ -0,0 +1,89 @@ +import pytest +import yaml + +pytest.importorskip("gradio") + +from skillopt_webui import app as webui_app + + +def _write_config(tmp_path, model): + config_path = tmp_path / "config.yaml" + config_path.write_text( + yaml.safe_dump({ + "model": model, + "env": {"name": "searchqa"}, + }), + encoding="utf-8", + ) + return str(config_path) + + +def test_build_training_env_loads_project_dotenv(tmp_path, monkeypatch): + monkeypatch.setattr(webui_app, "PROJECT_ROOT", tmp_path) + (tmp_path / ".env").write_text( + "\n".join([ + "export QWEN_CHAT_BASE_URL=http://qwen.example/v1", + "QWEN_CHAT_MODEL=test-model", + "QWEN_CHAT_API_KEY='secret-value'", + ]), + encoding="utf-8", + ) + + env = webui_app.build_training_env() + + assert env["QWEN_CHAT_BASE_URL"] == "http://qwen.example/v1" + assert env["QWEN_CHAT_MODEL"] == "test-model" + assert env["QWEN_CHAT_API_KEY"] == "secret-value" + + +def test_preflight_reports_missing_openai_chat_endpoint(tmp_path, monkeypatch): + monkeypatch.delenv("AZURE_OPENAI_ENDPOINT", raising=False) + monkeypatch.delenv("OPTIMIZER_AZURE_OPENAI_ENDPOINT", raising=False) + monkeypatch.delenv("TARGET_AZURE_OPENAI_ENDPOINT", raising=False) + config_path = _write_config( + tmp_path, + { + "backend": "qwen", + "optimizer_backend": "openai_chat", + "target_backend": "openai_chat", + }, + ) + + error = webui_app.validate_training_config(config_path, {}) + + assert "missing Azure/OpenAI-compatible endpoint for optimizer, target" in error + assert "model.backend is qwen" in error + + +def test_preflight_reports_unreachable_qwen_endpoint(tmp_path, monkeypatch): + monkeypatch.setattr(webui_app, "_can_connect_to_url", lambda _url: False) + config_path = _write_config( + tmp_path, + { + "backend": "qwen", + "optimizer_backend": "qwen_chat", + "target_backend": "qwen_chat", + "qwen_chat_base_url": "http://127.0.0.1:9/v1", + }, + ) + + error = webui_app.validate_training_config(config_path, {}) + + assert "cannot connect to qwen_chat endpoint" in error + assert "127.0.0.1:9" in error + + +def test_preflight_accepts_reachable_qwen_endpoint(tmp_path, monkeypatch): + seen_urls = [] + monkeypatch.setattr(webui_app, "_can_connect_to_url", lambda url: seen_urls.append(url) or True) + config_path = _write_config( + tmp_path, + { + "optimizer_backend": "qwen_chat", + "target_backend": "qwen_chat", + "qwen_chat_base_url": "http://qwen.example/v1", + }, + ) + + assert webui_app.validate_training_config(config_path, {}) is None + assert seen_urls == ["http://qwen.example/v1", "http://qwen.example/v1"]