commit e09edc5f1601ff243d2d55d41f9946376f098d4c Author: wehub-resource-sync Date: Mon Jul 13 12:28:42 2026 +0800 chore: import upstream snapshot with attribution diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 0000000..88ba107 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,20 @@ +{ + "name": "webwright", + "owner": { + "name": "Microsoft Research", + "url": "https://github.com/microsoft/Webwright" + }, + "plugins": [ + { + "name": "webwright", + "source": { + "source": "github", + "repo": "microsoft/Webwright" + }, + "description": "Turn your coding agent into a SOTA browser agent. Drives a local Playwright workspace via one bash command at a time, saving screenshots and an action log into final_runs/run_/, and visually self-verifies the result.", + "version": "0.1.0", + "category": "automation", + "keywords": ["browser", "playwright", "web-agent", "automation", "rpa"] + } + ] +} diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..350eb33 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,15 @@ +{ + "name": "webwright", + "version": "0.1.0", + "description": "Turn your coding agent into a SOTA browser agent. Drives a local Playwright workspace via one bash command at a time, saving screenshots and an action log into final_runs/run_/, and visually self-verifies the result.", + "author": { + "name": "Microsoft Research", + "url": "https://github.com/microsoft/Webwright" + }, + "homepage": "https://github.com/microsoft/Webwright", + "repository": "https://github.com/microsoft/Webwright", + "license": "MIT", + "keywords": ["browser", "playwright", "web-agent", "automation", "rpa"], + "commands": "./skills/webwright/commands", + "skills": "./skills" +} diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json new file mode 100644 index 0000000..8783217 --- /dev/null +++ b/.codex-plugin/plugin.json @@ -0,0 +1,23 @@ +{ + "name": "webwright", + "version": "0.1.0", + "description": "Turn your coding agent into a SOTA browser agent. Drives a local Playwright workspace via one bash command at a time, saving screenshots and an action log into final_runs/run_/, and visually self-verifies the result.", + "author": { + "name": "Microsoft Research", + "url": "https://github.com/microsoft/Webwright" + }, + "homepage": "https://github.com/microsoft/Webwright", + "repository": "https://github.com/microsoft/Webwright", + "license": "MIT", + "keywords": ["browser", "playwright", "web-agent", "automation", "rpa"], + "skills": "./skills/", + "interface": { + "displayName": "Webwright", + "shortDescription": "SOTA browser agent driven by your coding agent", + "longDescription": "Drives a local Playwright workspace via one bash command at a time, saving screenshots and an action log into final_runs/run_/, and visually self-verifies the result.", + "developerName": "Microsoft Research", + "category": "Productivity", + "capabilities": ["Read", "Write"], + "websiteURL": "https://github.com/microsoft/Webwright" + } +} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c914d72 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +**/__pycache__/ +*.pyc +*.egg-info/ +outputs/ +.tmp/ +.claude/ +.venv/ +venv/ \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..686e5e7 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,10 @@ +# Microsoft Open Source Code of Conduct + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). + +Resources: + +- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) +- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) +- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns +- Employees can reach out at [aka.ms/opensource/moderation-support](https://aka.ms/opensource/moderation-support) diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..9e841e7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/README.md b/README.md new file mode 100644 index 0000000..891bbd7 --- /dev/null +++ b/README.md @@ -0,0 +1,410 @@ +# Webwright + +

+ Webwright logo +

+ +

Turn Your Coding Models to Be State-of-the-art Browser Agents

+ +

+ Python + Playwright + Backends + Footprint +

+ +- ๐Ÿ“ **Blog:** [Webwright: A Terminal Is All You Need For Web Agents](https://www.microsoft.com/en-us/research/articles/webwright-a-terminal-is-all-you-need-for-web-agents/) +- ๐ŸŒ **Project Page:** [microsoft.github.io/Webwright](https://microsoft.github.io/Webwright/) + +Webwright gives LLM a terminal where it can launch multiple browser sessions to inspect the page and complete a web task. It captures and inspects page screenshots/states only when needed. It enforces each web task to be completed end-to-end within a re-runnable Python script, i.e. your web agent browsing history is a single code file. No multi-agent system, no graph engine, no plugin layer, no hidden orchestration โ€” just a terminal, a browser, and a model. + +Already got your favorite agents, and wonder how to make Claude Code, Codex, Hermes, OpenClaw more capable in browser tasks? Consider adding [Webwright plugin/skills](#-use-as-a-claude-code-skill)! + +--- + +## ๐Ÿ“ฐ News + +- **2026-05-11** โ€” Support Task2UI mode: Webwright completes the task and renders task results into an HTML-based web app you can easily view and reuse. +- **2026-05-06** โ€” Codex and Claude Code plugin manifests added; install via `/plugin install webwright@webwright`. OpenClaw and Hermes Agent integrations shipped; the same `skills/webwright/` folder now loads across Claude Code, Codex, OpenClaw, and Hermes. +- **2026-05-04** โ€” Initial public release: ~1.5k LoC, OpenAI / Anthropic / OpenRouter backends, Playwright environment. + +--- + +
+๐Ÿ’ก Motivation: Beyond Step-by-Step Web Interaction in a Stateful Browser + +Most web agents today treat the browser session itself as the workspace: at each step the model receives the current page state and predicts a single next operation โ€” a click, a type, a DOM selector, or a short tool call. Whatever the format, the agent is locked into predicting one web action at a time inside a predefined interaction loop. That harness was useful when LLMs were weaker. As models get stronger at writing and debugging code, the same harness becomes a bottleneck. + +Webwright takes a different stance: **separate the agent from the browser**, and treat the browser as something the agent can launch, inspect, and discard while developing a program. The persistent artifact is not the browser session โ€” it's the **code and logs in the local workspace**. + +- ๐Ÿงฑ **Robust, reusable interaction with web environments** โ€” instead of fragile pixel-level actions, a coding agent with a terminal queries elements, waits for conditions, and handles dynamic behaviors like lazy loading or re-rendering. The resulting scripts can be rerun, adapted, and shared across tasks rather than rediscovered from scratch. +- โšก **Efficient composition of complex workflows** โ€” multi-step interactions like selecting a date or filling a form become a compact program. Loops, functions, and abstractions let the agent generalize across similar tasks (e.g. different dates) without re-predicting the same low-level sequences. Fewer interaction rounds, faster execution, less error accumulation on long horizons. +- ๐Ÿงช **Workspace-as-state, not browser-as-state** โ€” the agent can write exploratory scripts, spawn fresh browser sessions, and decide for itself when to capture screenshots and inspect failures, much like a human engineer iterating on an RPA script. +- ๐Ÿช„ **Surprisingly effective despite being minimal** โ€” this stripped-down setup turns out to handle complex and especially long-horizon web tasks well (see [Performance](#-performance)). + +
+ +--- + +
+๐ŸŒŸ Why Webwright + +Most web agent frameworks bury the actual agent loop under layers of abstractions. Webwright takes the opposite stance: + +- ๐Ÿชถ **Lightweight by design** โ€” core agent loop in a single ~450-line file, Playwright environment in ~570 lines, CLI in ~150 lines. +- ๐Ÿงฉ **Pluggable model backends** โ€” OpenAI, Anthropic, and OpenRouter, each ~150โ€“200 lines. +- ๐Ÿ” **Zero hidden frameworks** โ€” just `httpx`, `pydantic`, `playwright`, and `typer`. +- ๐Ÿ” **Flat prompt โ†’ observe โ†’ execute script loop** โ€” readable end-to-end, easy to debug, easy to fork. +- ๐Ÿงช **Run-artifact first** โ€” every run writes trajectories and screenshots to disk for inspection. + +If you want a minimal, easy-to-debug starting point for browser-using agents instead of another heavyweight platform, this is it. + +
+ +--- + +
+๐Ÿ†š How Webwright Differs From Other Browser-Agent Repos + +How they differ at the architectural level: + +| | **Stagehand (Browserbase)** | **agent-browser (Vercel)** | **browser-use** | **Webwright** | +| ------------------- | ------------------------------------------------------------ | ------------------------------------------------------------------------- | ----------------------------------------------------- | ------------------------------------------------------------------------- | +| **Paradigm** | Hybrid: code + NL primitives (`act` / `extract` / `agent`) | CLI tool that *another* agent (Claude Code, Codex, etc.) calls | Autonomous LLM agent loop over DOM/AX snapshots | **Coding agent with a terminal**; browser is just an environment it spawns | +| **Action space** | Playwright code, or NL โ†’ LLM-translated Playwright | Discrete subcommands (`open`, `click @e2`, `snapshot`, `eval`) | Indexed click/type actions selected by the LLM | **Free-form Python (writes Playwright scripts itself)** | +| **What is "state"?**| The browser session | The browser session (held by daemon across CLI calls) | The browser session | **The local workspace โ€” code, screenshots, logs.** Browser is disposable. | +| **Loop shape** | Imperative; `agent()` does multi-step when needed | One CLI invocation per micro-step | observe โ†’ predict next action โ†’ execute โ†’ repeat | write code โ†’ execute โ†’ inspect screenshots โ†’ repair (code-as-action) | +
+ + +--- + +## ๐ŸŽฅ Demo +https://github.com/user-attachments/assets/4ed94cd5-11be-4daa-b2d7-1260a803baca + +--- + +## ๐Ÿ“Š Performance + +State-of-the-art on two real-website benchmarks with a 100-step budget โ€” see the [blog post](https://www.microsoft.com/en-us/research/articles/webwright-a-terminal-is-all-you-need-for-web-agents/) for full details. + +- ๐Ÿ† **Online-Mind2Web (300 tasks):** **86.7%** with GPT-5.4 โ€” highest among open-sourced harnesses in the AutoEval category. Claude Opus 4.7 reaches **84.7%**, and is stronger on the hard split (**80.5%** vs. 76.6% for GPT-5.4 at N=100). +- ๐Ÿš€ **Odysseys (200 long-horizon tasks):** **60.1%** with GPT-5.4 (avg. 76.1 steps) โ€” **+15.6 points** over the prior SOTA (Opus 4.6 at 44.5%, using vision based approach and persistent browser) and **+26.6 points** over base GPT-5.4 (33.5% using xy-coordinate prediction and persistent browser). +- ๐Ÿง  **Code-as-action beats coordinate prediction:** Webwright substantially outperforms a reproduced GPT-5.4 screenshot+xy-coordinate baseline across all difficulty splits. +- ๐Ÿงฐ **Small models + reusable tools:** generated scripts can be packaged as parameterized CLI tools โ€” even **Qwen-3.5-9B** completes tasks well on Online-Mind2Web sites with 5+ tools available. + +

+ Odysseys long-horizon eval @ 100 steps + Online-Mind2Web AutoEval @ 100 steps +

+ +--- + +## ๐Ÿ—บ๏ธ Project Map + +``` +webwright/ +โ”œโ”€โ”€ pyproject.toml # package: webwright +โ”œโ”€โ”€ src/webwright/ +โ”‚ โ”œโ”€โ”€ run/cli.py # CLI entrypoint (`webwright`) +โ”‚ โ”œโ”€โ”€ agents/default.py # core agent loop +โ”‚ โ”œโ”€โ”€ environments/ # Playwright browser workspace +โ”‚ โ”œโ”€โ”€ tools/ # image_qa, self_reflection +โ”‚ โ”œโ”€โ”€ models/ # openai_model, anthropic_model, base +โ”‚ โ”œโ”€โ”€ config/ # base.yaml, model_openai.yaml, model_claude.yaml +โ”‚ โ””โ”€โ”€ utils/ +โ”œโ”€โ”€ assets/ +โ”‚ โ””โ”€โ”€ task_showcase/ # tiny Flask dashboard for repeatable runs +โ”‚ โ”œโ”€โ”€ app.py +โ”‚ โ”œโ”€โ”€ templates/ # dashboard.html, task.html +โ”‚ โ””โ”€โ”€ tasks// # task.json + report.json per task +โ”œโ”€โ”€ tests/ +โ””โ”€โ”€ outputs/ # run artifacts (trajectories, screenshots) +``` + +--- + +## ๐Ÿ“ฐ Task Showcase (repeatable runs as a dashboard) + +A tiny Flask app under [`assets/task_showcase/`](assets/task_showcase/README.md) consolidates +Webwright runs for **repeatable** odyssey tasks (deals, inventory, listings, +job boards, weather, etc.) into a single dashboard. Each task ships only two +files โ€” `task.json` (metadata) and `report.json` (curated, structured output: +sources + result sections like tables, lists, summaries) โ€” and the templates +render them generically, so adding a new task is just dropping a new folder +in `assets/task_showcase/tasks/`. + +```bash +pip install flask +python assets/task_showcase/app.py # http://127.0.0.1:5005 +``` + +To have Webwright produce a renderer-ready task folder at runtime, stack the +Task Showcase overlay: + +```bash +python -m webwright.run.cli \ + -c base.yaml -c model_openai.yaml -c task_showcase.yaml \ + -t "" \ + --task-id my_repeatable_task \ + -o outputs/default +``` + +> **Note:** `report.json` is only generated when `-c task_showcase.yaml` is +> included. A plain `base.yaml` run produces `trajectory.json` and debug +> artifacts but no `report.json`. + +The run writes `task_showcase/tasks//task.json` and `report.json` +inside the output workspace. Render those generated files without copying them +back into the repo: + +```bash +python assets/task_showcase/app.py \ + --tasks-dir outputs/default//task_showcase/tasks +``` + +--- + +## ๐Ÿš€ Quick Start + +### Prerequisites + +- Python 3.10+ +- Chromium installed through Playwright +- An API key for your chosen backend (OpenAI, Anthropic, or OpenRouter) + +### Install + +```bash +pip install -e . +playwright install chromium +``` + +### Run + +Export credentials for the configured backend (for example, `OPENAI_API_KEY` +with `model_openai.yaml` or `ANTHROPIC_API_KEY` with `model_claude.yaml`). The +`image_qa` and `self_reflection` tools use the same configured model by default, +so an Anthropic run does not require an OpenAI key. Then: + +```bash +python -m webwright.run.cli \ + -c base.yaml -c model_openai.yaml \ + -t "Search for flights from SEA to JFK on 2026-08-15 to 2026-08-20" \ + --start-url https://www.google.com/flights \ + --task-id demo_openai \ + -o outputs/default +``` + +### ๐Ÿšฉ Flags + +| Flag | Description | +|------|-------------| +| `-c` | Config file(s) from `src/webwright/config/` (stackable). | +| `-t` | Task instruction. | +| `--start-url` | Initial page. | +| `--task-id` | Output subfolder name. | +| `-o` | Output directory. | + +--- + +## ๐Ÿ”Œ Use as a Plugin + +Webwright ships plugin manifests for both [Claude Code](https://docs.claude.com/en/docs/claude-code/plugins) ([`.claude-plugin/plugin.json`](.claude-plugin/plugin.json)) and [OpenAI Codex](https://developers.openai.com/codex/plugins) ([`.codex-plugin/plugin.json`](.codex-plugin/plugin.json)), with the shared skill at [`skills/webwright/`](skills/webwright/) and slash commands at [`skills/webwright/commands/`](skills/webwright/commands/). The host agent drives the Webwright loop natively โ€” no extra LLM API key or cost beyond your host subscription. Hosts that read PNG screenshots natively skip the `image_qa` / `self_reflection` tools. + +Common runtime deps (install once after either path): + +```bash +pip install -e . +playwright install chromium +``` + +
+Claude Code + +### Install + +Install through the bundled marketplace inside Claude Code: + +```text +# 1. Add this repo as a Claude Code plugin marketplace +/plugin marketplace add microsoft/Webwright + +# 2. Install the plugin from that marketplace +/plugin install webwright@webwright +``` + +Prefer a local checkout? Point the marketplace command at the cloned repo instead: + +```text +/plugin marketplace add /absolute/path/to/Webwright +/plugin install webwright@webwright +``` + +### Use + +**Start a new Claude Code session** after installing โ€” plugins are loaded at session start and won't appear until you restart. + +You can either ask Claude Code in plain English (the skill auto-activates from its description), or use one of the slash commands: + +``` +/webwright:run search Google Flights for flights from SEA to JFK on 2026-08-15 to 2026-08-20 +/webwright:craft search a ticket on Google Flights from LAX to SFO depart June 7 return June 14 +``` + +- `/webwright:run` (or any plain prompt) produces a **one-shot** `final_script.py` for the literal task values. +- `/webwright:craft` produces a **reusable CLI tool**: `final_script.py` becomes one parameterized function with a Google-style `Args:` docstring and an `argparse` wrapper whose flags default to the concrete task values, so you can rerun it later with different arguments โ€” e.g. `python final_script.py --origin JFK --destination LAX --depart-date 2026-07-01`. + +In both modes Claude Code scaffolds a workspace with `plan.md`, runs instrumented Playwright scripts under `final_runs/run_/`, and visually self-verifies each critical point against the saved screenshots. + +
+ +
+OpenAI Codex + +### Install + +Codex reads Claude-style marketplaces, so the same repo works as a Codex plugin marketplace. From the Codex CLI: + +```bash +# 1. Add this repo as a Codex plugin marketplace +codex plugin marketplace add microsoft/Webwright + +# 2. Open the plugin browser and install Webwright +codex +/plugins +``` + +Prefer a local checkout? + +```bash +codex plugin marketplace add /absolute/path/to/Webwright +``` + +Then restart Codex so the new marketplace and plugin are picked up. + +### Use + +In a new Codex thread, either ask in plain English (the skill auto-activates from its description) or invoke the bundled skill explicitly with `@webwright`: + +``` +@webwright search Google Flights for flights from SEA to JFK on 2026-08-15 to 2026-08-20 +``` + +Codex scaffolds a workspace with `plan.md`, runs instrumented Playwright scripts under `final_runs/run_/`, and visually self-verifies each critical point against the saved screenshots. + +To turn the plugin off without uninstalling, set its entry in `~/.codex/config.toml` to `enabled = false` and restart Codex. + +
+ +
+๐Ÿฆž OpenClaw + +### Install + +Install directly from a local checkout (path, archive, npm spec, git repo, or `clawhub:` spec all work): + +```bash +openclaw plugins install /absolute/path/to/Webwright +openclaw gateway restart # reload so the plugin and skill are picked up +``` + +Verify: + +```bash +openclaw plugins list | grep webwright +openclaw skills list | grep webwright # should show "โœ“ ready" +``` + +### Use + +The `webwright` skill is now available to any OpenClaw agent surface (CLI, Telegram, etc.) โ€” invoke it by asking the agent in natural language, or via the slash commands shipped under [`skills/webwright/commands/`](skills/webwright/commands/), e.g. `/webwright run `. + +To uninstall: `openclaw plugins uninstall webwright`. + +
+ +
+Hermes Agent + +### Install + +[Hermes Agent](https://github.com/NousResearch/hermes-agent) is a [skills-compatible client](https://agentskills.io), so the same `skills/webwright/` folder loads as a Hermes skill. Symlink it into your Hermes user-skills directory: + +```bash +mkdir -p ~/.hermes/skills +ln -sfn /absolute/path/to/Webwright/skills/webwright ~/.hermes/skills/webwright +``` + +No Hermes-specific manifest is needed; only `SKILL.md` is loaded. + +### Use + +Start Hermes (`hermes`) and ask it to drive a web task in natural language โ€” the skill auto-activates from its description. You can also invoke it explicitly with `/webwright`. + +Note: the named subcommands shipped under [`skills/webwright/commands/`](skills/webwright/commands/) (`/webwright:run`, `/webwright:craft`) are a Claude Code / Codex convention and are inert in Hermes; the skill itself still works end-to-end. + +
+ +## ๐Ÿ“ƒ Trajectory Comparison & Viewer + +You can run the same tasks using the Webwright harness and its Codex / GitHub Copilot skill variant, and see how token usage and trajectories stack up between different harnesses. The trajectory viewer supports Codex, GitHub Copilot and Webwright harness traces. + +![Trajectory comparison](assets/trajectory-compare.png) + +### How to use + +```bash +cd assets/compare_trajectory/ +python3 -m http.server +``` + +Open the webpage in your browser and upload the Webwright `raw_responses.jsonl` and attach `trajectory.json` to view. Then on the other side you can upload your Codex or GitHub Copilot trace. + +### Obtaining Codex traces: + +``` +ls ~/.codex/sessions/2026/MONTH/DAY/SESSION_ID.jsonl +``` + +### Obtaining GitHub Copilot traces: + +``` +/export file session +-> session.md is the uploadable trace +``` + +### Quick Comparison + +#### "Find the cheapest used 8-cylinder bmw made between 2005-2015 and priced from 25,000 to 50,000 dollars with mileage less than 50,000 miles or less." + +| Tokens | Webwright Harness (Local Browser Mode) | Codex Webwright Skill | +| --- | ---: | ---: | +| Input | 420,433 | 3,271,143 | +| Output | 3,593 | 20,040 | +| Reasoning | 0 | 4,410 | +| Cached | 217,216 | 3,081,3440 | +| Total | 424,026 | 3,291,183 | + +Individual runs and results may vary. + +--- + +## Credits + +- [SWE-agent/mini-swe-agent](https://github.com/SWE-agent/mini-swe-agent/tree/main) โ€” design inspiration for the minimal agent loop. +- [Playwright](https://playwright.dev/) โ€” browser automation. + +## Citation + +If you use Webwright in your research or build on it, please cite this repository: + +```bibtex +@misc{webwright2026, + title = {Webwright: A terminal is all you need for web agents}, + author = {Lu, Yadong and Xu, Lingrui and Huang, Chao and Awadallah, Ahmed}, + year = {2026}, + howpublished = {\url{https://github.com/microsoft/Webwright}}, + note = {GitHub repository} +} +``` diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..c247730 --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,7 @@ +# WeHub ๆฅๆบ่ฏดๆ˜Ž + +- ๅŽŸๅง‹้กน็›ฎ๏ผš`microsoft/Webwright` +- ๅŽŸๅง‹ไป“ๅบ“๏ผšhttps://github.com/microsoft/Webwright +- ๅฏผๅ…ฅๆ–นๅผ๏ผšไธŠๆธธ้ป˜่ฎคๅˆ†ๆ”ฏ็š„ๆœ€ๆ–ฐๅฟซ็…ง +- ๅŽŸไฝœ่€…ใ€็‰ˆๆƒๅ’Œ่ฎธๅฏ่ฏไฟกๆฏไปฅๅŽŸๅง‹ไป“ๅบ“ๅŠๆœฌไป“ๅบ“ LICENSE ไธบๅ‡† +- ๆœฌๆ–‡ไปถไป…็”จไบŽ่ฎฐๅฝ•ๆฅๆบ๏ผŒไธไปฃ่กจ WeHub ๆ˜ฏๅŽŸ้กน็›ฎไฝœ่€… diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..e751608 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,14 @@ + + +## Security + +Microsoft takes the security of our software products and services seriously, which +includes all source code repositories in our GitHub organizations. + +**Please do not report security vulnerabilities through public GitHub issues.** + +For security reporting information, locations, contact information, and policies, +please review the latest guidance for Microsoft repositories at +[https://aka.ms/SECURITY.md](https://aka.ms/SECURITY.md). + + \ No newline at end of file diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 0000000..eaf439a --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,25 @@ +# TODO: The maintainer of this repo has not yet edited this file + +**REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project? + +- **No CSS support:** Fill out this template with information about how to file issues and get help. +- **Yes CSS support:** Fill out an intake form at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). CSS will work with/help you to determine next steps. +- **Not sure?** Fill out an intake as though the answer were "Yes". CSS will help you decide. + +*Then remove this first heading from this SUPPORT.MD file before publishing your repo.* + +# Support + +## How to file issues and get help + +This project uses GitHub Issues to track bugs and feature requests. Please search the existing +issues before filing new issues to avoid duplicates. For new issues, file your bug or +feature request as a new Issue. + +For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE +FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER +CHANNEL. WHERE WILL YOU HELP PEOPLE?**. + +## Microsoft Support Policy + +Support for this **PROJECT or PRODUCT** is limited to the resources listed above. diff --git a/assets/compare_trajectory/app.js b/assets/compare_trajectory/app.js new file mode 100644 index 0000000..6538df2 --- /dev/null +++ b/assets/compare_trajectory/app.js @@ -0,0 +1,2058 @@ +import { Tiktoken } from 'https://esm.sh/js-tiktoken/lite'; + +const { createApp } = window.Vue; + +const MODEL_TO_ENCODING = { + 'gpt-4.1': 'o200k_base', + 'gpt-4o': 'o200k_base', + 'gpt-4o-mini': 'o200k_base', + o1: 'o200k_base', + o3: 'o200k_base', + 'o3-mini': 'o200k_base', + 'gpt-4-turbo': 'cl100k_base', + 'gpt-4': 'cl100k_base', + 'gpt-3.5-turbo': 'cl100k_base' +}; + +const encodingCache = new Map(); + +function safeJsonParse(text) { + try { + return JSON.parse(text); + } catch (error) { + return null; + } +} + +function shortText(value, limit = 220) { + const clean = String(value || '').replace(/\s+/g, ' ').trim(); + if (!clean) { + return ''; + } + return clean.length > limit ? clean.slice(0, limit - 3) + '...' : clean; +} + +function shortMultiline(value, limit = 440) { + const clean = String(value || '').trim(); + if (!clean) { + return ''; + } + return clean.length > limit ? clean.slice(0, limit - 3) + '...' : clean; +} + +function numberFormat(value) { + if (value === null || value === undefined || value === '') { + return 'n/a'; + } + const number = Number(value); + if (!Number.isFinite(number)) { + return String(value); + } + return new Intl.NumberFormat().format(number); +} + +function formatTimestamp(value) { + if (!value) { + return 'n/a'; + } + const date = new Date(value); + if (Number.isNaN(date.getTime())) { + return String(value); + } + return date.toLocaleString(); +} + +function parseArguments(raw) { + if (!raw) { + return {}; + } + if (typeof raw === 'object') { + return raw; + } + const parsed = safeJsonParse(raw); + return parsed && typeof parsed === 'object' ? parsed : {}; +} + +function primaryCommand(raw) { + const input = String(raw || '').trim(); + if (!input) { + return 'none'; + } + const pieces = input.split(/&&|\|\|/).map((piece) => piece.trim()).filter(Boolean); + const target = pieces.find((piece) => !/^cd\s+/.test(piece)) || pieces[0] || input; + if (target.startsWith('python - <<')) { + return 'python-heredoc'; + } + if (target.startsWith('printf ')) { + return 'printf'; + } + const match = target.match(/^([A-Za-z0-9._/-]+)/); + return match ? match[1] : shortText(target, 40); +} + +function parseExitCode(text) { + const match = String(text || '').match(/(?:Return code:|Process exited with code| b[1] - a[1] || a[0].localeCompare(b[0])) + .map(([label, value]) => ({ label, value })); +} + +function toBarRows(rows, limit = 6) { + const subset = rows.slice(0, limit); + const max = subset.reduce((value, row) => Math.max(value, row.value), 0) || 1; + return subset.map((row) => ({ + label: row.label, + value: row.value, + width: Math.max(8, Math.round((row.value / max) * 100)) + })); +} + +function tokenRowsFromTotals(tokens) { + if (!tokens) { + return [{ label: 'availability', value: 'n/a' }]; + } + return [ + { label: 'input', value: numberFormat(tokens.input_tokens) }, + { label: 'output', value: numberFormat(tokens.output_tokens) }, + { label: 'reasoning', value: numberFormat(tokens.reasoning_output_tokens) }, + { label: 'cached', value: numberFormat(tokens.cached_input_tokens) }, + { label: 'total', value: numberFormat(tokens.total_tokens) } + ]; +} + +function extractTrajectoryUsage(text, fileName) { + const parsed = safeJsonParse(String(text || '').trim()); + if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.messages)) { + throw new Error('Attachment must be a valid trajectory.json file.'); + } + let latestUsage = null; + parsed.messages.forEach((message) => { + const usage = message && message.extra && message.extra.usage; + const cumulative = usage && usage.cumulative_response; + if (cumulative && typeof cumulative.total_tokens === 'number') { + latestUsage = cumulative; + } + }); + if (!latestUsage) { + throw new Error('trajectory.json does not contain messages[].extra.usage.cumulative_response.'); + } + return { + fileName, + exactTokens: latestUsage, + tokenRows: tokenRowsFromTotals(latestUsage), + tokenSourceLabel: 'Exact tokens from attached trajectory.json.' + }; +} + +function applyTrajectoryCompanion(trace, companion) { + return Object.assign({}, trace, { + exactTokens: companion.exactTokens, + tokenRows: companion.tokenRows, + tokenSourceLabel: companion.tokenSourceLabel, + tokenModeLabel: 'exact', + companionFileName: companion.fileName + }); +} + +function stripTrajectoryCompanion(trace) { + const exactTokens = trace.baseExactTokens || null; + return Object.assign({}, trace, { + exactTokens, + tokenRows: exactTokens ? tokenRowsFromTotals(exactTokens) : [{ label: 'availability', value: 'estimate pending' }], + tokenSourceLabel: trace.baseTokenSourceLabel, + tokenModeLabel: exactTokens ? 'exact' : 'estimate', + companionFileName: '' + }); +} + +function statusFromExit(exitCode, sessionId, output, done) { + if (exitCode === 0) { + return { label: 'ok', css: 'good' }; + } + if (exitCode !== null && exitCode !== 0) { + return { label: 'error', css: 'bad' }; + } + if (done === true) { + return { label: 'done', css: 'good' }; + } + if (sessionId || /process running/i.test(String(output || ''))) { + return { label: 'streaming', css: 'warn' }; + } + return { label: 'unknown', css: '' }; +} + +function splitConcatenatedJsonObjects(text) { + const source = String(text || ''); + const segments = []; + let start = -1; + let depth = 0; + let inString = false; + let escaped = false; + for (let index = 0; index < source.length; index += 1) { + const char = source[index]; + if (start === -1) { + if (char === '{') { + start = index; + depth = 1; + inString = false; + escaped = false; + } + continue; + } + if (inString) { + if (escaped) { + escaped = false; + } else if (char === '\\') { + escaped = true; + } else if (char === '"') { + inString = false; + } + continue; + } + if (char === '"') { + inString = true; + continue; + } + if (char === '{') { + depth += 1; + continue; + } + if (char === '}') { + depth -= 1; + if (depth === 0) { + segments.push(source.slice(start, index + 1)); + start = -1; + } + } + } + return segments.map((segment) => safeJsonParse(segment)).filter(Boolean); +} + +function getTokenizerSelection(value) { + const [mode, name] = String(value || 'model:o3').split(':'); + const encoding = mode === 'model' ? MODEL_TO_ENCODING[name] || 'o200k_base' : name || 'o200k_base'; + return { mode, name: name || 'o3', encoding }; +} + +async function loadEncoder(encoding) { + if (encodingCache.has(encoding)) { + return encodingCache.get(encoding); + } + const controller = new AbortController(); + const timeoutId = window.setTimeout(() => controller.abort(), 8000); + let response; + try { + response = await fetch(`https://tiktoken.pages.dev/js/${encoding}.json`, { + signal: controller.signal + }); + } finally { + window.clearTimeout(timeoutId); + } + if (!response.ok) { + throw new Error(`Failed to load ${encoding} ranks from CDN`); + } + const ranks = await response.json(); + const encoder = new Tiktoken(ranks); + encodingCache.set(encoding, encoder); + return encoder; +} + +function quickTokenEstimate(text) { + const source = String(text || ''); + if (!source.trim()) { + return 0; + } + return Math.max(1, Math.ceil(source.length / 4)); +} + +function mergeEstimateParts(parts) { + const grouped = new Map(); + for (const part of parts || []) { + if (!part || !String(part.text || '').trim()) { + continue; + } + const key = part.label || 'other'; + if (!grouped.has(key)) { + grouped.set(key, []); + } + grouped.get(key).push(String(part.text)); + } + return Array.from(grouped.entries()).map(([label, texts]) => ({ + label, + text: texts.join('\n\n') + })); +} + +function contentToText(content) { + if (typeof content === 'string') { + return content; + } + if (Array.isArray(content)) { + return content.map((part) => { + if (typeof part === 'string') { + return part; + } + if (part && typeof part === 'object') { + if (typeof part.text === 'string') { + return part.text; + } + if (typeof part.content === 'string') { + return part.content; + } + } + return part ? JSON.stringify(part) : ''; + }).filter(Boolean).join('\n\n'); + } + return content ? JSON.stringify(content) : ''; +} + +function normalizeObservation(message) { + const extra = message && message.extra ? message.extra : {}; + if (extra.observation && typeof extra.observation === 'object') { + return { + output: String(extra.observation.command_output || ''), + exitCode: typeof extra.observation.returncode === 'number' ? extra.observation.returncode : parseExitCode(extra.observation.command_output || ''), + command: String(extra.observation.command || ''), + success: extra.observation.success + }; + } + const text = contentToText(message.content); + if (!/^Observation:/i.test(text)) { + return null; + } + const outputMatch = text.match(/Command output:\n([\s\S]*)$/i); + const commandMatch = text.match(/Command:\s*([^\n]+)/i); + return { + output: outputMatch ? outputMatch[1].trim() : text, + exitCode: parseExitCode(text), + command: commandMatch ? commandMatch[1].trim() : '', + success: /Status:\s*ok/i.test(text) + }; +} + +function stripSessionMarkup(text) { + return String(text || '') + .replace(/
/gi, '') + .replace(/<\/details>/gi, '') + .replace(/.*?<\/summary>/gis, '') + .replace(/.*?<\/sub>/gis, '') + .replace(/^---$/gm, '') + .trim(); +} + +function extractCodeFences(text) { + const fences = []; + const regex = /(^|\n)(`{3,})([^\n]*)\n([\s\S]*?)\n\2(?=\n|$)/g; + let match = regex.exec(text); + while (match) { + fences.push({ + fence: match[2], + language: (match[3] || '').trim(), + content: match[4] + }); + match = regex.exec(text); + } + return fences; +} + +function normalizeCodeLanguage(language) { + const value = String(language || '').trim().toLowerCase(); + if (!value) { + return ''; + } + if (value === 'sh' || value === 'shell' || value === 'zsh') { + return 'bash'; + } + if (value === 'py') { + return 'python'; + } + if (value === 'patch') { + return 'diff'; + } + return value; +} + +function looksLikeShell(text) { + return /(^|\n)\s*(\$ |(?:cd|ls|pwd|cat|rg|sed|find|git|npm|pnpm|yarn|uv|python|python3|bash|sh|chmod|cp|mv|rm)\b)/m.test(String(text || '')); +} + +function inferCodeLanguage(text, fallback = '') { + const normalizedFallback = normalizeCodeLanguage(fallback); + const source = String(text || '').trim(); + if (!source) { + return normalizedFallback || 'plaintext'; + } + if (normalizedFallback) { + return normalizedFallback; + } + if (/^\*\*\* Begin Patch/m.test(source) || /^\*\*\* (Add|Update|Delete) File:/m.test(source)) { + return 'diff'; + } + if (looksLikePython(source)) { + return 'python'; + } + if (safeJsonParse(source) !== null) { + return 'json'; + } + if (looksLikeShell(source)) { + return 'bash'; + } + return 'plaintext'; +} + +function makeTextSection(text, label = '') { + return { type: 'text', label, text: String(text || '').trim() }; +} + +function makeCodeSection(text, language = '', label = '') { + return { + type: 'code', + label, + language: inferCodeLanguage(text, language), + text: String(text || '').trim() + }; +} + +function buildRichTextSections(text, options = {}) { + const source = String(text || '').trim(); + if (!source) { + return []; + } + if (options.forceCode) { + return [makeCodeSection(source, options.language || '', options.codeLabel || 'Code')]; + } + const sections = []; + const regex = /(^|\n)(`{3,})([^\n]*)\n([\s\S]*?)\n\2(?=\n|$)/g; + let cursor = 0; + let match = regex.exec(source); + while (match) { + const start = match.index + match[1].length; + const before = source.slice(cursor, start).trim(); + if (before) { + sections.push(makeTextSection(before, sections.length ? '' : (options.textLabel || ''))); + } + const language = normalizeCodeLanguage(match[3]); + const content = String(match[4] || '').trim(); + if (content) { + sections.push(makeCodeSection(content, language || options.language || '', options.codeLabel || 'Code')); + } + cursor = regex.lastIndex; + match = regex.exec(source); + } + const tail = source.slice(cursor).trim(); + if (tail) { + sections.push(makeTextSection(tail, sections.length ? '' : (options.textLabel || ''))); + } + if (!sections.length) { + if (options.detectCode && (options.language || looksLikePython(source) || looksLikeShell(source) || safeJsonParse(source) !== null)) { + return [makeCodeSection(source, options.language || '', options.codeLabel || 'Code')]; + } + return [makeTextSection(source, options.textLabel || '')]; + } + return sections; +} + +function buildCommandSections(command, channel, outputText = '', outputLabel = 'Output') { + const sections = []; + const commandText = String(command || '').trim(); + const resultText = String(outputText || '').trim(); + if (commandText) { + sections.push(makeCodeSection(commandText, inferCodeLanguage(commandText, channel), 'Command')); + } + if (resultText) { + sections.push(makeCodeSection(resultText, inferCodeLanguage(resultText), outputLabel)); + } + return sections; +} + +function createDetailEntry({ + id, + kind, + kindLabel, + summary = '', + timeLabel = 'n/a', + chips = [], + sections = [] +}) { + return { + id, + kind, + kindLabel: kindLabel || kind.replace(/_/g, ' '), + summary, + timeLabel, + chips: uniqueStrings(chips), + sections: sections.filter((section) => section && String(section.text || '').trim()) + }; +} + +function extractBoldSummary(text) { + const match = String(text || '').match(/\*\*([^*\n][\s\S]*?)\*\*/); + return match ? shortText(match[1].trim(), 140) : ''; +} + +function extractPatchTargets(patchText) { + const targets = []; + const regex = /^\*\*\*\s+(Add|Update|Delete) File:\s+(.+)$/gm; + let match = regex.exec(String(patchText || '')); + while (match) { + targets.push({ action: match[1], path: match[2].trim() }); + match = regex.exec(String(patchText || '')); + } + return targets; +} + +function summarizePatchTargets(targets) { + if (!targets.length) { + return 'apply_patch'; + } + const first = targets[0]; + const suffix = targets.length > 1 ? ' +' + (targets.length - 1) + ' more' : ''; + return 'apply_patch ' + first.action + ' ' + first.path + suffix; +} + +function extractLargeOutputPath(text) { + const match = String(text || '').match(/Saved to:\s*(\/\S+)/); + return match ? match[1] : ''; +} + +function extractSessionMetadata(text) { + const source = String(text || ''); + const readValue = (label) => { + const match = source.match(new RegExp('^> - \\*\\*' + label + ':\\*\\*\\s+(.+?)\\s*$', 'm')); + return match ? match[1].replace(/`/g, '').trim() : ''; + }; + return { + sessionId: readValue('Session ID'), + started: readValue('Started'), + duration: readValue('Duration'), + exported: readValue('Exported') + }; +} + +function looksLikePython(text) { + return /(from\s+[A-Za-z0-9_\.]+\s+import\s+|import\s+[A-Za-z0-9_\.]+|async\s+def\s+|def\s+[A-Za-z_][A-Za-z0-9_]*\s*\(|class\s+[A-Za-z_][A-Za-z0-9_]*\s*[(:]|asyncio\.run\(|await\s+[A-Za-z_][A-Za-z0-9_\.]*\(|if\s+__name__\s*==\s*['"]__main__['"])/m.test(String(text || '')); +} + +function extractPythonHeredocs(text) { + const blocks = []; + const regex = /(?:^|\n)[^\n]*\bpython(?:3(?:\.\d+)?)?\s+-\s*<<['"]?([A-Za-z_][A-Za-z0-9_]*)['"]?\n([\s\S]*?)\n\1(?=\n|$)/g; + let match = regex.exec(String(text || '')); + while (match) { + const body = String(match[2] || '').trim(); + if (body && looksLikePython(body)) { + blocks.push({ title: 'python heredoc', text: body, sourceLabel: 'python heredoc' }); + } + match = regex.exec(String(text || '')); + } + return blocks; +} + +function extractPythonFromPatch(text) { + const lines = String(text || '').split(/\r?\n/); + const blocks = []; + let active = null; + const flush = () => { + if (!active) { + return; + } + const body = active.lines.join('\n').replace(/^\n+|\n+$/g, ''); + if (body && looksLikePython(body)) { + blocks.push({ + title: active.path, + text: body, + sourceLabel: active.action.toLowerCase() + ' patch' + }); + } + active = null; + }; + lines.forEach((line) => { + const fileMatch = line.match(/^\*\*\*\s+(Add|Update|Delete) File:\s+(.+)$/); + if (fileMatch) { + flush(); + const action = fileMatch[1]; + const path = fileMatch[2].trim(); + active = path.endsWith('.py') && action !== 'Delete' ? { action, path, lines: [] } : null; + return; + } + if (!active) { + return; + } + if (/^\*\*\* End Patch$/.test(line)) { + flush(); + return; + } + if (line.startsWith('@@') || line.startsWith('-')) { + return; + } + if (line.startsWith('+')) { + active.lines.push(line.slice(1)); + return; + } + active.lines.push(line); + }); + flush(); + return blocks; +} + +function extractPythonBlocks(text) { + const blocks = []; + const rawText = String(text || '').trim(); + extractCodeFences(text).forEach((fence) => { + const language = String(fence.language || '').toLowerCase(); + const content = String(fence.content || '').trim(); + if (!content) { + return; + } + if (language === 'python' || language === 'py' || (!language && looksLikePython(content))) { + blocks.push({ title: 'python fence', text: content, sourceLabel: language || 'python fence' }); + } + }); + blocks.push(...extractPythonHeredocs(text)); + blocks.push(...extractPythonFromPatch(text)); + if (rawText && looksLikePython(rawText)) { + blocks.push({ title: 'python snippet', text: rawText, sourceLabel: 'python snippet' }); + } + const seen = new Set(); + return blocks.filter((block) => { + const key = String(block.text || '').trim(); + if (!key || seen.has(key)) { + return false; + } + seen.add(key); + return true; + }); +} + +function uniqueStrings(values) { + return Array.from(new Set((values || []).filter(Boolean))); +} + +function appendPythonScripts(target, seen, text, options = {}) { + extractPythonBlocks(text).forEach((block) => { + const scriptText = String(block.text || '').trim(); + if (!scriptText || seen.has(scriptText)) { + return; + } + seen.add(scriptText); + const order = target.length + 1; + target.push({ + id: (options.idPrefix || 'python-script') + '-' + order, + order, + title: block.title || options.title || 'Python script ' + order, + text: scriptText, + sections: [makeCodeSection(scriptText, 'python', 'Python')], + timeLabel: options.timeLabel || 'n/a', + sourceLabel: options.sourceLabel || block.sourceLabel || '', + chips: uniqueStrings([options.sourceLabel || block.sourceLabel || 'python'].concat(options.chips || [])) + }); + }); +} + +function extractActionPayload(record, action = null) { + const source = record && typeof record === 'object' ? record : {}; + const actionSource = action && typeof action === 'object' ? action : {}; + const bashCommand = String(source.bash_command || actionSource.bash_command || '').trim(); + if (bashCommand) { + return { + text: bashCommand, + field: 'bash_command', + channel: 'bash_command', + estimateLabel: 'bash_command fields' + }; + } + const pythonCode = String(source.python_code || actionSource.python_code || '').trim(); + if (pythonCode) { + return { + text: pythonCode, + field: 'python_code', + channel: 'python_code', + estimateLabel: 'python_code fields' + }; + } + const genericCommand = String(source.command || actionSource.command || '').trim(); + if (genericCommand) { + const looksPython = looksLikePython(genericCommand); + return { + text: genericCommand, + field: 'command', + channel: looksPython ? 'python_code' : 'command', + estimateLabel: looksPython ? 'python_code fields' : 'command fields' + }; + } + return { + text: '', + field: '', + channel: '', + estimateLabel: '' + }; +} + +function extractModelToolPayload(args, fallbackCommand) { + if (typeof args === 'string' && args.trim()) { + return args.trim(); + } + if (!args || typeof args !== 'object') { + return String(fallbackCommand || '').trim(); + } + for (const key of ['input', 'cmd', 'command', 'bash_command']) { + if (typeof args[key] === 'string' && args[key].trim()) { + return args[key].trim(); + } + } + const picked = {}; + ['query', 'path', 'pattern', 'skill', 'filePath', 'old_path', 'new_path'].forEach((key) => { + if (typeof args[key] === 'string' && args[key].trim()) { + picked[key] = args[key].trim(); + } + }); + if (Array.isArray(args.urls) && args.urls.length) { + picked.urls = args.urls; + } + return Object.keys(picked).length ? JSON.stringify(picked, null, 2) : String(fallbackCommand || '').trim(); +} + +function extractFirstPath(text) { + const match = String(text || '').match(/\/(?:[^\s`*]|\\ )+/); + return match ? match[0].replace(/\\ /g, ' ') : ''; +} + +function parseMarkdownSections(text) { + const lines = String(text || '').split(/\r?\n/); + const sections = []; + let pendingTime = ''; + let index = 0; + while (index < lines.length) { + const line = lines[index]; + const timeMatch = line.match(/^.*?โฑ๏ธ\s*([^<]+)<\/sub>$/); + if (timeMatch) { + pendingTime = timeMatch[1].trim(); + index += 1; + continue; + } + const headingMatch = line.match(/^###\s+(โ„น๏ธ Info|๐Ÿ‘ค User|๐Ÿ’ฌ Copilot|โœ…\s+`([^`]+)`)\s*$/); + if (!headingMatch) { + index += 1; + continue; + } + const tool = headingMatch[2] || ''; + let type = 'info'; + if (tool) { + type = 'tool'; + } else if (/User/.test(headingMatch[1])) { + type = 'user'; + } else if (/Copilot/.test(headingMatch[1])) { + type = 'copilot'; + } + let cursor = index + 1; + const bodyLines = []; + while (cursor < lines.length) { + if (lines[cursor].match(/^.*?โฑ๏ธ/)) { + break; + } + if (lines[cursor].match(/^###\s+(โ„น๏ธ Info|๐Ÿ‘ค User|๐Ÿ’ฌ Copilot|โœ…\s+`[^`]+`)\s*$/)) { + break; + } + bodyLines.push(lines[cursor]); + cursor += 1; + } + sections.push({ type, tool, timeLabel: pendingTime || 'n/a', body: bodyLines.join('\n').trim() }); + pendingTime = ''; + index = cursor; + } + return sections; +} + +function parseSessionToolSection(section, side, commandOrder, eventOrder) { + const body = section.body || ''; + const fences = extractCodeFences(body); + const plainBody = stripSessionMarkup(body); + const title = extractBoldSummary(plainBody); + const argsFence = fences.find((fence) => fence.language === 'json'); + const parsedArgs = argsFence ? safeJsonParse(argsFence.content.trim()) : null; + let command = ''; + let output = ''; + let exitCode = null; + let extraChips = []; + let payloadText = ''; + if (section.tool === 'bash') { + const commandFence = fences.find((fence) => /^\$\s/m.test(fence.content)); + const outputFenceIndex = commandFence ? fences.indexOf(commandFence) + 1 : -1; + command = commandFence ? commandFence.content.replace(/^\$\s?/, '').trim() : ''; + output = outputFenceIndex > 0 && fences[outputFenceIndex] ? fences[outputFenceIndex].content.trim() : plainBody; + exitCode = parseExitCode(output); + const savedPath = extractLargeOutputPath(output); + if (title) { + extraChips.push(title); + } + if (savedPath) { + extraChips.push('saved output'); + } + payloadText = command; + } else if (section.tool === 'apply_patch') { + const patchValue = typeof parsedArgs === 'string' ? parsedArgs : parsedArgs && typeof parsedArgs.input === 'string' ? parsedArgs.input : ''; + const patchTargets = extractPatchTargets(patchValue); + command = patchValue ? ('apply_patch\n' + patchValue.trim()) : 'apply_patch'; + output = fences.length > 1 ? fences[fences.length - 1].content.trim() : plainBody; + extraChips = [summarizePatchTargets(patchTargets)].concat(patchTargets.slice(0, 3).map((target) => target.action.toLowerCase())); + payloadText = patchValue; + } else if (section.tool === 'view') { + const path = extractFirstPath(plainBody); + command = path ? 'view ' + path : 'view'; + output = fences.length ? fences[fences.length - 1].content.trim() : plainBody; + if (title) { + extraChips.push(title); + } + payloadText = extractModelToolPayload(parsedArgs, command); + } else if (section.tool === 'glob') { + const pattern = fences.length ? fences[0].content.trim() : shortText(plainBody, 120); + command = pattern ? 'glob ' + pattern : 'glob'; + output = fences.length > 1 ? fences[1].content.trim() : plainBody; + if (title) { + extraChips.push(title); + } + payloadText = extractModelToolPayload(parsedArgs, command); + } else if (section.tool === 'skill') { + const argsText = parsedArgs ? JSON.stringify(parsedArgs) : shortText(plainBody, 160); + command = 'skill ' + shortText(argsText, 80); + output = fences.length ? fences[fences.length - 1].content.trim() : plainBody; + if (title) { + extraChips.push(title); + } + payloadText = extractModelToolPayload(parsedArgs, command); + } else { + const path = extractFirstPath(plainBody); + command = section.tool + (path ? ' ' + path : ''); + output = fences.length ? fences[fences.length - 1].content.trim() : plainBody; + if (title) { + extraChips.push(title); + } + payloadText = extractModelToolPayload(parsedArgs, command); + } + if (!payloadText) { + payloadText = extractModelToolPayload(parsedArgs, command); + } + if (title && section.tool === 'apply_patch') { + output = (title + '\n' + output).trim(); + } + const state = statusFromExit(exitCode, '', output, false); + const pythonScripts = []; + appendPythonScripts(pythonScripts, new Set(), payloadText, { + idPrefix: side + '-session-python-' + commandOrder, + timeLabel: section.timeLabel, + sourceLabel: section.tool, + chips: [section.tool] + }); + return { + run: { + id: side + '-session-command-' + commandOrder, + order: commandOrder, + command, + primaryCommand: primaryCommand(command || section.tool), + channel: section.tool, + timeLabel: section.timeLabel, + exitCode, + outputPreview: shortMultiline(output, 520), + outputText: output, + status: state.label, + statusClass: state.css, + sections: buildCommandSections(command, section.tool, output) + }, + events: [ + { + id: side + '-session-event-' + eventOrder, + kind: 'tool_call', + summary: section.tool === 'apply_patch' ? (command || plainBody) : shortMultiline(command || plainBody, 460), + timeLabel: section.timeLabel, + chips: [section.tool, primaryCommand(command || section.tool)].concat(extraChips) + }, + { + id: side + '-session-event-' + (eventOrder + 1), + kind: 'tool_result', + summary: shortMultiline(output || plainBody, 460), + timeLabel: section.timeLabel, + chips: [section.tool].concat(exitCode !== null ? ['exit ' + exitCode] : []).concat(extractLargeOutputPath(output) ? ['saved output'] : []) + } + ], + detailEntries: [ + createDetailEntry({ + id: side + '-session-detail-tool-' + commandOrder, + kind: 'tool', + kindLabel: 'Tool', + summary: section.tool === 'apply_patch' + ? (command || plainBody) + : shortMultiline(command || output || plainBody, 460), + timeLabel: section.timeLabel, + chips: [section.tool, primaryCommand(command || section.tool)] + .concat(extraChips) + .concat(exitCode !== null ? ['exit ' + exitCode] : []) + .concat(extractLargeOutputPath(output) ? ['saved output'] : []), + sections: buildCommandSections(command || payloadText, section.tool, output || plainBody) + }) + ], + estimateParts: [ + title ? { label: 'tool titles', text: title } : null, + parsedArgs ? { label: 'tool arguments', text: typeof parsedArgs === 'string' ? parsedArgs : JSON.stringify(parsedArgs, null, 2) } : null, + command ? { label: 'tool commands', text: command } : null, + output ? { label: 'tool outputs', text: output } : null + ].filter(Boolean), + pythonScripts + }; +} + +function detectFormat(text) { + const source = String(text || '').trim(); + if (!source) { + return null; + } + if (/^#\s+๐Ÿค–\s+Copilot CLI Session/m.test(source) || /^###\s+๐Ÿ’ฌ\s+Copilot/m.test(source)) { + return 'copilot_session_md'; + } + if (source[0] === '{' || source[0] === '[') { + const parsed = safeJsonParse(source); + if (parsed && typeof parsed === 'object' && Array.isArray(parsed.messages) && parsed.info) { + return 'trajectory_json'; + } + } + const lines = source.split(/\r?\n/).filter(Boolean); + if (!lines.length) { + return null; + } + const first = safeJsonParse(lines[0]); + if (!first || typeof first !== 'object') { + return null; + } + if (first.timestamp && first.type) { + return 'codex_jsonl'; + } + if (first.event === 'raw_text' && typeof first.raw_text === 'string') { + return 'raw_response_jsonl'; + } + return null; +} + +function finalizeTrace(base) { + const commandFamilies = toBarRows(countBy(base.commandRuns, (run) => run.primaryCommand)); + const defaultChannelFamilies = toBarRows(countBy(base.events, (event) => event.kind)); + const exactTokens = base.exactTokens || null; + const tokenSourceLabel = base.tokenSourceLabel || 'Estimated from extracted trace text.'; + return { + side: base.side, + kind: base.kind, + formatLabel: base.formatLabel, + description: base.description, + fileName: base.fileName, + filePath: base.filePath, + taskPrompt: base.taskPrompt, + events: base.events, + thoughtEntries: base.thoughtEntries, + commandRuns: base.commandRuns, + finalResponses: base.finalResponses, + pythonScripts: base.pythonScripts || [], + detailEntries: base.detailEntries || [], + metrics: base.metrics, + commandFamilies, + channelFamilies: base.channelFamilies || defaultChannelFamilies, + exactTokens, + baseExactTokens: exactTokens, + tokenRows: exactTokens ? tokenRowsFromTotals(exactTokens) : [{ label: 'availability', value: 'estimate pending' }], + tokenSourceLabel, + baseTokenSourceLabel: tokenSourceLabel, + tokenModeLabel: exactTokens ? 'exact' : 'estimate', + tokenEstimateParts: mergeEstimateParts(base.tokenEstimateParts || []), + countedFieldLabels: base.countedFieldLabels || [], + tokenBasisRows: [], + tokenEstimateTotal: null, + tokenEstimateError: '', + tokenSelectionLabel: '', + companionFileName: '', + summary: { + thoughts: base.thoughtEntries.length, + commands: base.commandRuns.length, + commandRuns: base.commandRuns.length, + pythonScripts: (base.pythonScripts || []).length, + finalResponses: base.finalResponses.length, + events: base.events.length + } + }; +} + +function normalizeCodex(text, fileName, filePath, side) { + const lines = text.split(/\r?\n/).filter(Boolean); + const entries = []; + const callById = new Map(); + const runByCallId = new Map(); + const runBySession = new Map(); + const commandRuns = []; + const thoughtEntries = []; + const finalResponses = []; + const pythonScripts = []; + const pythonSeen = new Set(); + const events = []; + const detailEntries = []; + const estimateParts = []; + let latestTokens = null; + let thoughtOrder = 0; + let commandOrder = 0; + let userPrompt = ''; + lines.forEach((line) => { + const item = safeJsonParse(line); + if (!item || typeof item !== 'object') { + return; + } + entries.push(item); + if (item.type === 'event_msg' && item.payload && item.payload.type === 'token_count') { + latestTokens = item.payload.info ? item.payload.info.total_token_usage : null; + } + }); + entries.forEach((item, index) => { + const id = side + '-codex-' + index; + const timeLabel = formatTimestamp(item.timestamp || ''); + if (item.type === 'event_msg' && item.payload && item.payload.type === 'user_message') { + const summary = shortMultiline(item.payload.message, 360); + if (!userPrompt) { + userPrompt = summary; + } + estimateParts.push({ label: 'user messages', text: String(item.payload.message || '') }); + events.push({ id, kind: 'user_message', summary, timeLabel, chips: ['user'] }); + detailEntries.push(createDetailEntry({ + id: id + '-detail', + kind: 'user_message', + kindLabel: 'User', + summary, + timeLabel, + chips: ['user'], + sections: buildRichTextSections(item.payload.message, { textLabel: 'Prompt' }) + })); + return; + } + if (item.type === 'event_msg' && item.payload && item.payload.type === 'agent_message') { + thoughtOrder += 1; + const textBody = shortMultiline(item.payload.message, 520); + thoughtEntries.push({ + id, + order: thoughtOrder, + text: textBody, + timeLabel, + chips: [item.payload.phase || 'commentary'], + sections: buildRichTextSections(item.payload.message, { textLabel: 'Reasoning', detectCode: true }) + }); + estimateParts.push({ label: 'agent messages', text: String(item.payload.message || '') }); + appendPythonScripts(pythonScripts, pythonSeen, item.payload.message, { + idPrefix: side + '-codex-python', + timeLabel, + sourceLabel: 'agent message', + chips: [item.payload.phase || 'commentary'] + }); + events.push({ id, kind: 'thought', summary: textBody, timeLabel, chips: [item.payload.phase || 'commentary'] }); + detailEntries.push(createDetailEntry({ + id: id + '-detail', + kind: 'thought', + kindLabel: 'Thought', + summary: textBody, + timeLabel, + chips: [item.payload.phase || 'commentary'], + sections: buildRichTextSections(item.payload.message, { textLabel: 'Reasoning', detectCode: true }) + })); + if (/final|answer|result/i.test(item.payload.phase || '') && textBody) { + finalResponses.push({ id: id + '-final', order: finalResponses.length + 1, text: textBody, timeLabel }); + } + return; + } + if (item.type === 'event_msg' && item.payload && item.payload.type === 'token_count') { + const total = item.payload.info && item.payload.info.total_token_usage ? item.payload.info.total_token_usage : {}; + events.push({ + id, + kind: 'token_snapshot', + summary: 'total=' + numberFormat(total.total_tokens) + ', input=' + numberFormat(total.input_tokens) + ', output=' + numberFormat(total.output_tokens), + timeLabel, + chips: ['tokens'] + }); + return; + } + if (item.type === 'response_item' && item.payload && item.payload.type === 'function_call') { + const args = parseArguments(item.payload.arguments); + const tool = item.payload.name || 'unknown'; + const command = args.cmd || args.command || args.bash_command || ''; + const payloadText = extractModelToolPayload(args, command); + callById.set(item.payload.call_id, { tool, args, command }); + if (command) { + commandOrder += 1; + const run = { + id: item.payload.call_id, + order: commandOrder, + command, + primaryCommand: primaryCommand(command), + channel: tool, + timeLabel, + exitCode: null, + outputPreview: '', + sessionId: '', + status: 'started', + statusClass: '', + outputText: '', + sections: buildCommandSections(command, tool) + }; + commandRuns.push(run); + runByCallId.set(item.payload.call_id, run); + } + if (payloadText) { + estimateParts.push({ label: 'model-emitted tool payloads', text: payloadText }); + appendPythonScripts(pythonScripts, pythonSeen, payloadText, { + idPrefix: side + '-codex-python', + timeLabel, + sourceLabel: tool, + chips: [tool] + }); + } + events.push({ + id, + kind: 'tool_call', + summary: command ? shortMultiline(command, 420) : shortText(JSON.stringify(args), 240), + timeLabel, + chips: command ? [tool, primaryCommand(command)] : [tool] + }); + detailEntries.push(createDetailEntry({ + id: id + '-detail', + kind: 'tool_call', + kindLabel: 'Tool Call', + summary: command ? shortMultiline(command, 420) : shortText(JSON.stringify(args), 240), + timeLabel, + chips: command ? [tool, primaryCommand(command)] : [tool], + sections: command + ? buildCommandSections(command, tool) + : [makeCodeSection(JSON.stringify(args, null, 2), 'json', 'Arguments')] + })); + return; + } + if (item.type === 'response_item' && item.payload && item.payload.type === 'function_call_output') { + const call = callById.get(item.payload.call_id) || {}; + const outputText = Array.isArray(item.payload.output) ? JSON.stringify(item.payload.output) : String(item.payload.output || ''); + const exitCode = parseExitCode(outputText); + const sessionId = parseSessionId(outputText); + let run = runByCallId.get(item.payload.call_id) || null; + if (!run && call.tool === 'write_stdin' && call.args && call.args.session_id) { + run = runBySession.get(String(call.args.session_id)) || null; + } + if (run) { + if (sessionId && !run.sessionId) { + run.sessionId = sessionId; + runBySession.set(sessionId, run); + } + if (exitCode !== null) { + run.exitCode = exitCode; + } + if (outputText) { + run.outputPreview = shortMultiline(outputText, 520); + run.outputText = outputText; + run.sections = buildCommandSections(run.command, run.channel, outputText); + } + const state = statusFromExit(run.exitCode, sessionId, outputText, false); + run.status = state.label; + run.statusClass = state.css; + } + if (outputText) { + estimateParts.push({ label: 'tool outputs', text: outputText }); + } + events.push({ + id, + kind: 'tool_result', + summary: shortMultiline(outputText, 460), + timeLabel, + chips: [call.tool || 'tool_result'].concat(exitCode !== null ? ['exit ' + exitCode] : sessionId ? ['session ' + sessionId] : []) + }); + detailEntries.push(createDetailEntry({ + id: id + '-detail', + kind: 'tool_result', + kindLabel: 'Tool Result', + summary: shortMultiline(outputText, 460), + timeLabel, + chips: [call.tool || 'tool_result'].concat(exitCode !== null ? ['exit ' + exitCode] : sessionId ? ['session ' + sessionId] : []), + sections: [makeCodeSection(outputText, inferCodeLanguage(outputText), 'Output')] + })); + } + }); + commandRuns.forEach((run) => { + if (run.status === 'started') { + const state = statusFromExit(run.exitCode, '', run.outputPreview, false); + run.status = state.label; + run.statusClass = state.css; + } + }); + const channelFamilies = toBarRows([ + { label: 'thought entries', value: thoughtEntries.length }, + { label: 'command runs', value: commandRuns.length }, + { label: 'tool results', value: events.filter((event) => event.kind === 'tool_result').length }, + { label: 'token snapshots', value: events.filter((event) => event.kind === 'token_snapshot').length }, + { label: 'final responses', value: finalResponses.length } + ]); + return finalizeTrace({ + side, + kind: 'codex_jsonl', + formatLabel: 'Codex JSONL', + description: 'Codex CLI session JSONL with user, agent, tool, and token events.', + fileName, + filePath, + taskPrompt: userPrompt, + events, + thoughtEntries, + commandRuns, + finalResponses, + pythonScripts, + detailEntries, + exactTokens: latestTokens, + tokenSourceLabel: latestTokens + ? 'Exact tokens from token_count snapshot.' + : 'Estimated from extracted trace text.', + tokenEstimateParts: estimateParts, + countedFieldLabels: [ + 'event_msg.user_message.message', + 'event_msg.agent_message.message', + 'response_item.function_call.arguments.(cmd|command|bash_command)', + 'response_item.function_call_output.output' + ], + metrics: [ + { label: 'Events', value: numberFormat(events.length), sub: 'normalized event records' }, + { label: 'Thoughts', value: numberFormat(thoughtEntries.length), sub: 'agent commentary entries' }, + { label: 'Commands', value: numberFormat(commandRuns.length), sub: 'shell commands reconstructed from tool calls' }, + { label: 'Token source', value: latestTokens ? 'exact' : 'estimate', sub: latestTokens ? 'token_count snapshot present' : 'fallback to tokenizer estimate' } + ], + channelFamilies + }); +} + +function normalizeRawResponses(text, fileName, filePath, side) { + const lines = text.split(/\r?\n/).filter(Boolean); + const events = []; + const thoughtEntries = []; + const commandRuns = []; + const finalResponses = []; + const pythonScripts = []; + const pythonSeen = new Set(); + const detailEntries = []; + const estimateParts = []; + let thoughtOrder = 0; + let commandOrder = 0; + let finalOrder = 0; + let extractedFrames = 0; + let emptyFrames = 0; + let taskPrompt = ''; + lines.forEach((line, lineIndex) => { + const outer = safeJsonParse(line); + if (!outer || typeof outer !== 'object') { + return; + } + const timeLabel = formatTimestamp(outer.timestamp || ''); + if (outer.event !== 'raw_text') { + return; + } + const frames = splitConcatenatedJsonObjects(outer.raw_text); + frames.forEach((frame, frameIndex) => { + const idBase = side + '-raw-' + lineIndex + '-' + frameIndex; + const actionPayload = extractActionPayload(frame); + const hasThought = !!String(frame.thought || '').trim(); + const hasCommand = !!actionPayload.text; + const hasFinal = !!String(frame.final_response || '').trim(); + const frameSections = []; + const frameChips = ['attempt ' + (outer.attempt || 'n/a')]; + if (!hasThought && !hasCommand && !hasFinal) { + emptyFrames += 1; + return; + } + extractedFrames += 1; + if (hasThought) { + thoughtOrder += 1; + const textBody = shortMultiline(frame.thought, 520); + thoughtEntries.push({ + id: idBase + '-thought', + order: thoughtOrder, + text: textBody, + timeLabel, + chips: ['attempt ' + (outer.attempt || 'n/a')], + sections: buildRichTextSections(frame.thought, { textLabel: 'Reasoning', detectCode: true }) + }); + estimateParts.push({ label: 'thought fields', text: String(frame.thought || '') }); + appendPythonScripts(pythonScripts, pythonSeen, frame.thought, { + idPrefix: side + '-raw-python', + timeLabel, + sourceLabel: 'thought field', + chips: ['attempt ' + (outer.attempt || 'n/a')] + }); + events.push({ id: idBase + '-thought-event', kind: 'thought', summary: textBody, timeLabel, chips: ['attempt ' + (outer.attempt || 'n/a')] }); + if (!taskPrompt && /task|search|goal/i.test(textBody)) { + taskPrompt = textBody; + } + frameSections.push(...buildRichTextSections(frame.thought, { textLabel: 'Reasoning', detectCode: true })); + } + if (hasCommand) { + commandOrder += 1; + const command = actionPayload.text; + const state = statusFromExit(null, '', '', frame.done === true); + commandRuns.push({ + id: idBase + '-command', + order: commandOrder, + command, + primaryCommand: primaryCommand(command), + channel: actionPayload.channel, + timeLabel, + exitCode: null, + outputPreview: hasFinal ? shortMultiline(frame.final_response, 320) : '', + outputText: hasFinal ? String(frame.final_response || '').trim() : '', + status: state.label, + statusClass: state.css, + sections: buildCommandSections(command, actionPayload.channel, hasFinal ? String(frame.final_response || '') : '', hasFinal ? 'Result' : 'Output') + }); + estimateParts.push({ label: actionPayload.estimateLabel, text: command }); + appendPythonScripts(pythonScripts, pythonSeen, command, { + idPrefix: side + '-raw-python', + timeLabel, + sourceLabel: actionPayload.field || actionPayload.channel || 'command', + chips: ['attempt ' + (outer.attempt || 'n/a')] + }); + events.push({ id: idBase + '-command-event', kind: 'command', summary: shortMultiline(command, 460), timeLabel, chips: ['attempt ' + (outer.attempt || 'n/a'), primaryCommand(command)] }); + frameSections.push(makeCodeSection(command, actionPayload.channel, 'Command')); + frameChips.push(primaryCommand(command)); + } + if (hasFinal) { + finalOrder += 1; + const responseText = shortMultiline(frame.final_response, 520); + finalResponses.push({ id: idBase + '-final', order: finalOrder, text: responseText, timeLabel }); + estimateParts.push({ label: 'final_response fields', text: String(frame.final_response || '') }); + appendPythonScripts(pythonScripts, pythonSeen, frame.final_response, { + idPrefix: side + '-raw-python', + timeLabel, + sourceLabel: 'final_response', + chips: ['done=' + Boolean(frame.done)] + }); + events.push({ id: idBase + '-final-event', kind: 'final_response', summary: responseText, timeLabel, chips: ['done=' + Boolean(frame.done)] }); + frameSections.push(...buildRichTextSections(frame.final_response, { textLabel: 'Response', detectCode: true })); + frameChips.push('done=' + Boolean(frame.done)); + } + detailEntries.push(createDetailEntry({ + id: idBase + '-frame-detail', + kind: 'frame', + kindLabel: 'Step', + summary: shortMultiline( + actionPayload.text || String(frame.final_response || '') || String(frame.thought || ''), + 460 + ), + timeLabel, + chips: frameChips, + sections: frameSections + })); + }); + }); + const channelFamilies = toBarRows([ + { label: 'thought entries', value: thoughtEntries.length }, + { label: 'command frames', value: commandRuns.length }, + { label: 'final responses', value: finalResponses.length }, + { label: 'outer lines', value: lines.length }, + { label: 'empty frames dropped', value: emptyFrames } + ]); + return finalizeTrace({ + side, + kind: 'raw_response_jsonl', + formatLabel: 'Webwright Responses JSONL', + description: 'Webwright raw_responses.jsonl with embedded thought, bash, and final-response frames.', + fileName, + filePath, + taskPrompt, + events, + thoughtEntries, + commandRuns, + finalResponses, + pythonScripts, + detailEntries, + tokenSourceLabel: 'Estimated from extracted trace text.', + tokenEstimateParts: estimateParts, + countedFieldLabels: ['raw_text frame.thought', 'raw_text frame.bash_command', 'raw_text frame.python_code', 'raw_text frame.final_response'], + metrics: [ + { label: 'Outer lines', value: numberFormat(lines.length), sub: 'top-level raw_text records' }, + { label: 'Frames', value: numberFormat(extractedFrames), sub: 'embedded JSON frames extracted from raw_text' }, + { label: 'Thoughts', value: numberFormat(thoughtEntries.length), sub: 'inner thought strings' }, + { label: 'Commands', value: numberFormat(commandRuns.length), sub: 'inner action strings' } + ], + channelFamilies + }); +} + +function normalizeTrajectory(text, fileName, filePath, side) { + const parsed = safeJsonParse(text); + if (!parsed || typeof parsed !== 'object' || !Array.isArray(parsed.messages)) { + throw new Error('Invalid trajectory.json structure'); + } + const events = []; + const thoughtEntries = []; + const commandRuns = []; + const finalResponses = []; + const pythonScripts = []; + const pythonSeen = new Set(); + const detailEntries = []; + const estimateParts = []; + let latestTokens = null; + let taskPrompt = ''; + let thoughtOrder = 0; + let commandOrder = 0; + let finalOrder = 0; + let pendingRun = null; + parsed.messages.forEach((message, index) => { + const id = side + '-trajectory-' + index; + const timeLabel = 'step ' + (index + 1); + const role = message.role || 'unknown'; + const contentText = contentToText(message.content); + if (role === 'user') { + const observation = normalizeObservation(message); + if (observation && pendingRun) { + if (observation.command && !pendingRun.command) { + pendingRun.command = observation.command; + pendingRun.primaryCommand = primaryCommand(observation.command); + } + pendingRun.exitCode = observation.exitCode; + pendingRun.outputPreview = shortMultiline(observation.output, 520); + pendingRun.outputText = observation.output; + const state = statusFromExit(observation.exitCode, '', observation.output, false); + pendingRun.status = state.label; + pendingRun.statusClass = state.css; + pendingRun.sections = buildCommandSections(pendingRun.command, pendingRun.channel, observation.output); + estimateParts.push({ label: 'observation outputs', text: observation.output }); + events.push({ id, kind: 'observation', summary: shortMultiline(observation.output, 460), timeLabel, chips: observation.exitCode !== null ? ['exit ' + observation.exitCode] : ['observation'] }); + detailEntries.push(createDetailEntry({ + id: id + '-observation-detail', + kind: 'observation', + kindLabel: 'Observation', + summary: shortMultiline(observation.output, 460), + timeLabel, + chips: observation.exitCode !== null ? ['exit ' + observation.exitCode] : ['observation'], + sections: [makeCodeSection(observation.output, inferCodeLanguage(observation.output), 'Output')] + })); + } else if (contentText) { + if (!taskPrompt) { + taskPrompt = shortMultiline(contentText, 360); + } + estimateParts.push({ label: 'user messages', text: contentText }); + events.push({ id, kind: 'user_message', summary: shortMultiline(contentText, 420), timeLabel, chips: ['user'] }); + detailEntries.push(createDetailEntry({ + id: id + '-detail', + kind: 'user_message', + kindLabel: 'User', + summary: shortMultiline(contentText, 420), + timeLabel, + chips: ['user'], + sections: buildRichTextSections(contentText, { textLabel: 'Prompt' }) + })); + } + return; + } + if (role !== 'assistant') { + return; + } + const extra = message.extra || {}; + const raw = extra.raw_response && typeof extra.raw_response === 'object' ? extra.raw_response : {}; + const usage = extra.usage && typeof extra.usage === 'object' ? (extra.usage.cumulative_response || extra.usage.last_response || extra.usage) : null; + if (usage && typeof usage.total_tokens === 'number') { + latestTokens = usage; + } + const thoughtText = String(raw.thought || contentText || '').trim(); + const action = Array.isArray(extra.actions) && extra.actions.length ? extra.actions[0] : null; + const actionPayload = extractActionPayload(raw, action); + const commandText = actionPayload.text; + const finalText = String(raw.final_response || '').trim(); + const done = raw.done === true || extra.done === true; + if (thoughtText) { + thoughtOrder += 1; + estimateParts.push({ label: 'assistant thoughts', text: thoughtText }); + appendPythonScripts(pythonScripts, pythonSeen, thoughtText, { + idPrefix: side + '-trajectory-python', + timeLabel, + sourceLabel: 'assistant thought', + chips: [done ? 'done' : 'assistant'] + }); + thoughtEntries.push({ + id: id + '-thought', + order: thoughtOrder, + text: shortMultiline(thoughtText, 520), + timeLabel, + chips: [done ? 'done' : 'assistant'], + sections: buildRichTextSections(thoughtText, { textLabel: 'Reasoning', detectCode: true }) + }); + events.push({ id: id + '-thought-event', kind: 'thought', summary: shortMultiline(thoughtText, 460), timeLabel, chips: [done ? 'done' : 'assistant'] }); + detailEntries.push(createDetailEntry({ + id: id + '-thought-detail', + kind: 'thought', + kindLabel: 'Thought', + summary: shortMultiline(thoughtText, 460), + timeLabel, + chips: [done ? 'done' : 'assistant'], + sections: buildRichTextSections(thoughtText, { textLabel: 'Reasoning', detectCode: true }) + })); + } + if (commandText) { + commandOrder += 1; + const run = { + id: id + '-command', + order: commandOrder, + command: commandText, + primaryCommand: primaryCommand(commandText), + channel: actionPayload.channel, + timeLabel, + exitCode: null, + outputPreview: '', + outputText: '', + status: 'started', + statusClass: '', + sections: buildCommandSections(commandText, actionPayload.channel) + }; + pendingRun = run; + commandRuns.push(run); + estimateParts.push({ label: actionPayload.estimateLabel || 'assistant commands', text: commandText }); + appendPythonScripts(pythonScripts, pythonSeen, commandText, { + idPrefix: side + '-trajectory-python', + timeLabel, + sourceLabel: actionPayload.field || actionPayload.channel || 'assistant command', + chips: ['assistant', primaryCommand(commandText)] + }); + events.push({ id: id + '-command-event', kind: 'command', summary: shortMultiline(commandText, 460), timeLabel, chips: ['assistant', primaryCommand(commandText)] }); + detailEntries.push(createDetailEntry({ + id: id + '-command-detail', + kind: 'command', + kindLabel: 'Command', + summary: shortMultiline(commandText, 460), + timeLabel, + chips: ['assistant', primaryCommand(commandText)], + sections: buildCommandSections(commandText, actionPayload.channel) + })); + } + if (finalText) { + finalOrder += 1; + estimateParts.push({ label: 'assistant final responses', text: finalText }); + appendPythonScripts(pythonScripts, pythonSeen, finalText, { + idPrefix: side + '-trajectory-python', + timeLabel, + sourceLabel: 'assistant final response', + chips: ['done=' + done] + }); + finalResponses.push({ id: id + '-final', order: finalOrder, text: shortMultiline(finalText, 520), timeLabel }); + events.push({ id: id + '-final-event', kind: 'final_response', summary: shortMultiline(finalText, 460), timeLabel, chips: ['done=' + done] }); + detailEntries.push(createDetailEntry({ + id: id + '-final-detail', + kind: 'final_response', + kindLabel: 'Final Response', + summary: shortMultiline(finalText, 460), + timeLabel, + chips: ['done=' + done], + sections: buildRichTextSections(finalText, { textLabel: 'Response', detectCode: true }) + })); + } + }); + commandRuns.forEach((run) => { + if (run.status === 'started') { + const state = statusFromExit(run.exitCode, '', run.outputPreview, false); + run.status = state.label; + run.statusClass = state.css; + } + }); + return finalizeTrace({ + side, + kind: 'trajectory_json', + formatLabel: 'Trajectory JSON', + description: 'Webwright trajectory.json transcript with assistant actions, observations, and usage snapshots.', + fileName, + filePath, + taskPrompt, + events, + thoughtEntries, + commandRuns, + finalResponses, + pythonScripts, + detailEntries, + exactTokens: latestTokens, + tokenSourceLabel: latestTokens + ? 'Exact tokens from usage snapshot.' + : 'Estimated from extracted trace text.', + tokenEstimateParts: estimateParts, + countedFieldLabels: [ + 'messages[].content', + 'messages[].extra.raw_response.thought', + 'messages[].extra.raw_response.bash_command', + 'messages[].extra.raw_response.python_code', + 'messages[].extra.raw_response.final_response', + 'messages[].extra.observation.command_output' + ], + metrics: [ + { label: 'Messages', value: numberFormat(parsed.messages.length), sub: 'top-level transcript turns' }, + { label: 'Thoughts', value: numberFormat(thoughtEntries.length), sub: 'assistant reasoning entries' }, + { label: 'Commands', value: numberFormat(commandRuns.length), sub: 'assistant action commands' }, + { label: 'API calls', value: numberFormat(parsed.info && parsed.info.api_calls), sub: 'info.api_calls from trajectory metadata' } + ] + }); +} + +function normalizeCopilotSessionMarkdown(text, fileName, filePath, side) { + const sections = parseMarkdownSections(text); + const metadata = extractSessionMetadata(text); + const events = []; + const thoughtEntries = []; + const commandRuns = []; + const finalResponses = []; + const pythonScripts = []; + const pythonSeen = new Set(); + const detailEntries = []; + const estimateParts = []; + let taskPrompt = ''; + let thoughtOrder = 0; + let commandOrder = 0; + let eventOrder = 0; + if (metadata.sessionId || metadata.duration || metadata.started || metadata.exported) { + const summary = [ + metadata.sessionId ? 'session ' + metadata.sessionId : '', + metadata.started ? 'started ' + metadata.started : '', + metadata.duration ? 'duration ' + metadata.duration : '', + metadata.exported ? 'exported ' + metadata.exported : '' + ].filter(Boolean).join(' | '); + events.push({ + id: side + '-session-meta', + kind: 'info', + summary, + timeLabel: 'session', + chips: ['session metadata'] + }); + eventOrder += 1; + } + sections.forEach((section) => { + if (section.type === 'user') { + const clean = stripSessionMarkup(section.body); + const textBody = shortMultiline(clean, 520); + if (!taskPrompt && textBody) { + taskPrompt = textBody; + } + if (clean) { + estimateParts.push({ label: 'user sections', text: clean }); + events.push({ id: side + '-session-user-' + eventOrder, kind: 'user_message', summary: textBody, timeLabel: section.timeLabel, chips: ['user'] }); + detailEntries.push(createDetailEntry({ + id: side + '-session-user-detail-' + eventOrder, + kind: 'user_message', + kindLabel: 'User', + summary: textBody, + timeLabel: section.timeLabel, + chips: ['user'], + sections: buildRichTextSections(clean, { textLabel: 'Prompt', detectCode: true }) + })); + eventOrder += 1; + } + return; + } + if (section.type === 'copilot') { + const clean = stripSessionMarkup(section.body); + if (!clean) { + return; + } + thoughtOrder += 1; + estimateParts.push({ label: 'copilot sections', text: clean }); + appendPythonScripts(pythonScripts, pythonSeen, clean, { + idPrefix: side + '-session-python', + timeLabel: section.timeLabel, + sourceLabel: 'copilot section', + chips: ['copilot'] + }); + thoughtEntries.push({ + id: side + '-session-thought-' + thoughtOrder, + order: thoughtOrder, + text: shortMultiline(clean, 520), + timeLabel: section.timeLabel, + chips: ['copilot'], + sections: buildRichTextSections(clean, { textLabel: 'Narrative', detectCode: true }) + }); + events.push({ id: side + '-session-copilot-' + eventOrder, kind: 'thought', summary: shortMultiline(clean, 460), timeLabel: section.timeLabel, chips: ['copilot'] }); + detailEntries.push(createDetailEntry({ + id: side + '-session-copilot-detail-' + eventOrder, + kind: 'thought', + kindLabel: 'Copilot', + summary: shortMultiline(clean, 460), + timeLabel: section.timeLabel, + chips: ['copilot'], + sections: buildRichTextSections(clean, { textLabel: 'Narrative', detectCode: true }) + })); + eventOrder += 1; + if (/final|done|complete|result/i.test(clean)) { + finalResponses.push({ id: side + '-session-final-' + finalResponses.length, order: finalResponses.length + 1, text: shortMultiline(clean, 520), timeLabel: section.timeLabel }); + } + return; + } + if (section.type === 'tool') { + commandOrder += 1; + const parsed = parseSessionToolSection(section, side, commandOrder, eventOrder); + commandRuns.push(parsed.run); + parsed.events.forEach((event) => events.push(event)); + parsed.detailEntries.forEach((entry) => detailEntries.push(entry)); + estimateParts.push(...parsed.estimateParts); + parsed.pythonScripts.forEach((script) => { + if (pythonSeen.has(script.text)) { + return; + } + pythonSeen.add(script.text); + pythonScripts.push(Object.assign({}, script, { + id: side + '-session-python-' + (pythonScripts.length + 1), + order: pythonScripts.length + 1 + })); + }); + eventOrder += parsed.events.length; + return; + } + const infoText = stripSessionMarkup(section.body); + if (infoText) { + events.push({ id: side + '-session-info-' + eventOrder, kind: 'info', summary: shortMultiline(infoText, 420), timeLabel: section.timeLabel, chips: ['info'] }); + eventOrder += 1; + } + }); + const channelFamilies = toBarRows([ + { label: 'user sections', value: events.filter((event) => event.kind === 'user_message').length }, + { label: 'copilot sections', value: thoughtEntries.length }, + { label: 'tool sections', value: commandRuns.length }, + { label: 'info sections', value: events.filter((event) => event.kind === 'info').length } + ]); + return finalizeTrace({ + side, + kind: 'copilot_session_md', + formatLabel: 'Copilot Session MD', + description: 'GitHub Copilot CLI markdown session export with user, copilot, and tool sections.', + fileName, + filePath, + taskPrompt, + events, + thoughtEntries, + commandRuns, + finalResponses, + pythonScripts, + detailEntries, + tokenSourceLabel: 'Estimated from extracted trace text.', + tokenEstimateParts: estimateParts, + countedFieldLabels: ['session header metadata', '### User body text', '### Copilot body text', 'tool titles', '### โœ… tool Arguments blocks', '### โœ… tool commands / outputs'], + metrics: [ + { label: 'Sections', value: numberFormat(sections.length), sub: 'top-level markdown sections' }, + { label: 'Thoughts', value: numberFormat(thoughtEntries.length), sub: 'copilot narrative blocks' }, + { label: 'Commands', value: numberFormat(commandRuns.length), sub: 'tool sections mapped to comparable command rows' }, + { label: 'Finals', value: numberFormat(finalResponses.length), sub: 'copilot blocks that look like terminal summaries' }, + { label: 'Session duration', value: metadata.duration || 'n/a', sub: metadata.started ? 'started ' + metadata.started : 'header metadata if present' } + ], + channelFamilies + }); +} + +function normalizeTrace(text, fileName, filePath, side) { + const format = detectFormat(text); + if (!format) { + throw new Error('Supported inputs: Codex JSONL, Webwright Responses JSONL (raw_responses.jsonl), trajectory.json, and Copilot session markdown.'); + } + if (format === 'codex_jsonl') { + return normalizeCodex(text, fileName, filePath, side); + } + if (format === 'raw_response_jsonl') { + return normalizeRawResponses(text, fileName, filePath, side); + } + if (format === 'trajectory_json') { + return normalizeTrajectory(text, fileName, filePath, side); + } + return normalizeCopilotSessionMarkdown(text, fileName, filePath, side); +} + +const app = createApp({ + data() { + return { + activeView: 'summary', + traceTab: 'python', + dragTarget: null, + left: null, + right: null, + errors: { left: '', right: '' }, + attachmentErrors: { left: '', right: '' }, + tokenTarget: 'model:o3', + tokenizerStatus: 'Ready.' + }; + }, + computed: { + slots() { + return [ + { key: 'left', title: 'Trace A', subtitle: 'Load Codex JSONL, Webwright Responses JSONL (raw_responses.jsonl), trajectory.json, or Copilot session markdown on this side.' }, + { key: 'right', title: 'Trace B', subtitle: 'Load a second trace and compare thoughts, commands, token source, and extracted outputs.' } + ]; + }, + tokenizerOptions() { + return [ + { value: 'model:o3', label: 'o3' }, + { value: 'model:o3-mini', label: 'o3-mini' }, + { value: 'model:gpt-4o', label: 'gpt-4o' }, + { value: 'encoding:o200k_base', label: 'o200k_base' }, + { value: 'encoding:cl100k_base', label: 'cl100k_base' } + ]; + }, + bothLoaded() { + return !!(this.left && this.right); + }, + hasAnyTrace() { + return !!(this.left || this.right); + }, + loadedTraces() { + return [this.left, this.right].filter(Boolean); + }, + leftTrace() { + return this.left; + }, + rightTrace() { + return this.right; + }, + detailTabs() { + return [ + { label: 'Python Scripts', value: 'python' }, + { label: 'Thoughts', value: 'thoughts' }, + { label: 'Commands', value: 'commands' }, + { label: 'All', value: 'all' } + ]; + }, + comparisonRows() { + if (!this.bothLoaded) { + return []; + } + return [ + { label: 'Format', left: this.left.formatLabel, right: this.right.formatLabel }, + { label: 'Thought entries', left: numberFormat(this.left.thoughtEntries.length), right: numberFormat(this.right.thoughtEntries.length) }, + { label: 'Command runs', left: numberFormat(this.left.commandRuns.length), right: numberFormat(this.right.commandRuns.length) }, + { label: 'Final responses', left: numberFormat(this.left.finalResponses.length), right: numberFormat(this.right.finalResponses.length) }, + { label: 'Python scripts', left: numberFormat(this.left.pythonScripts.length), right: numberFormat(this.right.pythonScripts.length) }, + { label: 'Token total', left: this.tokenTotalDisplay(this.left), right: this.tokenTotalDisplay(this.right) }, + { label: 'Token source', left: this.left.tokenModeLabel, right: this.right.tokenModeLabel }, + { label: 'Counted fields', left: this.left.countedFieldLabels.join(', '), right: this.right.countedFieldLabels.join(', ') }, + { + label: 'Top command families', + left: this.left.commandFamilies.map((row) => row.label + ' ' + row.value).join(', '), + right: this.right.commandFamilies.map((row) => row.label + ' ' + row.value).join(', ') + }, + { label: 'Task hint', left: this.left.taskPrompt || 'n/a', right: this.right.taskPrompt || 'n/a' } + ]; + }, + tokenizerSelectionLabel() { + const selection = getTokenizerSelection(this.tokenTarget); + return selection.mode === 'model' ? selection.name + ' -> ' + selection.encoding : selection.encoding; + } + }, + watch: { + tokenTarget() { + this.refreshAllTokens(); + } + }, + mounted() { + this.highlightRenderedCode(); + }, + updated() { + this.highlightRenderedCode(); + }, + methods: { + codeClass(language) { + return 'language-' + (normalizeCodeLanguage(language) || 'plaintext'); + }, + shouldShowEntrySummary(entry) { + if (!entry || !String(entry.summary || '').trim()) { + return false; + } + if (entry.kind === 'frame' && Array.isArray(entry.sections) && entry.sections.length) { + return false; + } + const sections = Array.isArray(entry.sections) ? entry.sections : []; + if (sections.length !== 1) { + return true; + } + const onlySection = sections[0]; + if (!onlySection || onlySection.type !== 'text') { + return true; + } + return shortMultiline(onlySection.text, 460) !== shortMultiline(entry.summary, 460); + }, + traceBySide(side) { + return this[side]; + }, + errorBySide(side) { + return this.errors[side]; + }, + attachmentErrorBySide(side) { + return this.attachmentErrors[side]; + }, + clearTrace(side) { + this[side] = null; + this.errors[side] = ''; + this.attachmentErrors[side] = ''; + }, + clearCompanion(side) { + const trace = this[side]; + if (!trace || trace.kind !== 'raw_response_jsonl') { + return; + } + this.attachmentErrors[side] = ''; + this[side] = stripTrajectoryCompanion(trace); + }, + tokenTotalDisplay(trace) { + if (!trace) { + return 'n/a'; + } + if (trace.exactTokens && typeof trace.exactTokens.total_tokens === 'number') { + return 'Exact tokens: ' + numberFormat(trace.exactTokens.total_tokens); + } + if (typeof trace.tokenEstimateTotal === 'number') { + return 'Est. output tokens: ~' + numberFormat(trace.tokenEstimateTotal); + } + if (trace.tokenEstimateError) { + return 'Est. output tokens failed'; + } + return 'Estimating output tokens...'; + }, + tokenNote(trace) { + if (!trace) { + return ''; + } + return trace.tokenSourceLabel; + }, + async handleFileInput(side, event) { + const file = event.target.files && event.target.files[0]; + if (!file) { + return; + } + await this.loadFile(side, file); + event.target.value = ''; + }, + async handleDrop(side, event) { + this.dragTarget = null; + const file = event.dataTransfer && event.dataTransfer.files && event.dataTransfer.files[0]; + if (!file) { + return; + } + await this.loadFile(side, file); + }, + async handleCompanionInput(side, event) { + const file = event.target.files && event.target.files[0]; + if (!file) { + return; + } + this.attachmentErrors[side] = ''; + try { + const trace = this[side]; + if (!trace || trace.kind !== 'raw_response_jsonl') { + throw new Error('Attach trajectory.json only to a Webwright Responses JSONL trace (raw_responses.jsonl).'); + } + const text = await file.text(); + const companion = extractTrajectoryUsage(text, file.name); + const updatedTrace = applyTrajectoryCompanion(trace, companion); + this[side] = updatedTrace; + this.enrichTraceTokens(updatedTrace); + } catch (error) { + this.attachmentErrors[side] = error && error.message ? error.message : 'Failed to attach trajectory.json'; + } finally { + event.target.value = ''; + } + }, + async enrichTraceTokens(trace) { + const selection = getTokenizerSelection(this.tokenTarget); + trace.tokenSelectionLabel = selection.mode === 'model' ? selection.name + ' / ' + selection.encoding : selection.encoding; + trace.tokenEstimateError = ''; + if (!trace.tokenEstimateParts.length) { + trace.tokenBasisRows = []; + if (!trace.exactTokens) { + trace.tokenRows = [{ label: 'availability', value: 'no text fields to estimate' }]; + } + return trace; + } + const quickBasisRows = trace.tokenEstimateParts + .map((part) => ({ label: part.label, value: quickTokenEstimate(part.text) })) + .filter((row) => row.value > 0); + trace.tokenEstimateTotal = quickBasisRows.reduce((sum, row) => sum + row.value, 0); + if (!trace.exactTokens) { + trace.tokenRows = [ + { label: 'basis', value: 'estimated' }, + { label: 'encoding', value: selection.encoding + ' pending' }, + { label: 'est. output tokens', value: '~' + numberFormat(trace.tokenEstimateTotal) } + ]; + } + try { + this.tokenizerStatus = 'Counting tokens with ' + selection.encoding + '...'; + const encoder = await loadEncoder(selection.encoding); + const basisRows = trace.tokenEstimateParts.map((part) => ({ label: part.label, value: encoder.encode(part.text).length })).filter((row) => row.value > 0); + trace.tokenBasisRows = toBarRows(basisRows, 8); + trace.tokenEstimateTotal = basisRows.reduce((sum, row) => sum + row.value, 0); + trace.tokenModeLabel = trace.exactTokens ? 'exact' : 'estimate'; + trace.tokenRows = trace.exactTokens + ? tokenRowsFromTotals(trace.exactTokens) + : [ + { label: 'basis', value: 'estimated' }, + { label: 'encoding', value: selection.encoding }, + { label: 'est. output tokens', value: '~' + numberFormat(trace.tokenEstimateTotal) } + ]; + this.tokenizerStatus = 'Ready.'; + } catch (error) { + trace.tokenEstimateError = error && error.message ? error.message : 'Failed to estimate tokens'; + trace.tokenBasisRows = toBarRows(quickBasisRows, 8); + if (!trace.exactTokens) { + trace.tokenRows = [ + { label: 'basis', value: 'estimated' }, + { label: 'encoding', value: 'quick fallback' }, + { label: 'est. output tokens', value: '~' + numberFormat(trace.tokenEstimateTotal) } + ]; + } + this.tokenizerStatus = trace.tokenEstimateError + '. Showing quick estimate.'; + } + return trace; + }, + async refreshAllTokens() { + const traces = [this.left, this.right].filter(Boolean); + for (const trace of traces) { + await this.enrichTraceTokens(trace); + } + }, + async loadFile(side, file) { + this.errors[side] = ''; + this.attachmentErrors[side] = ''; + try { + const text = await file.text(); + const trace = normalizeTrace(text, file.name, file.path || '', side); + this[side] = trace; + this.enrichTraceTokens(trace); + } catch (error) { + this[side] = null; + this.errors[side] = error && error.message ? error.message : 'Failed to parse file'; + } + }, + filteredEvents(trace) { + if (!trace) { + return []; + } + return trace.events.filter((event) => event.kind !== 'token_snapshot'); + }, + highlightRenderedCode() { + if (!window.hljs) { + return; + } + this.$nextTick(() => { + document.querySelectorAll('#app pre code').forEach((node) => { + if (node.dataset.highlighted === 'yes') { + delete node.dataset.highlighted; + } + window.hljs.highlightElement(node); + }); + }); + } + } +}); + +window.__traceJsonlCompareVm = app.mount('#app'); diff --git a/assets/compare_trajectory/index.html b/assets/compare_trajectory/index.html new file mode 100644 index 0000000..edf3de3 --- /dev/null +++ b/assets/compare_trajectory/index.html @@ -0,0 +1,387 @@ + + + + + + Trace JSONL Compare + + + + + + + +
+
+
+
+

Webwright Trajectory Comparison

+
+ Compare Codex JSONL, Webwright raw and trajectory traces, and Copilot session markdown +
+
+ +
+
+ +
+
+
+
+

{{ slot.title }}

+
{{ slot.subtitle }}
+
+
+ {{ traceBySide(slot.key).formatLabel }} + commands {{ traceBySide(slot.key).summary.commandRuns }} +
+
+ +
+ + +
+ {{ errorBySide(slot.key) }} +
+
+ {{ attachmentErrorBySide(slot.key) }} +
+
+
+
+ +
+
+
+ Load at least one trace to inspect extracted thought entries, shell commands, token provenance, and end-state outputs. +
+
+ + +
+
+ + + + + + + diff --git a/assets/compare_trajectory/styles.css b/assets/compare_trajectory/styles.css new file mode 100644 index 0000000..90bf0d9 --- /dev/null +++ b/assets/compare_trajectory/styles.css @@ -0,0 +1,654 @@ +:root { + --bg: #edf2f4; + --panel: #ffffff; + --panel-2: #f6f9fa; + --panel-3: #e9f1f3; + --line: #d4dee3; + --line-strong: #b7c6cc; + --text: #132127; + --muted: #5c7078; + --accent: #0f766e; + --good: #166534; + --good-soft: #dcfce7; + --warn: #b45309; + --warn-soft: #ffedd5; + --bad: #b91c1c; + --bad-soft: #fee2e2; + --shadow: 0 12px 32px rgba(19, 33, 39, 0.08); + --radius: 18px; + --mono: "IBM Plex Mono", ui-monospace, monospace; + --sans: "Public Sans", "Segoe UI", sans-serif; +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + font-family: var(--sans); + color: var(--text); + background: + radial-gradient(circle at top left, rgba(15, 118, 110, 0.08), transparent 24%), + linear-gradient(180deg, #f8fbfb 0%, var(--bg) 100%); +} + +#app[v-cloak] { + visibility: hidden; +} + +.boot-overlay { + position: fixed; + inset: 0; + z-index: 100; + display: none; + align-items: center; + justify-content: center; + padding: 24px; + background: rgba(237, 242, 244, 0.92); + backdrop-filter: blur(10px); +} + +#app[v-cloak] + .boot-overlay { + display: flex; +} + +.boot-panel { + min-width: min(420px, 100%); + padding: 24px 26px; + border: 1px solid rgba(183, 198, 204, 0.8); + border-radius: 20px; + background: rgba(255, 255, 255, 0.92); + box-shadow: var(--shadow); +} + +.boot-spinner { + width: 42px; + height: 42px; + border-radius: 999px; + border: 3px solid rgba(15, 118, 110, 0.16); + border-top-color: var(--accent); + border-right-color: rgba(15, 118, 110, 0.48); + animation: boot-spin 0.85s linear infinite; +} + +.boot-title { + margin-top: 16px; + font: 700 14px var(--mono); + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--accent); +} + +.boot-copy { + margin-top: 8px; + color: var(--muted); + font-size: 14px; + line-height: 1.6; +} + +@keyframes boot-spin { + from { + transform: rotate(0deg); + } + to { + transform: rotate(360deg); + } +} + +button, +input, +textarea, +select { + font: inherit; +} + +.shell { + min-height: 100vh; +} + +.topbar { + position: sticky; + top: 0; + z-index: 20; + backdrop-filter: blur(16px); + background: rgba(248, 251, 251, 0.9); + border-bottom: 1px solid rgba(183, 198, 204, 0.7); +} + +.topbar-inner, +.upload-rail, +.content { + max-width: 1480px; + margin: 0 auto; + padding-left: 24px; + padding-right: 24px; +} + +.topbar-inner { + display: grid; + grid-template-columns: 1fr auto; + gap: 18px; + align-items: center; + padding-top: 16px; + padding-bottom: 16px; +} + +.brand-title { + margin: 0; + font-size: 24px; + line-height: 1; + letter-spacing: -0.04em; +} + +.brand-copy { + margin-top: 6px; + color: var(--muted); + font-size: 13px; + line-height: 1.5; + max-width: 72ch; +} + +.toolbar-copy { + color: var(--muted); + font-size: 12px; + line-height: 1.5; + max-width: 30ch; +} + +.tab-strip, +.filter-strip, +.badge-row, +.button-row, +.metric-badges, +.event-meta { + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +.tab, +.toggle, +.button, +.button-label { + appearance: none; + border: 1px solid var(--line-strong); + background: #fff; + color: var(--text); + border-radius: 999px; + padding: 9px 14px; + cursor: pointer; + transition: transform 0.15s ease, border-color 0.15s ease, background 0.15s ease; + font-size: 13px; + font-weight: 600; + text-decoration: none; + display: inline-flex; + align-items: center; + justify-content: center; +} + +.tab:hover, +.toggle:hover, +.button:hover, +.button-label:hover { + border-color: var(--accent); + transform: translateY(-1px); +} + +.tab.active, +.toggle.active, +.button.primary, +.button-label.primary { + background: var(--accent); + color: #fff; + border-color: var(--accent); +} + +.button-label input { + display: none; +} + +.upload-rail { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 16px; + padding-top: 16px; + padding-bottom: 16px; +} + +.upload-card, +.panel, +.summary-card, +.trace-card, +.mini-card, +.event, +.command-run, +.thought-entry { + background: var(--panel); + border: 1px solid var(--line); + border-radius: var(--radius); + box-shadow: var(--shadow); +} + +.upload-card { + padding: 16px; + min-height: 154px; + display: flex; + flex-direction: column; + gap: 14px; +} + +.upload-head, +.trace-card-head, +.run-head, +.event-head, +.thought-head, +.panel-head { + display: flex; + justify-content: space-between; + gap: 12px; + align-items: start; +} + +.upload-title, +.panel-title, +.trace-name, +.section-title { + margin: 0; + line-height: 1.2; +} + +.upload-title { + font-size: 16px; +} + +.panel-title { + font-size: 18px; +} + +.trace-name { + font-size: 18px; +} + +.upload-subtitle { + margin-top: 5px; + color: var(--muted); + font-size: 12px; + line-height: 1.5; +} + +.upload-body { + flex: 1; + display: flex; + flex-direction: column; + justify-content: center; + gap: 12px; + border: 1.5px dashed var(--line-strong); + border-radius: 16px; + padding: 16px; + background: var(--panel-2); +} + +.attachment-box { + margin-top: 4px; + padding-top: 12px; + border-top: 1px solid var(--line); +} + +.upload-file { + font-weight: 700; + font-size: 15px; + word-break: break-word; +} + +.upload-path, +.muted, +.summary-copy, +.event-time, +.trace-file { + color: var(--muted); + font-size: 12px; + line-height: 1.55; +} + +.badge { + display: inline-flex; + align-items: center; + gap: 6px; + border: 1px solid var(--line); + border-radius: 999px; + padding: 6px 10px; + font: 700 11px var(--mono); + text-transform: uppercase; + letter-spacing: 0.06em; + background: var(--panel-3); + color: var(--muted); +} + +.badge.good { + color: var(--good); + background: var(--good-soft); + border-color: #b9e6c7; +} + +.badge.warn { + color: var(--warn); + background: var(--warn-soft); + border-color: #f6cf9e; +} + +.badge.bad { + color: var(--bad); + background: var(--bad-soft); + border-color: #f3baba; +} + +.content { + padding-top: 8px; + padding-bottom: 28px; +} + +.panel, +.summary-card, +.trace-card, +.mini-card { + padding: 16px; + margin-bottom: 18px; + box-shadow: none; +} + +.summary-grid, +.trace-grid, +.pair-grid, +.trace-pane-grid, +.metric-grid { + display: grid; + gap: 18px; +} + +.summary-grid, +.trace-grid, +.pair-grid, +.trace-pane-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); +} + +.summary-card h3, +.trace-card h3, +.mini-card h3 { + margin: 0 0 8px; + font-size: 15px; + line-height: 1.3; +} + +.metric-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + margin-top: 12px; +} + +.metric { + border: 1px solid var(--line); + border-radius: 14px; + background: var(--panel-2); + padding: 12px; +} + +.metric-label { + color: var(--muted); + font-size: 11px; + letter-spacing: 0.08em; + text-transform: uppercase; + margin-bottom: 8px; +} + +.metric-value { + font-size: 25px; + line-height: 1; + font-weight: 800; + letter-spacing: -0.04em; +} + +.metric-sub { + margin-top: 8px; + color: var(--muted); + font-size: 12px; + line-height: 1.5; +} + +.compare-table, +.command-table, +.thought-table { + width: 100%; + border-collapse: collapse; +} + +.compare-table th, +.compare-table td, +.command-table th, +.command-table td, +.thought-table th, +.thought-table td { + text-align: left; + border-top: 1px solid var(--line); + padding: 11px 10px; + vertical-align: top; + font-size: 13px; +} + +.compare-table th, +.command-table th, +.thought-table th { + color: var(--muted); + font-size: 11px; + letter-spacing: 0.08em; + text-transform: uppercase; +} + +.token-table { + margin-top: 12px; +} + +.mono { + font-family: var(--mono); + font-size: 12px; + white-space: pre-wrap; + word-break: break-word; + line-height: 1.55; +} + +.task-box { + margin-top: 12px; + padding: 12px; + border-radius: 14px; + border: 1px solid var(--line); + background: var(--panel-2); + font-size: 13px; + line-height: 1.6; +} + +.task-box strong { + display: block; + margin-bottom: 6px; + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.08em; + color: var(--muted); +} + +.run-list, +.event-list, +.thought-list, +.section-stack { + display: flex; + flex-direction: column; + gap: 10px; + margin-top: 14px; +} + +.run-list, +.event-list, +.thought-list { + max-height: 520px; + overflow: auto; + padding-right: 2px; +} + +.command-run, +.event, +.thought-entry { + padding: 12px; + background: var(--panel-2); + box-shadow: none; +} + +.detail-card, +.command-run, +.thought-entry { + border: 1px solid rgba(183, 198, 204, 0.78); + background: + linear-gradient(180deg, rgba(255, 255, 255, 0.92), rgba(246, 249, 250, 0.96)); +} + +.run-title, +.event-kind, +.thought-title { + font: 700 11px var(--mono); + color: var(--accent); + text-transform: uppercase; + letter-spacing: 0.08em; +} + +.run-command, +.event-summary, +.thought-body { + font-size: 13px; + line-height: 1.55; + white-space: pre-wrap; + word-break: break-word; + margin-top: 8px; +} + +.detail-summary { + margin-top: 6px; + color: var(--text); + font-size: 13px; + line-height: 1.55; + white-space: pre-wrap; +} + +.detail-sections { + display: flex; + flex-direction: column; + gap: 10px; + margin-top: 10px; +} + +.detail-section { + border: 1px solid rgba(212, 222, 227, 0.9); + border-radius: 14px; + background: rgba(255, 255, 255, 0.94); + overflow: hidden; +} + +.detail-section-label { + padding: 9px 12px; + border-bottom: 1px solid rgba(212, 222, 227, 0.88); + font: 700 11px var(--mono); + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--muted); + background: rgba(233, 241, 243, 0.78); +} + +.detail-text { + padding: 12px; + font-size: 13px; + line-height: 1.65; + white-space: pre-wrap; + word-break: break-word; +} + +.detail-code-wrap { + margin: 0; + padding: 0; + overflow: auto; + background: #f5f7f8; +} + +.detail-code { + display: block; + padding: 14px 16px; + font: 500 12px/1.65 var(--mono); + white-space: pre; + min-width: 100%; +} + +.detail-code.hljs { + background: #f5f7f8; +} + +.bar-list { + display: flex; + flex-direction: column; + gap: 10px; + margin-top: 14px; +} + +.bar-row { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 12px; + align-items: center; +} + +.bar-wrap { + margin-top: 6px; + height: 8px; + border-radius: 999px; + background: #e6eef1; + overflow: hidden; +} + +.bar { + height: 100%; + border-radius: 999px; + background: linear-gradient(90deg, #0f766e, #14b8a6); +} + +.empty-state { + padding: 44px 22px; + text-align: center; + color: var(--muted); + border: 1.5px dashed var(--line-strong); + border-radius: 18px; + background: var(--panel-2); +} + +@media (max-width: 1160px) { + .topbar-inner, + .upload-rail, + .summary-grid, + .trace-grid, + .pair-grid, + .trace-pane-grid { + grid-template-columns: 1fr; + } + + .topbar-inner { + align-items: start; + } + +} + +@media (max-width: 760px) { + .topbar-inner, + .upload-rail, + .content { + padding-left: 14px; + padding-right: 14px; + } + + .metric-grid { + grid-template-columns: 1fr; + } +} diff --git a/assets/demo.mp4 b/assets/demo.mp4 new file mode 100644 index 0000000..6614fa6 Binary files /dev/null and b/assets/demo.mp4 differ diff --git a/assets/odysseys_eval_step100.png b/assets/odysseys_eval_step100.png new file mode 100644 index 0000000..a53af4c Binary files /dev/null and b/assets/odysseys_eval_step100.png differ diff --git a/assets/om2w_autoeval_step100.png b/assets/om2w_autoeval_step100.png new file mode 100644 index 0000000..49005c8 Binary files /dev/null and b/assets/om2w_autoeval_step100.png differ diff --git a/assets/task_showcase/README.md b/assets/task_showcase/README.md new file mode 100644 index 0000000..872e7cc --- /dev/null +++ b/assets/task_showcase/README.md @@ -0,0 +1,61 @@ +# Task Showcase โ€” repeatable web-agent runs as a dashboard + +A tiny Flask app that consolidates Webwright runs for **repeatable** odyssey +tasks (those whose answer changes over time โ€” deals, inventory, listings, +job boards, weather, etc.) into a single dashboard. + +Each task ships only two files: + +``` +tasks// + task.json โ€“ metadata (title, theme, cadence, level, prompt, website) + report.json โ€“ curated, structured agent output: {sources, result.sections} +``` + +The Flask app is fully generic: templates iterate `sources` and +`result.sections` without any per-task branching, so adding a new task is +just dropping in a new folder with those two files. + +## Run + +```bash +pip install flask +python app.py # serves http://127.0.0.1:5005 +``` + +To render JSON generated by a Webwright run without copying it back into this +folder, point the app at that run's generated tasks directory: + +```bash +python app.py --tasks-dir ../../outputs/default//task_showcase/tasks +``` + +The `task_showcase.yaml` runtime overlay writes the same directory shape under +the run workspace and then smoke-tests this renderer against it. + +## report.json shape + +```jsonc +{ + "sources": [ + {"name": "Slickdeals", "url": "https://slickdeals.net/", "note": "front-page best deal"} + ], + "result": { + "headline": "Today's bargain roundup", + "sections": [ + {"type": "summary", "title": "...", "body": "..."}, + {"type": "table", "title": "...", "columns": [...], "rows": [[...], ...]}, + {"type": "list", "title": "...", "entries": ["..."]}, + {"type": "kv", "title": "...", "entries": [["k", "v"], ...]}, + {"type": "cards", "title": "...", "entries": [ + {"title": "...", "subtitle": "...", "fields": [["k","v"]], "url": "..."} + ]} + ] + } +} +``` + +The 6 included tasks (Slickdeals, Pokรฉmon TCG, Austin apartments, The Ophelia +Pittsburgh, NZโ†’AU legal roles, this-week driving weather) each map to an +odyssey benchmark task whose underlying pages refresh on a schedule, so the +dashboard makes sense to re-run repeatedly. diff --git a/assets/task_showcase/app.py b/assets/task_showcase/app.py new file mode 100644 index 0000000..81427d8 --- /dev/null +++ b/assets/task_showcase/app.py @@ -0,0 +1,264 @@ +"""Flask dashboard for repeatable task-showcase JSON. + +Each task lives under ``tasks//``. The only required files are: + + task.json โ€“ metadata written by the build script + report.json โ€“ structured output consumed by the generic template + +Optional run artifacts such as ``final_script_log.txt``, ``steps.jsonl``, and +``screenshots/`` are used when present, but the renderer does not require them. + +Routes: + / โ€“ dashboard listing available tasks + /task/ โ€“ per-task report view + /task//screenshot/.. โ€“ optional screenshot file +""" +from __future__ import annotations + +import argparse +import json +import os +import re +from pathlib import Path + +from flask import Flask, abort, render_template, send_from_directory + +ROOT = Path(__file__).resolve().parent + + +def _resolve_tasks_dir(value: str | Path | None = None) -> Path: + if value is None: + value = os.environ.get("TASK_SHOWCASE_TASKS_DIR") or ROOT / "tasks" + return Path(value).expanduser().resolve() + + +TASKS_DIR = _resolve_tasks_dir() + +app = Flask(__name__) + + +# ---------- task metadata ---------- + +def list_tasks() -> list[dict]: + out: list[dict] = [] + if not TASKS_DIR.exists(): + return out + for d in sorted(TASKS_DIR.iterdir()): + info_path = d / "task.json" + if not info_path.exists(): + continue + info = json.loads(info_path.read_text()) + info["short_id"] = d.name + out.append(info) + return out + + +# ---------- log + steps parsing ---------- + +_STEP_LINE = re.compile(r"^step\s+(\d+)\s+action:\s*(.*)$", re.IGNORECASE) +_FINAL_LINE = re.compile(r"^Final Response:\s*(.*)$", re.IGNORECASE) +_URL_RE = re.compile(r"https?://[^\s'\"<>;,)]+") +_TRAIL_PUNCT = re.compile(r"[.,;:)\]]+$") + + +def _clean_url(u: str) -> str: + return _TRAIL_PUNCT.sub("", u) + + +def parse_log(log_path: Path) -> tuple[dict[int, str], str]: + """Return ({step_num: action_text}, final_response).""" + steps: dict[int, str] = {} + final = "" + if not log_path.exists(): + return steps, final + for raw in log_path.read_text(encoding="utf-8").splitlines(): + m = _STEP_LINE.match(raw.strip()) + if m: + steps[int(m.group(1))] = m.group(2).strip() + continue + f = _FINAL_LINE.match(raw.strip()) + if f: + final = f.group(1).strip() + return steps, final + + +def parse_steps_jsonl(path: Path) -> list[dict]: + rows: list[dict] = [] + if not path.exists(): + return rows + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + continue + for r in rows: + try: + r["step_num"] = int(r.get("step_num", 0)) + except (TypeError, ValueError): + r["step_num"] = 0 + rows.sort(key=lambda r: r["step_num"]) + return rows + + +def build_steps(task_dir: Path) -> tuple[list[dict], str]: + log_steps, final = parse_log(task_dir / "final_script_log.txt") + raw_rows = parse_steps_jsonl(task_dir / "steps.jsonl") + + out: list[dict] = [] + for row in raw_rows: + n = row["step_num"] + action_text = log_steps.get(n) or row.get("action", "").strip() + # the first row of steps.jsonl sometimes contains the full multi-line + # log dump. If so, take only its first "step N action:" line. + if action_text and "\nstep " in action_text.lower(): + first_line = action_text.splitlines()[0] + m = _STEP_LINE.match(first_line) + if m: + action_text = m.group(2).strip() + shot = row.get("screenshot") or "" + if shot: + shot = Path(shot).name + urls = [_clean_url(u) for u in _URL_RE.findall(action_text)] + seen: set[str] = set() + urls = [u for u in urls if not (u in seen or seen.add(u))] + out.append({ + "step_num": n, + "action": action_text, + "screenshot": shot, + "urls": urls, + }) + return out, final + + +def _host(u: str) -> str: + from urllib.parse import urlparse + try: + return urlparse(u).netloc.replace("www.", "") or u + except Exception: + return u + + +def collect_pages(steps: list[dict], fallback_site: str | None) -> list[dict]: + """Build a unique URL list across all steps for the Pages grid.""" + pages: list[dict] = [] + seen: set[str] = set() + for s in steps: + for u in s["urls"]: + if u in seen: + continue + seen.add(u) + from urllib.parse import urlparse + try: + pu = urlparse(u) + tail = pu.path.rstrip("/").split("/")[-1] or pu.netloc + title = f"{pu.netloc}/{tail}" if tail != pu.netloc else pu.netloc + except Exception: + title = u + pages.append({ + "step": s["step_num"], + "url": u, + "title": title, + "host": _host(u), + "screenshot": s.get("screenshot") or "", + }) + if not pages and fallback_site: + pages.append({ + "step": 1, "url": fallback_site, "title": fallback_site, + "host": _host(fallback_site), "screenshot": "", + }) + return pages + + +def collect_sources(pages: list[dict]) -> list[dict]: + """Group pages by host -> [{host, count, sample_url}].""" + from collections import OrderedDict + grouped: "OrderedDict[str, dict]" = OrderedDict() + for p in pages: + h = p["host"] + if h not in grouped: + grouped[h] = {"host": h, "count": 0, "sample_url": p["url"]} + grouped[h]["count"] += 1 + return list(grouped.values()) + + +# ---------- routes ---------- + +@app.route("/") +def index(): + tasks = list_tasks() + return render_template("dashboard.html", tasks=tasks) + + +@app.route("/task/") +def task_view(short_id: str): + task_dir = TASKS_DIR / short_id + info_path = task_dir / "task.json" + if not info_path.exists(): + abort(404) + info = json.loads(info_path.read_text()) + info["short_id"] = short_id + steps, final = build_steps(task_dir) + # last-modified timestamp of the run for the "Updated" line + import datetime as _dt + log_path = task_dir / "final_script_log.txt" + if log_path.exists(): + ts = _dt.datetime.fromtimestamp(log_path.stat().st_mtime) + updated = ts.strftime("%-m/%-d/%Y, %-I:%M:%S %p") + else: + updated = "" + # Per-task structured report lives next to the run artifacts. + report_path = task_dir / "report.json" + if report_path.exists(): + report = json.loads(report_path.read_text()) + else: + report = {} + sources = report.get("sources", []) + result = report.get("result", {"sections": []}) + num_steps = len(steps) + if not num_steps: + try: + num_steps = int(info.get("num_steps") or 0) + except (TypeError, ValueError): + num_steps = 0 + return render_template( + "task.html", + info=info, + final_response=final, + sources=sources, + result=result, + updated=updated, + num_steps=num_steps, + ) + + +@app.route("/task//screenshot/") +def screenshot(short_id: str, filename: str): + folder = (TASKS_DIR / short_id / "screenshots").resolve() + target = (folder / filename).resolve() + if not target.exists(): + abort(404) + return send_from_directory(folder, filename) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--host", default="0.0.0.0") + parser.add_argument("--port", type=int, default=5005) + parser.add_argument( + "--tasks-dir", + type=Path, + default=None, + help="Directory containing /task.json and report.json folders.", + ) + parser.add_argument("--debug", action="store_true") + args = parser.parse_args() + global TASKS_DIR + TASKS_DIR = _resolve_tasks_dir(args.tasks_dir) + app.run(host=args.host, port=args.port, debug=args.debug) + + +if __name__ == "__main__": + main() diff --git a/assets/task_showcase/tasks/austin_apartments/report.json b/assets/task_showcase/tasks/austin_apartments/report.json new file mode 100644 index 0000000..af03c0f --- /dev/null +++ b/assets/task_showcase/tasks/austin_apartments/report.json @@ -0,0 +1,137 @@ +{ + "sources": [ + { + "name": "Zillow โ€” Domain/North Austin", + "url": "https://www.zillow.com/austin-tx/apartments/", + "note": "1+ bd, โ‰ค$1,800, W/D + pool" + }, + { + "name": "Apartments.com โ€” same filter", + "url": "https://www.apartments.com/austin-tx/1-bedrooms/", + "note": "1 bd, โ‰ค$1,800, W/D + pool" + }, + { + "name": "CapMetro trip planner", + "url": "https://www.capmetro.org/ride/plan/trip-planner" + }, + { + "name": "Niche โ€” North Burnet", + "url": "https://www.niche.com/places-to-live/n/north-burnet-austin-tx/" + }, + { + "name": "Texas AG โ€” Renters' Rights", + "url": "https://www.texasattorneygeneral.gov/consumer-protection/home-real-estate-and-travel/renters-rights" + } + ], + "result": { + "headline": "Apartment-hunting brief โ€” Domain / North Austin", + "sections": [ + { + "type": "table", + "title": "Zillow comparison", + "columns": [ + "Property", + "Address", + "Rent", + "Highlights" + ], + "rows": [ + [ + "Broadstone North ATX", + "10601 Center Lake Dr, 78753", + "$1,385+", + "Pool ยท In-unit laundry" + ], + [ + "The Met", + "10101 Metropolitan Dr, 78758", + "$1,720+", + "Pool ยท In-unit laundry" + ], + [ + "Residences at the Domain", + "11400 Domain Dr, 78758", + "$1,660+", + "Pool ยท In-unit laundry" + ] + ] + }, + { + "type": "table", + "title": "Apartments.com comparison", + "columns": [ + "Property", + "Address", + "Rent", + "Highlights" + ], + "rows": [ + [ + "The Met", + "10101 Metropolitan Dr, 78758", + "$1,720+", + "Pool ยท W/D" + ], + [ + "The Addison Apartments", + "2601 Esperanza Crossing, 78758", + "$1,190+", + "Pool ยท W/D" + ], + [ + "Presidium 183", + "9111 Research Blvd, 78758", + "$1,367+", + "Pool ยท W/D" + ] + ] + }, + { + "type": "list", + "title": "Shortlist (best overall fit)", + "entries": [ + "The Addison Apartments โ€” essentially in the Domain; easiest bus-and-walk commute", + "Residences at the Domain โ€” best pure location fit, inside the Domain district", + "Broadstone North ATX โ€” cheapest workable option; relies on a bus connection" + ] + }, + { + "type": "kv", + "title": "Transit & neighborhood", + "entries": [ + [ + "Best transit", + "The Addison โ€” short bus-and-walk to Domain" + ], + [ + "Best location", + "Residences at the Domain โ€” inside the district" + ], + [ + "North Burnet on Niche", + "Overall A+, Public Schools Aโˆ’, urban feel" + ], + [ + "Walkability winners", + "Residences at the Domain & The Addison" + ], + [ + "Caveat", + "Niche safety detail was limited โ€” verify before signing" + ] + ] + }, + { + "type": "list", + "title": "Lease-review checklist", + "entries": [ + "Deposit amount, deduction rules, refund timing", + "Repair-request process, habitability, emergency maintenance", + "Every recurring + one-time fee disclosed before applying", + "Required disclosures and safety-device information", + "Move-out notice, early termination, statutory exceptions" + ] + } + ] + } +} \ No newline at end of file diff --git a/assets/task_showcase/tasks/austin_apartments/task.json b/assets/task_showcase/tasks/austin_apartments/task.json new file mode 100644 index 0000000..5f4d293 --- /dev/null +++ b/assets/task_showcase/tasks/austin_apartments/task.json @@ -0,0 +1,11 @@ +{ + "task_id": "8fcdeed84a0deb05342b07c26116792a5b6a6a3f", + "short_id": "austin_apartments", + "title": "Austin Domain apartments under budget", + "theme": "Real estate", + "cadence": "daily", + "level": "medium", + "website": "https://www.google.com", + "task_prompt": "Iโ€™m relocating to Austin in about two months for a new job near the Domain, so I want help narrowing down apartments that would actually work for day-to-day life without blowing my budget. Please start on Zillow and search the Domain/North Austin area for 1-bedroom apartments under $1,800 a month, and filter for places that have both in-unit washer/dryer and a pool because those are my non-negotiables. Open the three best-looking Zillow listings in separate tabs so I can compare the photos, map placement, and amenity details, and pull out the apartment name, full address, rent, and a couple of listing highlights from each. Then do the same search on Apartments.com with the same budget and amenity filters, again opening the three strongest options in their own tabs so I can visually compare them and note the same details. Once youโ€™ve got both sets, compare the six options, remove duplicates if the same property shows up on both sites, and tell me which apartments seem like the best overall fit based on price, amenities, and location near the Domain. After that, use CapMetroโ€™s site and map tools to check whether each shortlisted apartment is near a MetroRail stop or has a practical bus connection into the Domain area, because I want to know whether I could commute without driving every day; if the map view helps, pull it up and keep the most useful transit page open. Then look up the neighborhoods for those apartments on Niche so I can get a feel for what living there would be like, especially safety ratings, walkability info if itโ€™s shown, and whether there are grocery stores nearby for basic errands. Finally, go to the Texas Attorney General website and find the rentersโ€™ rights guidance that matters most before signing a lease in Texas, especially anything about deposits, repairs, fees, disclosures, and ending a lease, and leave that page open too so I can read it myself later. In the end, give me one clean apartment-hunting brief that combines the listing comparison, transit practicality to the Domain, neighborhood pros and cons, and a short lease-review checklist I can use when I start contacting properties.", + "num_steps": 15 +} diff --git a/assets/task_showcase/tasks/driving_weather/report.json b/assets/task_showcase/tasks/driving_weather/report.json new file mode 100644 index 0000000..8ab3a55 --- /dev/null +++ b/assets/task_showcase/tasks/driving_weather/report.json @@ -0,0 +1,61 @@ +{ + "sources": [ + { + "name": "Google โ€” Baltimore, MD weather", + "url": "https://www.google.com/search?q=Baltimore+Maryland+weather" + }, + { + "name": "Wunderground โ€” Syracuse, NY", + "url": "https://www.wunderground.com/forecast/us/ny/syracuse" + }, + { + "name": "NWS Cleveland โ€” Rittman/Marshallville", + "url": "https://forecast.weather.gov/MapClick.php?lat=40.978&lon=-81.7818" + }, + { + "name": "NWS Mount Holly", + "url": "https://www.weather.gov/phi/" + } + ], + "result": { + "headline": "This-week driving weather snapshot", + "sections": [ + { + "type": "table", + "title": "Per-location risk", + "columns": [ + "Location", + "Headline", + "Detail" + ], + "rows": [ + [ + "Baltimore, MD", + "Baseline 67ยฐF, partly sunny", + "Calm โ€” no driving concern at home" + ], + [ + "Syracuse, NY", + "Lowest next-7-day temp 36ยฐF", + "Chilly overnight low on Sat 5/2" + ], + [ + "Rittman / Marshallville, OH", + "Hazardous Weather Outlook posted", + "This afternoon: slight chance of showers, partly sunny, high near 54, NW wind ~10 mph" + ], + [ + "Mount Holly region (Mid-Atlantic)", + "Snowfall 0\"", + "Range 0\" โ€“ 0\" on the Snowfall Totals by Location table" + ] + ] + }, + { + "type": "summary", + "title": "Drive-risk verdict", + "body": "Baltimore baseline mild; Mid-Atlantic snowfall flat at 0\". Watch Ohio leg for the Hazardous Weather Outlook and a chilly overnight if heading toward Syracuse." + } + ] + } +} \ No newline at end of file diff --git a/assets/task_showcase/tasks/driving_weather/task.json b/assets/task_showcase/tasks/driving_weather/task.json new file mode 100644 index 0000000..31ec315 --- /dev/null +++ b/assets/task_showcase/tasks/driving_weather/task.json @@ -0,0 +1,11 @@ +{ + "task_id": "b6b8ad71aa3112840790066d7d62b498babdfa5c", + "short_id": "driving_weather", + "title": "This-week driving weather snapshot", + "theme": "Local feed", + "cadence": "every 6h", + "level": "easy", + "website": "https://www.google.com", + "task_prompt": "Iโ€™m trying to decide whether driving this week is a bad idea, so can you build me a quick weather risk snapshot that starts with what it feels like right now and then widens out to the bigger trouble spots? First, on Google, search for Baltimore, Maryland weather and grab the current temperature plus the plain-English condition like cloudy, sunny, rain, or whatever it says, just so I have a baseline for home conditions. Then go to Wunderground and look up Syracuse, New York, and check the 10-day/7-day style forecast to find the lowest temperature expected over the next 7 days, including which day it happens, because I want to compare that colder destination against Baltimore. After that, use the National Weather Service forecast page for the Rittman, Ohio area near Marshallville and tell me what the current forecast says and whether there are any active alerts posted there, since that would really affect an Ohio leg of the drive; please open the actual forecast page and leave it visible so I can see the alert area and forecast text myself. Finally, go to the NWS Mount Holly page, find the winter forecast graphic, and report the snowfall amount shown there so I can tell whether the Mid-Atlantic part looks like a nuisance event or something more serious; if the graphic opens separately, leave that tab open too so I can look at the map. In the end, send me a short location-by-location summary with the key weather risk for Baltimore, Syracuse, Rittman/Marshallville, and the Mount Holly region.", + "num_steps": 7 +} diff --git a/assets/task_showcase/tasks/nz_au_legal_jobs/report.json b/assets/task_showcase/tasks/nz_au_legal_jobs/report.json new file mode 100644 index 0000000..6888ca3 --- /dev/null +++ b/assets/task_showcase/tasks/nz_au_legal_jobs/report.json @@ -0,0 +1,68 @@ +{ + "sources": [ + { + "name": "Legal People Australia", + "url": "https://www.legalpeople.com.au/jobs", + "note": "starting recruiter" + }, + { + "name": "Seek โ€” Legal jobs", + "url": "https://www.seek.com.au/legal-jobs", + "note": "main AU job board" + }, + { + "name": "LinkedIn โ€” AU legal jobs", + "url": "https://www.linkedin.com/jobs/search/?keywords=lawyer&location=Australia" + }, + { + "name": "Burgess Paluch (recruiter)", + "url": "https://www.burgesspaluch.com.au/jobs/" + }, + { + "name": "Mahlab (recruiter)", + "url": "https://www.mahlab.com.au/jobs/" + } + ], + "result": { + "headline": "AU legal market sweep โ€” NZ-qualified, ~3 PQE", + "sections": [ + { + "type": "kv", + "title": "Shortlist totals", + "entries": [ + [ + "Roles shortlisted", + "18" + ], + [ + "Clearly eligible now", + "0" + ], + [ + "Likely eligible โ€” needs admission clarification", + "15" + ], + [ + "Probably not viable without current AU admission", + "3" + ] + ] + }, + { + "type": "summary", + "title": "Where to focus first", + "body": "Sydney, Melbourne, Canberra, Brisbane, and Perth across disputes/litigation, commercial, family, public law, and junior in-house roles. Prioritise listings that are silent on admission but a strong 2โ€“4 PQE fit. Deprioritise roles that expressly require current Australian admission or a practising certificate." + }, + { + "type": "list", + "title": "Practical filters used", + "entries": [ + "Only public pages โ€” recruiters, AU job boards, direct firm/in-house careers", + "Strictly 2โ€“4 PQE relevance", + "Bucketed by AU-admission status so the distinction is visible", + "8+ representative evidence tabs spread across cities & practice areas" + ] + } + ] + } +} \ No newline at end of file diff --git a/assets/task_showcase/tasks/nz_au_legal_jobs/task.json b/assets/task_showcase/tasks/nz_au_legal_jobs/task.json new file mode 100644 index 0000000..03e0b9c --- /dev/null +++ b/assets/task_showcase/tasks/nz_au_legal_jobs/task.json @@ -0,0 +1,11 @@ +{ + "task_id": "9825e1505ad6eacf0ae2af5709a74d9eb0280029", + "short_id": "nz_au_legal_jobs", + "title": "NZโ†’AU 2โ€“4 PQE legal roles tracker", + "theme": "Job tracker", + "cadence": "weekly", + "level": "hard", + "website": "https://www.legalpeople.com.au", + "task_prompt": "Iโ€™m trying to figure out where I could realistically apply in Australia as a New Zealand-qualified lawyer with about three years of post-qualification experience, and I want a serious browser-based market sweep rather than a skim of one recruiter site. Please start with Legal People Australia, but then widen naturally to other Australian legal recruiters, major job boards, and public law-firm or in-house careers pages so weโ€™re not missing obvious opportunities. Build me a shortlist of exactly 18 to 24 current roles that are genuinely relevant to someone around 2 to 4 PQE, and only include roles where the listing either explicitly welcomes New Zealand qualification or NZ admission, is silent but otherwise looks plausibly transferable, or clearly states Australian admission is required so that distinction is visible. For each shortlisted role, record the job title, employer or recruiter, city/state, practice area, stated PQE or experience range, whether NZ-qualified or NZ-admitted candidates are explicitly mentioned, whether Australian admission appears mandatory, whether a practising certificate or relocation detail is mentioned, and the application link; if a field is missing, mark it as not shown. Keep the work grounded in public pages only. Open and keep available the strongest evidence tabs for at least 8 representative roles spread across different cities or practice areas so I can sanity-check them later, including a mix of recruiter listings and direct employer pages if available. As you go, compare patterns across the market and separate the shortlist into three buckets: clearly eligible now, likely eligible but needs admission clarification, and probably not viable without current Australian admission. Finish with one organized tracker or memo that includes all 18 to 24 roles plus a concise recommendation on where I should focus first by city and practice area, and leave the finished tracker and the most useful evidence tabs open. I would prefer a CrpytoPad Spreadsheet for this tracker.", + "num_steps": 17 +} diff --git a/assets/task_showcase/tasks/ophelia_pittsburgh/report.json b/assets/task_showcase/tasks/ophelia_pittsburgh/report.json new file mode 100644 index 0000000..6e0d5f2 --- /dev/null +++ b/assets/task_showcase/tasks/ophelia_pittsburgh/report.json @@ -0,0 +1,133 @@ +{ + "sources": [ + { + "name": "Apartments.com โ€” The Ophelia", + "url": "https://www.apartments.com/the-ophelia-pittsburgh-pa/", + "note": "floor plans / availability" + }, + { + "name": "WeatherTech โ€” 2020 Toyota Highlander", + "url": "https://www.weathertech.com/cargo-liner-toyota-highlander.html?year=2020&ymmSearch=true", + "note": "cargo liner" + }, + { + "name": "AoonuAuto โ€” Honda Civic LED emblem", + "url": "https://aoonuauto.com/products/honda-civic-car-emblems-led-logo-badges" + }, + { + "name": "Zillow โ€” Pittsburgh homes for sale", + "url": "https://www.zillow.com/pittsburgh-pa/", + "note": "5 live listings" + } + ], + "result": { + "headline": "Renting at The Ophelia vs. buying nearby โ€” snapshot", + "sections": [ + { + "type": "table", + "title": "Available floor plans at The Ophelia", + "columns": [ + "Plan", + "Layout" + ], + "rows": [ + [ + "S6H", + "Studio / 1 Bath" + ], + [ + "S4", + "Studio / 1 Bath" + ] + ] + }, + { + "type": "table", + "title": "Vehicle move-in cost references", + "columns": [ + "Item", + "Product", + "Price" + ], + "rows": [ + [ + "WeatherTech", + "2020 Toyota Highlander Cargo/Trunk Liner", + "$250" + ], + [ + "LED emblem (Civic)", + "Honda CIVIC Car Emblems LED LOGO Badges", + "$109.99" + ] + ] + }, + { + "type": "cards", + "title": "Buying side โ€” 5 Zillow listings", + "entries": [ + { + "title": "523 N Euclid Ave", + "subtitle": "Pittsburgh, PA 15206", + "fields": [ + [ + "note", + "Highland Park / East Liberty โ€” closest comp" + ] + ], + "url": "https://www.zillow.com/homedetails/523-N-Euclid-Ave-Pittsburgh-PA-15206/11622890_zpid/" + }, + { + "title": "725 N Beatty St", + "subtitle": "Pittsburgh, PA 15206", + "fields": [ + [ + "note", + "Nearby comp to The Ophelia" + ] + ], + "url": "https://www.zillow.com/homedetails/725-N-Beatty-St-Pittsburgh-PA-15206/11622933_zpid/" + }, + { + "title": "234 Olympia St", + "subtitle": "Pittsburgh, PA 15211", + "fields": [ + [ + "note", + "Broadens price range" + ] + ], + "url": "https://www.zillow.com/homedetails/234-Olympia-St-Pittsburgh-PA-15211/11586269_zpid/" + }, + { + "title": "211 S Margery Dr", + "subtitle": "Pittsburgh, PA 15238", + "fields": [ + [ + "note", + "Broadens location range" + ] + ], + "url": "https://www.zillow.com/homedetails/211-S-Margery-Dr-Pittsburgh-PA-15238/11375044_zpid/" + }, + { + "title": "637 Kelso Rd", + "subtitle": "Pittsburgh, PA 15243", + "fields": [ + [ + "note", + "Broadens location range" + ] + ], + "url": "https://www.zillow.com/homedetails/637-Kelso-Rd-Pittsburgh-PA-15243/11360842_zpid/" + } + ] + }, + { + "type": "summary", + "title": "Bottom line", + "body": "The Ophelia is a studio-focused, lower-space but lower-commitment rental. Buying nearby gives more space and long-term equity potential. 523 N Euclid Ave and 725 N Beatty St are the most directly comparable buy-side options." + } + ] + } +} \ No newline at end of file diff --git a/assets/task_showcase/tasks/ophelia_pittsburgh/task.json b/assets/task_showcase/tasks/ophelia_pittsburgh/task.json new file mode 100644 index 0000000..bfe36d8 --- /dev/null +++ b/assets/task_showcase/tasks/ophelia_pittsburgh/task.json @@ -0,0 +1,11 @@ +{ + "task_id": "78ddd1aab59eebace5f6f523d90012aa6c871c54", + "short_id": "ophelia_pittsburgh", + "title": "The Ophelia, Pittsburgh โ€” availability watch", + "theme": "Real estate", + "cadence": "daily", + "level": "medium", + "website": "https://www.google.com", + "task_prompt": "Iโ€™m trying to decide whether renting at The Ophelia in Pittsburgh makes more sense than buying nearby, so could you help me look at both sides in the browser? Start on apartments.com and open The Opheliaโ€™s actual floor plan or availability page, then note at least two floor plans that are currently shown as available, including each planโ€™s name and the bedroom/bathroom setup, and leave that page open so I can look at the layouts myself. Since Pittsburgh winters are rough and Iโ€™m also thinking about car-related moving costs, go to WeatherTech and use their vehicle selector for a 2020 Toyota Highlander to find the floor mat and cargo liner options that fit, then open the cargo liner product page and keep that tab open as a reference. After that, use Google to find one LED emblem option for a 2023 Honda Civic, and click through to the actual product page so you can capture the product name and price rather than just a search snippet. Once you have those cost references, go to Zillow and search around the same Pittsburgh area for homes currently for sale that could realistically compete with renting there, then open five live listings in separate tabs and capture each propertyโ€™s address and listing URL so I can compare them side by side. In the end, give me a concise comparison that pulls together the two apartment floor plans, the WeatherTech cargo liner reference, the LED emblem option, and the five Zillow listings so I can get a real renting-versus-buying snapshot.", + "num_steps": 13 +} diff --git a/assets/task_showcase/tasks/pokemon_tcg/report.json b/assets/task_showcase/tasks/pokemon_tcg/report.json new file mode 100644 index 0000000..ff14418 --- /dev/null +++ b/assets/task_showcase/tasks/pokemon_tcg/report.json @@ -0,0 +1,62 @@ +{ + "sources": [ + { + "name": "Best Buy", + "url": "https://www.bestbuy.com/site/searchpage.jsp?st=pokemon+trading+card" + }, + { + "name": "Barnes & Noble", + "url": "https://www.barnesandnoble.com/s/pokemon+trading+card" + }, + { + "name": "Walmart", + "url": "https://www.walmart.com/search?q=pokemon+trading+card" + } + ], + "result": { + "headline": "In-stock Pokรฉmon TCG sweep", + "sections": [ + { + "type": "table", + "title": "Available items across retailers", + "columns": [ + "Retailer", + "Product", + "Price", + "Availability" + ], + "rows": [ + [ + "Best Buy", + "Pokรฉmon TCG โ€” Scarlet & Violet Journey Together Booster Bundle 6 Pk", + "$49.99", + "Add to cart" + ], + [ + "Best Buy", + "Pokรฉmon TCG โ€” Mega Evolution: Ascended Heroes Premium Poster Collection", + "$199.99", + "Add to cart" + ], + [ + "Barnes & Noble", + "How The Pokรฉmon TCG Became A Global Trading Card Game Phenomenon (book)", + "$21.99", + "In stock" + ], + [ + "Walmart", + "Pokรฉmon TCG โ€” Greninja ex Battle Deck", + "$23.13", + "Sold & shipped by Walmart.com" + ] + ] + }, + { + "type": "summary", + "title": "Cheapest in-stock pick", + "body": "Barnes & Noble โ€” โ€œHow The Pokรฉmon TCG Became A Global Trading Card Game Phenomenonโ€ at $21.99. Recommendation: buy from Barnes & Noble." + } + ] + } +} \ No newline at end of file diff --git a/assets/task_showcase/tasks/pokemon_tcg/task.json b/assets/task_showcase/tasks/pokemon_tcg/task.json new file mode 100644 index 0000000..2c1b92e --- /dev/null +++ b/assets/task_showcase/tasks/pokemon_tcg/task.json @@ -0,0 +1,11 @@ +{ + "task_id": "c6b29e8564a7ae86dc50a1f074bdc2b5abb3754a", + "short_id": "pokemon_tcg", + "title": "Pokรฉmon TCG availability sweep", + "theme": "Product inventory", + "cadence": "every 6h", + "level": "easy", + "website": "https://www.google.com", + "task_prompt": "I want to grab some Pokรฉmon trading cards pretty quickly, but only if theyโ€™re actually available to buy right now, so could you check a few retailers for me and keep the product pages open in separate tabs so I can visually compare them afterward? Start on BestBuy.com and search for Pokรฉmon trading card products, then find at least two items that show theyโ€™re in stock right now and note each productโ€™s full name, current price, and exactly what the availability message says on the page. After that, go to BarnesandNoble.com and find one Pokรฉmon card item thatโ€™s clearly in stock, open the actual product page so I can see the listing itself, and grab the title and listed price. Then head to Walmart.com, search for a Pokรฉmon card product, and make sure the one you pick is sold and shipped by Walmart rather than a marketplace seller, then record the product name and price and leave that product page open too. Once youโ€™ve got those pages, compare the in-stock options across all three stores and tell me which available item is the cheapest overall and where I should buy it. Then, make the tab for this option active.", + "num_steps": 12 +} diff --git a/assets/task_showcase/tasks/slickdeals/report.json b/assets/task_showcase/tasks/slickdeals/report.json new file mode 100644 index 0000000..7d5b030 --- /dev/null +++ b/assets/task_showcase/tasks/slickdeals/report.json @@ -0,0 +1,69 @@ +{ + "sources": [ + { + "name": "Slickdeals โ€” featured deal", + "url": "https://slickdeals.net/", + "note": "front-page best deal" + }, + { + "name": "CheapCharts โ€” iTunes deals", + "url": "https://www.cheapcharts.com/deals", + "note": "movie / TV / audiobook" + }, + { + "name": "Epic Games Store โ€” homepage", + "url": "https://store.epicgames.com/en-US/", + "note": "current featured sale" + }, + { + "name": "Split Fiction โ€” product page", + "url": "https://store.epicgames.com/en-US/p/split-fiction", + "note": "game-store option" + } + ], + "result": { + "headline": "Today's bargain roundup", + "sections": [ + { + "type": "summary", + "title": "Slickdeals featured deal", + "body": "Costco Members: Extra Savings on Select Apparel โ€” $10 Off When You Buy 3+" + }, + { + "type": "table", + "title": "CheapCharts picks (iTunes)", + "columns": [ + "Category", + "Title", + "Price" + ], + "rows": [ + [ + "Movie", + "Hats Off to Love", + "$4.99" + ], + [ + "TV season", + "Confess, Season 1", + "$4.99" + ], + [ + "Audiobook", + "The Return of Odin (Unabridged)", + "$4.99" + ] + ] + }, + { + "type": "list", + "title": "Epic Games Store โ€” confirmations", + "entries": [ + "Viewed Epic Games Store homepage", + "Viewed current featured sale: Discover: Epic Savings Sale", + "Viewed Split Fiction product page" + ] + } + ] + } +} \ No newline at end of file diff --git a/assets/task_showcase/tasks/slickdeals/task.json b/assets/task_showcase/tasks/slickdeals/task.json new file mode 100644 index 0000000..69bd36b --- /dev/null +++ b/assets/task_showcase/tasks/slickdeals/task.json @@ -0,0 +1,11 @@ +{ + "task_id": "0106b570440ffe4427d5e916f39ec986ab3de917", + "short_id": "slickdeals", + "title": "Slickdeals featured deal + CheapCharts iTunes picks", + "theme": "Deals roundup", + "cadence": "hourly", + "level": "easy", + "website": "https://www.google.com", + "task_prompt": "I want to make myself a quick bargain roundup and keep it grounded in deals that are actually live on the sites right now. Please start on Slickdeals and open whatever is currently being shown as the featured best deal, then grab the exact title and current price so I have a benchmark for what counts as a standout offer today; leave that deal page open in its own tab so I can look at it afterward. Then go to CheapCharts and browse the current iTunes deals to find one on-sale movie, one on-sale TV season, and one on-sale audiobook that feel like easy low-cost digital add-ons compared with the Slickdeals benchmark, and open each of those actual CheapCharts deal pages in separate tabs so I can visually compare them. After that, head to the Epic Games Store homepage, then open the current seasonal or featured sale page, and also pull up the product page for Split Fiction in another tab so I can include one game-store option alongside the media deals and have proof you actually viewed both Epic pages. When youโ€™re done, give me a concise roundup with the Slickdeals featured deal, the three CheapCharts picks labeled by category with prices, and a short confirmation that you viewed the Epic Games Store homepage, the current seasonal or featured sale page, and the Split Fiction product page.", + "num_steps": 9 +} diff --git a/assets/task_showcase/templates/dashboard.html b/assets/task_showcase/templates/dashboard.html new file mode 100644 index 0000000..50940db --- /dev/null +++ b/assets/task_showcase/templates/dashboard.html @@ -0,0 +1,99 @@ + + + + + Repeatable Task Showcase + + + +
+
Local Web-Agent Showcase ยท {{ tasks|length }} repeatable tasks
+

Task Dashboard

+

+ Six perfect-score odyssey runs whose underlying data refreshes on a + schedule. Open any card to see the agent's findings, sources, critical + points, and screenshots from the run. +

+ +
+ {% for t in tasks %} + +
+ {{ t.theme }} + {{ t.cadence }} +
+

{{ t.title }}

+
{{ t.task_prompt[:240] }}{% if t.task_prompt|length > 240 %}โ€ฆ{% endif %}
+ + {% endfor %} +
+
+ + diff --git a/assets/task_showcase/templates/task.html b/assets/task_showcase/templates/task.html new file mode 100644 index 0000000..d851346 --- /dev/null +++ b/assets/task_showcase/templates/task.html @@ -0,0 +1,265 @@ + + + + + {{ info.title }} โ€” Task Showcase + + + +
+ โ† All tasks +
+
+
{{ info.theme }} ยท {{ info.cadence or 'on demand' }}
+

{{ info.title }}

+
+
Updated {{ updated }}
+
+ + + + + + {% if sources %} +

Sources

+
+ {% for s in sources %} +
+
{{ s.name }}
+ {% if s.note %}
{{ s.note }}
{% endif %} + OK + Open โ†— +
+ {% endfor %} +
+ {% endif %} + + + {% if result.sections %} +

Result

+
+ {% for sec in result.sections %} + {% if sec.type == 'summary' %} +
+

{{ sec.title }}

+

{{ sec.body }}

+
+ {% elif sec.type == 'table' %} +
+

{{ sec.title }}

+ + + {% for c in sec.columns %}{% endfor %} + + + {% for row in sec.rows %} + {% for cell in row %}{% endfor %} + {% endfor %} + +
{{ c }}
{{ cell }}
+
+ {% elif sec.type == 'list' %} +
+

{{ sec.title }}

+
    + {% for item in sec['entries'] %}
  • {{ item }}
    • {% endfor %} +
+
+ {% elif sec.type == 'kv' %} +
+

{{ sec.title }}

+
+ {% for k, v in sec['entries'] %} +
{{ k }}
{{ v }}
+ {% endfor %} +
+
+ {% elif sec.type == 'cards' %} +
+

{{ sec.title }}

+
+ {% for c in sec['entries'] %} +
+
{{ c.title }}
+ {% if c.subtitle %}
{{ c.subtitle }}
{% endif %} + {% for k, v in c.fields or [] %} +
{{ k }}{{ v }}
+ {% endfor %} + {% if c.url %}Open โ†—{% endif %} +
+ {% endfor %} +
+
+ {% endif %} + {% endfor %} +
+ {% endif %} + + + {% if final_response %} +

Raw agent output

+
+
+ Show raw final response +

{{ final_response }}

+
+
+ {% endif %} + + +

Task

+
+
+ level{{ info.level }} + site{{ info.website }} + task_id{{ info.task_id }} +
+
{{ info.task_prompt }}
+
+
+ + diff --git a/assets/trajectory-compare.png b/assets/trajectory-compare.png new file mode 100644 index 0000000..4c0d948 Binary files /dev/null and b/assets/trajectory-compare.png differ diff --git a/assets/webwright_logo.svg b/assets/webwright_logo.svg new file mode 100644 index 0000000..1c60a79 --- /dev/null +++ b/assets/webwright_logo.svg @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Webwright + + + + + diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..fb4d113 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "webwright" +version = "0.1.0" +description = "Webwright: tiny SWE-style web agent harness" +requires-python = ">=3.10" +dependencies = [ + "httpx>=0.27", + "jinja2>=3.1", + "pydantic>=2.5", + "pyyaml>=6.0", + "rich>=13.0", + "typer>=0.12", + "playwright>=1.45", + "python-dotenv>=1.0", + "platformdirs>=4.0", +] + +[project.scripts] +webwright = "webwright.run.cli:app" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +webwright = ["config/*.yaml", "config/**/*.yaml"] diff --git a/skills/webwright/SKILL.md b/skills/webwright/SKILL.md new file mode 100644 index 0000000..c70e23c --- /dev/null +++ b/skills/webwright/SKILL.md @@ -0,0 +1,161 @@ +--- +name: webwright +description: Solve a user-specified web task code-as-action style by driving a local Playwright browser through one bash command at a time, saving screenshots and an action log into `final_runs/run_/`, and visually verifying the result. Use when the user asks to automate a web task (search, filter, form-fill, multi-step flow, data extraction) and wants reusable scripts plus screenshot evidence rather than a one-shot answer. +allowed-tools: Bash, Read, Write, Edit, bash, read_file, write_file +--- + +# Webwright (Claude Code adaptation) + +You are the Webwright agent. Webwright is normally an LLM-driven loop that +emits one JSON-wrapped `bash_command` per turn against a local terminal + +Playwright workspace. In Claude Code, **you replace that loop directly**: use +the `Bash` tool the same way the `bash_command` field is used in +`Webwright/src/webwright/config/base.yaml`. You do NOT need to wrap your +output in JSON โ€” that constraint only existed because the original harness +parsed model output. + +This skill keeps the *workspace contract* (plan.md, `final_runs/run_/` +folders, instrumented `final_script.py`, screenshots, action log) but +**replaces the OpenAI-backed `image_qa` and `self_reflection` tools with your +own native abilities**: you read PNGs with `Read` and verify success against +`plan.md` yourself. No `OPENAI_API_KEY` or other model API keys required. + +## Modes + +- **Default (one-shot).** `final_script.py` solves the task for the literal + values the user provided. Triggered by a plain prompt or by + `/webwright:run `. +- **CLI tool (parameterized).** `final_script.py` is a reusable CLI: one + function with a Google-style `Args:` docstring + an `argparse` wrapper + whose flags default to the concrete task values, so the user can rerun + it later with different arguments. Triggered by `/webwright:craft ` + or when the user asks to "parameterize", "make it reusable", "turn this + into a CLI", etc. See `reference/cli_tool_mode.md`. + +## Prerequisites (one-time) + +From the Webwright repo root: + +```bash +playwright install firefox +``` + +No API keys needed for this skill. + +## Workspace Contract + +Mirror what `base.yaml`'s `instance_template` requires: + +- Pick a `WORKSPACE_DIR` (e.g. `outputs//`) and work **only** there. + Keep all generated code, screenshots, logs, and notes inside it. +- The required final artifact path is `final_script.py`. +- Every clean execution of the final script lives in its own + `final_runs/run_/` folder. `` is an integer higher than any + existing `run_*` folder. +- Inside each run folder: + - `final_runs/run_/final_script.py` + - `final_runs/run_/screenshots/final_execution__.png` + - `final_runs/run_/final_script_log.txt` โ€” reset at the start of each + clean run; one `step action: ` line per + constraint-relevant interaction; the final datum (price, code, winner, + quote, etc.) printed at the end. +- Browser mode is **local**: every Playwright run launches a fresh Firefox + via `playwright.firefox.launch(headless=True)`. There is no persistent + browser state โ€” each script reconstructs state from scratch. (Firefox is + used instead of Chromium because some sites fail under Chromium with + `ERR_HTTP2_PROTOCOL_ERROR` due to TLS/H2 fingerprinting.) +- **Always use `viewport={"width": 1280, "height": 1800}`. Never call + `page.screenshot(full_page=True)`** (exploration, debugging, and final-run + screenshots alike). + +## Workflow + +1. **Plan.** Parse the task into a numbered checklist of *critical points* + โ€” every explicit constraint, filter, sort, selection, or required datum + that must be satisfied. Write it to `WORKSPACE_DIR/plan.md`: + + ```markdown + # Critical Points + - [ ] CP1: + - [ ] CP2: + ``` + + Each CP must be independently verifiable from a screenshot or a log line. + +2. **Explore.** Run scratch Playwright scripts (heredoc-style โ€” see + `reference/playwright_patterns.md`) to discover stable selectors and + confirm filter controls exist. Use `Read` on saved PNGs to inspect UI + state. Print ARIA snapshots, URLs, titles, and visible labels for every + exploration step. + +3. **Author `final_script.py`** in a fresh `final_runs/run_/`. Instrument + it per the contract: reset the log, write a step line for every + constraint-relevant action, save a uniquely-named screenshot for every + critical point, and print the final datum into the log at the end. + +4. **Execute** the final script once. Capture stdout/stderr. + +5. **Self-verify** (this replaces `webwright.tools.self_reflection`). Walk + `plan.md`: + - For each CP, identify a screenshot path AND/OR a log line that proves + it. `Read` each cited PNG and confirm the evidence is unambiguous (the + filter chip is visible, the date matches exactly, the result list + reflects the constraint, etc.). + - Tick the CP only when evidence is concrete. Be harsh with ambiguous, + occluded, or partially-applied states. + - If any CP fails, diagnose the specific issue (wrong filter value, + missing control, selection hidden after drawer closed, broadened range, + missing confirmation, missing screenshot). Fix `final_script.py`, + re-run inside `final_runs/run_/`, and re-verify. + +6. **Done.** Only when every CP in `plan.md` is checked off with cited + evidence. Report the final datum to the user. + +## Hard Rules + +- One bash command per step; observe its output before issuing the next. +- Use stable selectors and current-run evidence โ€” never guess UI state. +- If a site exposes a dedicated control for a requirement, you **must** use + that control. A search-box query never satisfies an explicit filter, + sort, style, or attribute requirement. +- Ranking language (`cheapest`, `best-selling`, `most reviewed`, + `highest-rated`, `lowest`, `latest`, โ€ฆ) must be grounded in the site's + actual sort/filter โ€” not in your own ordering of results. +- Numeric, date, quantity, and unit constraints are **exact**. Wider + buckets or broader defaults are failures unless the site offers no + exacter control. +- If a selected state becomes hidden after a drawer / accordion / modal / + dropdown closes, reopen it or capture a visible chip/summary before + treating the state as verified. +- Some required filters live behind expandable sections, drawers, + dropdowns, or mobile filter panels โ€” open them and inspect again before + declaring a filter unavailable. +- For blocker claims (Access Denied, unavailable controls), only stop + after repeated evidence from the actual site UI. +- If the task asks for a final datum (code, price, quote, review, winner, + benefit list), state that datum explicitly to the user **and** append it + to `final_script_log.txt`. +- Do **not** install extra packages with pip/apt. `playwright`, `httpx`, + `pydantic`, etc. are already installed. +- Once `final_script.py` exists, prefer incremental edits (`Edit`) over + rewriting the whole file. + +## Reference Files + +- `reference/playwright_patterns.md` โ€” browser-launch heredoc skeleton, + `aria_snapshot()` recipes, screenshot naming, log format. +- `reference/workflow.md` โ€” detailed walk-through of plan โ†’ explore โ†’ + final โ†’ self-verify, plus the completion checklist. +- `reference/cli_tool_mode.md` โ€” contract for CLI tool mode + (`# Parameters` table, reusable function + argparse, import-safety, + `step 0 params:` log line, completion gate). + +## Slash Commands + +Optional shortcuts under `commands/`: + +- `/webwright:run ` โ€” default one-shot mode. +- `/webwright:craft ` โ€” CLI tool mode. + +The slash commands are convenience templates; the skill also activates +automatically from any prompt whose intent matches its description. diff --git a/skills/webwright/commands/craft.md b/skills/webwright/commands/craft.md new file mode 100644 index 0000000..82f2a82 --- /dev/null +++ b/skills/webwright/commands/craft.md @@ -0,0 +1,62 @@ +--- +description: Craft a reusable Webwright CLI tool by parameterizing a web task. +argument-hint: +--- + +You are operating as the Webwright agent in **CLI tool mode**. First read +the `SKILL.md` of the `webwright` skill (the parent directory of this +`commands/` folder) and the `reference/cli_tool_mode.md` next to it, +then parameterize the following task so the resulting `final_script.py` +can be re-run later with different argument values: + +$ARGUMENTS + +Steps: + +1. **Identify parameters.** Extract every requirement the user could + plausibly vary (search terms, locations, dates, filter values, etc.). + Items truly fixed for the site (start URL, site name, selector + strategy) are NOT parameters โ€” keep them hard-coded. + +2. **Write `plan.md`.** Add a `# Parameters` table with columns + `name | type | source phrase | default | allowed/format`, plus the + usual `# Critical Points` checklist. Defaults must equal the + concrete task values so `python final_script.py` (no args) reproduces + the task. + +3. **Author `final_script.py`** in a fresh `final_runs/run_/`: + - One reusable function named after the task domain + (e.g. `def search_(arg_a, arg_b, ...): ...`). + - Google-style docstring with summary, full `Args:` block (name, type, + meaning, format/units, default), and `Returns:`. + - `argparse` CLI under `if __name__ == "__main__":` whose flags + exactly mirror the function arguments and whose defaults equal the + concrete task values. + - **Side-effect-free at import time** โ€” no browser launch, no network + call, no file write at module top-level. + - First log line after reset must be + `step 0 params: = = ...`. + - Same instrumentation as default mode: viewport 1280ร—1800, headless + local Firefox, no `full_page=True`, screenshots and final datum + saved into the run folder. + +4. **Reproduce the task with no arguments.** Run + `python final_runs/run_/final_script.py` and confirm it succeeds + end-to-end. + +5. **Import-safety smoke test.** Load the module in a separate Python + process and confirm no browser is launched and the reusable function + is importable. + +6. **Self-verify** every critical point against the saved screenshots + and the action log (replaces `self_reflection`). If any CP fails, + diagnose, fix the script (preserving the CLI shape), re-run inside + `final_runs/run_/`, and re-verify. + +7. **Show the user `--help`.** End by running + `python final_runs/run_/final_script.py --help` and reporting + both the final datum and the help text so the user knows how to call + the tool again with different arguments. + +Refer to `reference/cli_tool_mode.md` for the complete contract and +`reference/playwright_patterns.md` for the Playwright skeleton. diff --git a/skills/webwright/commands/run.md b/skills/webwright/commands/run.md new file mode 100644 index 0000000..515f16b --- /dev/null +++ b/skills/webwright/commands/run.md @@ -0,0 +1,34 @@ +--- +description: Run a one-shot web task with the Webwright Playwright workflow. +argument-hint: +--- + +You are operating as the Webwright agent. Solve the following web task +code-as-action style by driving a local Playwright browser through one +bash command at a time, saving screenshots and an action log into +`final_runs/run_/`, and visually verifying the result. + +Task: + +$ARGUMENTS + +For the full operating contract, first read the `SKILL.md` of the +`webwright` skill (the parent directory of this `commands/` folder). +Then follow the standard Webwright workflow: + +1. Pick a `WORKSPACE_DIR` and write `plan.md` with a numbered list of + critical points. +2. Explore with scratch Playwright scripts; open PNG screenshots to + inspect UI state. +3. Author and run an instrumented `final_script.py` inside a fresh + `final_runs/run_/` (viewport 1280ร—1800, headless local Firefox, + no `full_page=True`). +4. Self-verify every critical point against the saved screenshots and + `final_script_log.txt`. Diagnose, fix, and re-run in a new + `run_/` until every CP is ticked with cited evidence. +5. Report the final datum (price, code, winner, โ€ฆ) verbatim. + +Refer to `reference/playwright_patterns.md` and `reference/workflow.md` +(under the same skill directory) for details. Do **not** use CLI tool +mode for this task. + diff --git a/skills/webwright/reference/cli_tool_mode.md b/skills/webwright/reference/cli_tool_mode.md new file mode 100644 index 0000000..3d64cad --- /dev/null +++ b/skills/webwright/reference/cli_tool_mode.md @@ -0,0 +1,187 @@ +# CLI Tool Mode + +Default Webwright runs (`/webwright:run`, plain prompt) produce a one-shot +`final_script.py` that solves the task for the literal values the user +provided. **CLI tool mode** (`/webwright:craft`) instead produces a +**reusable, parameterized CLI tool**: the same script can be re-run later +with different argument values to perform the same kind of task. + +This mode is adapted from `webwright/src/webwright/config/crafted_cli.yaml`'s +"Final-Script Shape (CLI Tool, MANDATORY)" contract. The OpenAI-backed +`self_reflection` gate is replaced by your own self-verification against +`plan.md`. + +## When to use + +Trigger CLI tool mode when: + +- the user invokes `/webwright:craft โ€ฆ`, or +- the user says "make it reusable", "parameterize", "turn this into a CLI", + "I want to call this again with different X", or similar. + +Otherwise, stay in default one-shot mode. + +## `plan.md` โ€” add a `# Parameters` section + +Before writing the script, identify every requirement the user could +plausibly vary and list them in `plan.md` **in addition to** the usual +`# Critical Points` checklist: + +```markdown +# Task + + +# Parameters +| name | type | source phrase from task | default | allowed / format | +|---------|------|-------------------------|-------------|-------------------------| +| | str | "..." | "" | | +| | int | "..." | | | +| | str | "..." | "" | | + +# Critical Points +- [ ] CP1: ... +- [ ] CP2: ... +``` + +Rules: + +- Every entry in `# Parameters` must (a) become a function argument and + (b) become an `argparse --flag` with the listed default. +- Items that are truly fixed for the site (start URL, site name, selector + strategy) are NOT parameters โ€” keep them hard-coded. +- Defaults reproduce the original task exactly. Running + `python final_script.py` with no arguments must reproduce the task. +- Critical Points are still required; they are the verification contract. + +## `final_script.py` โ€” required shape + +1. **One reusable function** named after the task domain. Examples: + - `def search_(arg_a, arg_b, ...): ...` + - `def lookup_(query, filters): ...` + +2. **Google-style docstring** with summary, full `Args:` block, and + `Returns:`. Each `Args:` entry documents: + - the argument name and type, + - what it represents in the task domain, + - accepted format / units / allowed values, + - the default (mirroring the `# Parameters` table). + + ```python + def search_(arg_a: str, arg_b: int, arg_c: str) -> dict: + """. + + Args: + arg_a: ; . + Default: "". + arg_b: ; . + Default: . + arg_c: ; . + Default: "". + + Returns: + dict with keys ```` (), ```` (), + """ + ``` + +3. **`argparse` CLI** under `if __name__ == "__main__":`. Every function + argument has a matching `--` flag with `type=`, `help=` (copied + from the docstring), and `default=` equal to the concrete task value: + + ```python + if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser( + description=search_.__doc__.splitlines()[0]) + parser.add_argument("--arg-a", dest="arg_a", type=str, + default="", + help="") + parser.add_argument("--arg-b", dest="arg_b", type=int, + default=, + help="") + parser.add_argument("--arg-c", dest="arg_c", type=str, + default="", + help="") + args = parser.parse_args() + result = asyncio.run(_run(**vars(args))) + print(result) + ``` + +4. **Side-effect-free at import time.** No browser launch, no network + call, no file write at module top-level. The reusable function must be + importable from another Python process without triggering a run. + +5. **Action-log parameter echo.** The first line written to + `final_script_log.txt` after reset MUST be a `step 0 params: ...` + line listing every resolved argument as `name=value` pairs, e.g.: + + ``` + step 0 params: arg_a= arg_b= arg_c= + ``` + + so the resolved inputs are visible in any verification pass. + +6. Same instrumentation as default mode: viewport 1280ร—1800, headless + local Firefox, no `full_page=True`, screenshots saved as + `final_runs/run_/screenshots/final_execution__.png`, + final datum appended to `final_script_log.txt`. + +## Verification (replaces `self_reflection`) + +In addition to the default self-verification (every CP in `plan.md` +ticked with cited screenshot/log evidence), CLI mode requires: + +1. **Reproduce the task with no arguments.** Inside a fresh + `final_runs/run_/`: + + ```bash + cd final_runs/run_ && python final_script.py + ``` + + The run must succeed end-to-end and produce the expected screenshots + and `step 0 params: ...` log line. + +2. **Import-safety smoke test.** From any other directory: + + ```bash + python -c "import importlib.util, pathlib; \ + spec = importlib.util.spec_from_file_location('fs', 'final_runs/run_/final_script.py'); \ + m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m); \ + print([n for n in dir(m) if not n.startswith('_')])" + ``` + + This must complete instantly with no browser launch and print the + reusable function's name. + +3. **Optional second run with a different argument value.** Demonstrates + parameterization actually works. Run inside `final_runs/run__alt/` + (or just save its log/screenshot folder there). Skip only if the + alternate value would clearly fail (e.g. an unsupported value on the + target site). + +4. **Print `--help`.** End by showing the user: + + ```bash + python final_runs/run_/final_script.py --help + ``` + +## Completion gate (CLI mode) + +Set the task complete only when **all** are true: + +1. `plan.md` contains both `# Parameters` (with name, type, source phrase, + default, allowed/format) and `# Critical Points` checklists. +2. `final_script.py` defines exactly one reusable function with a + Google-style `Args:` docstring covering every parameter. +3. Every `# Parameters` entry maps 1-to-1 to a function argument **and** + an argparse `--flag` whose default equals the concrete task value. +4. The script is import-safe (smoke test passes). +5. `python final_script.py` (no args) inside `final_runs/run_/` + reproduced the task; all CPs verified against saved screenshots and + the action log. +6. `step 0 params: ...` line is present in `final_script_log.txt`. +7. The user has seen the final datum **and** the `--help` output so they + know how to call the tool again with different arguments. + +If any of those is false, do not declare done โ€” diagnose, fix the script +(preserving the CLI shape), re-run inside the next `run_/`, and +re-verify. diff --git a/skills/webwright/reference/playwright_patterns.md b/skills/webwright/reference/playwright_patterns.md new file mode 100644 index 0000000..5ba1565 --- /dev/null +++ b/skills/webwright/reference/playwright_patterns.md @@ -0,0 +1,181 @@ +# Playwright Patterns + +These are the canonical heredoc patterns the Webwright agent uses. In Claude +Code you run them via the `Bash` tool โ€” no JSON wrapping, no escaping +gymnastics, just one bash command per turn. + +## Browser launch skeleton (local mode) + +The Webwright skill uses **Playwright Firefox** as its default engine. Some +sites (e.g. cars.com / other Akamai-protected sites) reject Playwright +Chromium with `ERR_HTTP2_PROTOCOL_ERROR` due to TLS/H2 fingerprinting, but +load cleanly under Firefox. Run `playwright install firefox` once before +the first task. + +```bash +python - <<'PY' +import asyncio +import os +from pathlib import Path + +from playwright.async_api import async_playwright + +WORKSPACE = Path(os.environ.get("WORKSPACE_DIR", ".")) +SCREENSHOTS = WORKSPACE / "screenshots" +SCREENSHOTS.mkdir(parents=True, exist_ok=True) + +async def main(): + async with async_playwright() as playwright: + browser = await playwright.firefox.launch(headless=True) + context = await browser.new_context(viewport={"width": 1280, "height": 1800}) + page = await context.new_page() + + await page.goto("", wait_until="domcontentloaded") + await page.screenshot(path=str(SCREENSHOTS / "explore_1_start.png")) + + print("URL:", page.url) + print("TITLE:", await page.title()) + + # Inspect the region you care about with an ARIA snapshot + snapshot = await page.locator("body").aria_snapshot() + print("ARIA:", snapshot) + + await browser.close() + +asyncio.run(main()) +PY +``` + +Rules: + +- **Always** set `viewport={"width": 1280, "height": 1800}`. +- **Never** call `page.screenshot(full_page=True)` โ€” exploration, debugging, + and final-run screenshots alike. +- Each Playwright run is fresh: navigate from the start URL, reapply + filters, reconstruct state in code. There is no persistent session. + +## Targeting elements with role + name + +```python +await page.get_by_role("button", name="Filters").click() +await asyncio.sleep(1) + +# Snapshot the *parent* of the control to see siblings/options +panel = page.get_by_role("button", name="Filters").first.locator("..") +print(await panel.aria_snapshot()) + +await page.get_by_role("checkbox", name="BMW").check() +await asyncio.sleep(1) +``` + +If a selected state becomes hidden after a drawer/dropdown closes, reopen +it before capturing the verification screenshot. + +## Prefer interactive form filling over deep-link URLs + +When a task requires parameterizing a search (locations, dates, filters, +query strings), **drive the on-page form interactively** rather than +constructing a deep-link URL with the parameters baked into the query +string. Deep links are convenient for the one specific case the agent +explored, but they are brittle as a CLI surface: + +- Sites silently drop parameters they cannot parse, leaving downstream + fields blank. +- URL parsers vary by locale, A/B bucket, and signed-in state. +- A working deep link for one input set tells you nothing about whether + another set will populate. + +Interactive filling using the same controls a human would click is the +most reliable strategy across input variations. Make it the **primary** +path in the final script; only use a deep link as an opportunistic +shortcut, and always verify the form state afterwards and fall back to +interactive filling when any field is empty or wrong. + +```python +# After navigating, read the visible form state and decide. +form_state = await page.locator("input[aria-label]").evaluate_all( + "els => els.map(e => ({label: e.getAttribute('aria-label'), " + "value: e.value, hidden: e.offsetParent === null}))" +) +if not form_is_fully_populated(form_state, expected): + # Type into each field, pick from the suggestion list, fill grouped + # inputs via their shared modal (Tab between siblings to keep one + # modal open), then click the submit control. + await fill_form_interactively(page, expected) +``` + +Guidelines for the interactive path: + +- Use `get_by_role` / `aria-label` selectors, not brittle CSS classes. +- Type the value, wait for the suggestion listbox, then click the option + whose text contains the canonical token for the input. +- For paired fields rendered inside a single modal (date range pickers, + stepper groups, etc.), open the modal **once** and `Tab` between fields + instead of clicking each input separately โ€” clicking the second input + while the modal is open often gets blocked by the modal's own overlay. +- After filling, click the explicit submit control rather than relying on + auto-submit. +- Re-read the form state and assert each checkpoint (CP1..CPn) before + proceeding to results extraction. + +## Final-script instrumentation + +`final_runs/run_/final_script.py` must: + +- write to `final_runs/run_/screenshots/final_execution__.png`, +- reset and append to `final_runs/run_/final_script_log.txt`, +- print the final datum at the end of the log. + +```python +import asyncio, os +from pathlib import Path +from playwright.async_api import async_playwright + +RUN_DIR = Path(__file__).parent +SCREENSHOTS = RUN_DIR / "screenshots" +SCREENSHOTS.mkdir(parents=True, exist_ok=True) +LOG = RUN_DIR / "final_script_log.txt" +LOG.write_text("") # reset + +def log(step: int, msg: str) -> None: + line = f"step {step} action: {msg}\n" + LOG.open("a").write(line) + print(line, end="") + +async def main(): + async with async_playwright() as playwright: + browser = await playwright.firefox.launch(headless=True) + context = await browser.new_context(viewport={"width": 1280, "height": 1800}) + page = await context.new_page() + + await page.goto("", wait_until="domcontentloaded") + await page.screenshot(path=str(SCREENSHOTS / "final_execution_1_open_start_page.png")) + log(1, "open start page") + + # ... apply CP1, screenshot, log ... + # ... apply CP2, screenshot, log ... + + # End of run: capture the final datum visibly and in the log + final_value = "" + with LOG.open("a") as f: + f.write(f"\nFINAL_RESPONSE: {final_value}\n") + + await browser.close() + +asyncio.run(main()) +``` + +## Inspection commands + +```bash +# Latest run tree + log +ls -R final_runs/run_ +cat final_runs/run_/final_script_log.txt + +# Quick file read +sed -n '1,220p' final_runs/run_/final_script.py +``` + +For visual checks, use the `Read` tool on individual PNG files inside +`final_runs/run_/screenshots/` rather than calling an external image-QA +service. diff --git a/skills/webwright/reference/workflow.md b/skills/webwright/reference/workflow.md new file mode 100644 index 0000000..b4be97d --- /dev/null +++ b/skills/webwright/reference/workflow.md @@ -0,0 +1,107 @@ +# Workflow + +Detailed expansion of the six-step Webwright loop, adapted for Claude Code. +The original loop relied on `webwright.tools.image_qa` for visual QA and +`webwright.tools.self_reflection` for the final verdict. Both are replaced +here by your native abilities (`Read` on PNG files + reasoning against +`plan.md`). No `OPENAI_API_KEY` is required. + +## 1. Plan + +Parse the task into critical points (CPs) and write `WORKSPACE_DIR/plan.md`: + +```markdown +# Task + + +# Critical Points +- [ ] CP1: +- [ ] CP2: ... +``` + +Rules for CPs: + +- One CP per independently verifiable requirement. +- Numeric, date, quantity, and unit CPs must be exact. +- Ranking CPs ("cheapest", "best-selling", "highest-rated", โ€ฆ) must + reference the site's actual sort/filter control. +- If the task asks for a final datum, make it its own CP + (e.g. `CP5: Record the displayed cheapest economy fare`). + +## 2. Explore + +Goal: discover stable selectors, confirm every required filter control +exists, and identify how to capture evidence for each CP. + +- Run scratch Playwright scripts (see `playwright_patterns.md`) inside + `WORKSPACE_DIR/`. Save scratch PNGs under `WORKSPACE_DIR/screenshots/` + (separate from `final_runs/`). +- Print URL, title, and `aria_snapshot()` for the region of interest at + every step. +- Use `Read` on saved PNGs to confirm UI state when ARIA evidence is + ambiguous. +- If a filter looks unavailable, expand drawers / accordions / mobile + filter panels and inspect again before concluding it doesn't exist. +- A search-box query never substitutes for a dedicated filter control. + +## 3. Author `final_script.py` + +Create a fresh `final_runs/run_/` (use the next integer above any +existing `run_*`) and place `final_script.py` inside it. Instrument per +`playwright_patterns.md`: + +- viewport 1280ร—1800, headless local Firefox, no `full_page`; +- one `final_execution__.png` per CP; +- one `step action: ` log line per + constraint-relevant interaction; +- the final datum printed into `final_script_log.txt` at the end. + +Each screenshot should map to a CP from `plan.md` so verification is +trivial. + +## 4. Execute + +Run the script once. If it crashes, fix it inside the same run folder and +re-execute โ€” but if a partial run already produced screenshots that don't +match the fixed flow, delete them so the run folder reflects a single +clean execution. + +## 5. Self-verify (replaces `self_reflection`) + +For every CP in `plan.md`: + +1. Identify the screenshot(s) and/or log line that provide evidence. +2. `Read` each cited PNG. +3. Confirm the evidence is **unambiguous**: + - Filter chip / selected state visibly applied (not hidden behind a + closed drawer); + - Numeric / date values match exactly (not broadened); + - Sort applied via the site's control (not implied by result order); + - Required submit / search / apply action visibly taken; + - Final datum legibly displayed. +4. Tick the CP only when the evidence is concrete. Be harsh on partial, + occluded, or ambiguous states. + +If any CP fails, diagnose the *specific* issue โ€” wrong filter value, +missing control, hidden chip, broadened range, missing confirmation, +missing screenshot, etc. Fix `final_script.py`, run it again inside +`final_runs/run_/`, and re-verify against `plan.md`. + +Empty result sets are acceptable when the correct filters were demonstrably +applied. + +## 6. Done + +Stop only when **all** of the following are true: + +1. `plan.md` exists with every CP enumerated as a checklist item. +2. `final_runs/run_/final_script.py` ran cleanly from scratch and + produced `final_script_log.txt` plus all CP screenshots. +3. Every CP is checked off with a cited screenshot and/or log line. +4. The final datum (if the task asked for one) is reported to the user + verbatim and is also present in `final_script_log.txt`. +5. `ls -R final_runs/run_` and `cat final_runs/run_/final_script_log.txt` + show the expected artifacts. + +If any of those is false, do not declare done โ€” diagnose, fix, and re-run +in a new `run_/`. diff --git a/src/webwright/__init__.py b/src/webwright/__init__.py new file mode 100644 index 0000000..12b1da4 --- /dev/null +++ b/src/webwright/__init__.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any, Protocol + +try: + import dotenv +except ModuleNotFoundError: + class _DotenvShim: + @staticmethod + def load_dotenv(*args, **kwargs): + return False + dotenv = _DotenvShim() +try: + from platformdirs import user_config_dir +except ModuleNotFoundError: + def user_config_dir(appname: str) -> str: + return str(Path.home() / ".config" / appname) + +__version__ = "0.1.0" + +package_dir = Path(__file__).resolve().parent +global_config_dir = Path( + os.getenv("MSWEBA_GLOBAL_CONFIG_DIR") or user_config_dir("webwright") +) +global_config_dir.mkdir(parents=True, exist_ok=True) +global_config_file = global_config_dir / ".env" +dotenv.load_dotenv(dotenv_path=global_config_file) + + +class Model(Protocol): + config: Any + + def __call__(self, messages: list[dict[str, Any]], **kwargs) -> str: ... + + def query(self, messages: list[dict[str, Any]], **kwargs) -> dict[str, Any]: ... + + def format_message(self, **kwargs) -> dict[str, Any]: ... + + def format_observation_messages( + self, + message: dict[str, Any], + outputs: list[dict[str, Any]], + template_vars: dict[str, Any] | None = None, + ) -> list[dict[str, Any]]: ... + + def get_template_vars(self, **kwargs) -> dict[str, Any]: ... + + def serialize(self) -> dict[str, Any]: ... + + +class Environment(Protocol): + config: Any + + def prepare(self, **kwargs) -> None: ... + + def execute(self, action: dict[str, Any], cwd: str = "") -> dict[str, Any]: ... + + def get_template_vars(self, **kwargs) -> dict[str, Any]: ... + + def serialize(self) -> dict[str, Any]: ... + + def close(self) -> None: ... + + +class Agent(Protocol): + config: Any + + def run(self, task: str, **kwargs) -> dict[str, Any]: ... + + def save(self, path: Path | None, *extra_dicts) -> dict[str, Any]: ... + + +__all__ = [ + "Agent", + "Environment", + "Model", + "__version__", + "global_config_dir", + "global_config_file", + "package_dir", +] diff --git a/src/webwright/agents/__init__.py b/src/webwright/agents/__init__.py new file mode 100644 index 0000000..fcfad5b --- /dev/null +++ b/src/webwright/agents/__init__.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +import copy +import importlib + +from webwright import Agent, Environment, Model + +_AGENT_MAPPING = { + "default": "webwright.agents.default.DefaultAgent", +} + + +def get_agent_class(spec: str) -> type[Agent]: + full_path = _AGENT_MAPPING.get(spec, spec) + module_name, class_name = full_path.rsplit(".", 1) + module = importlib.import_module(module_name) + return getattr(module, class_name) + + +def get_agent(model: Model, env: Environment, config: dict, *, default_type: str = "default") -> Agent: + copied = copy.deepcopy(config) + agent_class = get_agent_class(copied.pop("agent_class", default_type)) + return agent_class(model, env, **copied) diff --git a/src/webwright/agents/default.py b/src/webwright/agents/default.py new file mode 100644 index 0000000..d05ea86 --- /dev/null +++ b/src/webwright/agents/default.py @@ -0,0 +1,467 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path +from typing import Any + +from jinja2 import StrictUndefined, Template +from pydantic import BaseModel + +from webwright import Environment, Model, __version__ +from webwright.exceptions import FormatError, InterruptAgentFlow, LimitsExceeded +from webwright.utils.serialize import recursive_merge + + +DEFAULT_SUMMARY_USER_PROMPT = """You are about to have your working context compacted to save tokens. + +Write a concise but COMPLETE summary of everything relevant from the conversation above so that a fresh +agent with only this summary (plus the original system prompt and task instructions) can continue the +task without losing progress. Include: + +- The original task goal and all critical points / constraints. +- The workspace directory and key file paths (plan.md, self_reflect_config.json, final_script.py, final_runs/). +- Which critical points have been satisfied, which are still open, and any known blockers. +- Key findings from prior exploration (working selectors, URLs, ARIA labels, pitfalls to avoid). +- The latest final_runs/run_/ state, most recent self_reflection verdict, and the next action to take. + +Write the summary as plain prose and bullet lists. Do NOT issue a new bash_command. Do NOT set done=true. +Put the entire summary in the `thought` field (or equivalent text field) and leave action fields empty.""" + + +class AgentConfig(BaseModel): + system_template: str + instance_template: str + step_limit: int = 15 + debug_log: bool = True + attach_instance_template_after_observation: bool = False + attach_plan_md_after_observation: bool = False + require_self_reflection_success: bool = False + summary_every_n_steps: int = 0 + summary_user_prompt: str = DEFAULT_SUMMARY_USER_PROMPT + # Strip the ARIA snapshot payload from observation messages older than the last N + # to bound context growth in browser-driven modes. Any value <= 0 disables pruning + # (default). Opt in per config (e.g. local_browser.yaml sets this to 1). + keep_last_n_observations: int = -1 + output_path: Path | None = None + + +def _sanitize_message_for_disk(message: dict[str, Any]) -> dict[str, Any]: + cloned = copy.deepcopy(message) + content = cloned.get("content") + if isinstance(content, list): + for part in content: + if isinstance(part, dict) and part.get("type") == "input_image": + part["image_url"] = "" + return cloned + + +def _observation_for_markdown(observation: dict[str, Any], *, model_usage: dict[str, Any] | None = None) -> dict[str, Any]: + cloned = copy.deepcopy(observation) + cloned.pop("aria_snapshot", None) + if model_usage: + cloned["model_usage"] = copy.deepcopy(model_usage) + return cloned + + +def _action_text(action: dict[str, Any]) -> str: + return str(action.get("bash_command") or action.get("command") or action.get("python_code") or "").strip() + + +def _python_action_text(action: dict[str, Any]) -> str: + return str(action.get("python_code") or "").strip() + + +def _markdown_code_fence_language(*, bash_command_text: str, python_code_text: str) -> str: + if bash_command_text: + return "bash" + if python_code_text: + return "python" + return "" + + +class DefaultAgent: + def __init__(self, model: Model, env: Environment, *, config_class: type = AgentConfig, **kwargs): + self.config = config_class(**kwargs) + self.messages: list[dict[str, Any]] = [] + self.model = model + self.env = env + self.extra_template_vars: dict[str, Any] = {} + self.n_calls = 0 + self.n_format_errors = 0 + + def _debug_dir(self) -> Path | None: + if self.config.output_path is None: + return None + return self.config.output_path.parent / "debug" + + def _write_debug_step_artifact( + self, + *, + step_index: int, + assistant_message: dict[str, Any], + outputs: list[dict[str, Any]] | None = None, + ) -> None: + if not self.config.debug_log: + return + debug_dir = self._debug_dir() + if debug_dir is None: + return + steps_dir = debug_dir / "steps" + steps_dir.mkdir(parents=True, exist_ok=True) + + extra = assistant_message.get("extra", {}) + actions = extra.get("actions", []) + action_text = "\n\n".join(_action_text(action) for action in actions if _action_text(action)) + python_code_text = "\n\n".join( + _python_action_text(action) for action in actions if _python_action_text(action) + ) + bash_command_text = "\n\n".join( + str(action.get("bash_command", "")).strip() + for action in actions + if str(action.get("bash_command", "")).strip() + ) + code_fence_language = _markdown_code_fence_language( + bash_command_text=bash_command_text, + python_code_text=python_code_text, + ) + payload = { + "step": step_index, + "thought": assistant_message.get("content", ""), + "python_code": python_code_text, + "bash_command": bash_command_text, + "command_text": action_text, + "raw_response": extra.get("raw_response", {}), + "done": extra.get("done", False), + "final_response": extra.get("final_response", ""), + "outputs": outputs or [], + } + (steps_dir / f"step_{step_index:04d}.json").write_text(json.dumps(payload, indent=2)) + + summary_path = debug_dir / "steps.md" + with summary_path.open("a", encoding="utf-8") as handle: + handle.write(f"## Step {step_index}\n\n") + # Attach the model input only for the first step + if step_index == 1: + user_input_text = "" + for msg in reversed(self.messages): + if msg.get("role") == "user": + content = msg.get("content", "") + if isinstance(content, list): + # Multi-part message: join text parts + parts = [p.get("text", "") for p in content if isinstance(p, dict) and p.get("type") in ("text", "input_text")] + user_input_text = "\n".join(p for p in parts if p) + else: + user_input_text = str(content) + break + if user_input_text: + handle.write("### Model Input\n\n") + handle.write(f"{user_input_text}\n\n") + handle.write("### Thought\n\n") + handle.write(f"{payload['thought']}\n\n") + handle.write("### Generated Code\n\n") + handle.write(f"```{code_fence_language}\n") + handle.write(f"{payload['command_text']}\n") + handle.write("```\n\n") + if outputs: + observation = outputs[0].get("observation", {}) + markdown_observation = _observation_for_markdown( + observation, + model_usage=extra.get("usage"), + ) + handle.write("### Observation\n\n") + handle.write("```json\n") + handle.write(f"{json.dumps(markdown_observation, indent=2)}\n") + handle.write("```\n\n") + + def get_template_vars(self, **kwargs) -> dict[str, Any]: + return recursive_merge( + self.config.model_dump(), + self.env.get_template_vars(), + self.model.get_template_vars(), + {"n_model_calls": self.n_calls}, + self.extra_template_vars, + kwargs, + ) + + def _render_template(self, template: str) -> str: + return Template(template, undefined=StrictUndefined).render(**self.get_template_vars()) + + def _plan_md_message(self) -> dict[str, Any] | None: + workspace_dir = self.get_template_vars().get("workspace_dir") + if not workspace_dir: + return None + plan_path = Path(workspace_dir) / "plan.md" + if not plan_path.exists() or not plan_path.is_file(): + return None + plan_text = plan_path.read_text(encoding="utf-8").strip() + if not plan_text: + return None + return self.model.format_message(role="user", content=f"Current plan.md:\n\n{plan_text}") + + def _self_reflection_gate_error(self) -> str | None: + """Return an error string if done=true should be blocked pending judge success.""" + if not self.config.require_self_reflection_success: + return None + return self._tool_gate_error() + + def _tool_gate_error(self) -> str | None: + """Require final_runs/run_/self_reflect_result.json with predicted_label == 1.""" + workspace_dir = self.get_template_vars().get("workspace_dir") + if not workspace_dir: + return ( + "Completion blocked: require_self_reflection_success is enabled but no workspace_dir is " + "available. Cannot locate final_runs/run_/self_reflect_result.json. Do not set done=true." + ) + final_runs_dir = Path(workspace_dir) / "final_runs" + if not final_runs_dir.is_dir(): + return ( + "Completion blocked: no final_runs/ directory exists yet. You must run final_script.py " + "in a final_runs/run_/ folder and then run " + "`python -m webwright.tools.self_reflection --config self_reflect_config.json " + "--workspace-dir \"{0}\" --output final_runs/run_/self_reflect_result.json` with " + "predicted_label == 1 before setting done=true." + ).format(workspace_dir) + run_dirs: list[tuple[int, Path]] = [] + for entry in final_runs_dir.iterdir(): + if not entry.is_dir() or not entry.name.startswith("run_"): + continue + suffix = entry.name[len("run_"):] + try: + run_id = int(suffix) + except ValueError: + continue + run_dirs.append((run_id, entry)) + if not run_dirs: + return ( + "Completion blocked: final_runs/ contains no run_/ folders. Create " + "final_runs/run_/, execute final_script.py there, then run self_reflection and " + "only set done=true after self_reflect_result.json reports predicted_label == 1." + ) + run_dirs.sort(key=lambda item: item[0]) + latest_run_id, latest_run_dir = run_dirs[-1] + judge_path = latest_run_dir / "self_reflect_result.json" + if not judge_path.is_file(): + return ( + f"Completion blocked: {judge_path} does not exist. Run " + f"`python -m webwright.tools.self_reflection --config self_reflect_config.json " + f"--workspace-dir \"{workspace_dir}\" --output {judge_path}` against the latest run " + f"(run_{latest_run_id}) and only set done=true after it exits 0 with " + f"predicted_label == 1." + ) + try: + judge_data = json.loads(judge_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + return ( + f"Completion blocked: could not parse {judge_path}: {exc}. Re-run self_reflection " + f"against run_{latest_run_id} and only set done=true after predicted_label == 1." + ) + predicted_label = judge_data.get("predicted_label") + if predicted_label != 1: + return ( + f"Completion blocked: {judge_path} has predicted_label={predicted_label!r} " + f"(expected 1). Diagnose the failure from self_reflect_result.json, fix final_script.py, " + f"re-run it in a new final_runs/run_{latest_run_id + 1}/ folder, and re-run " + f"self_reflection. Only set done=true after self_reflection exits 0 with " + f"predicted_label == 1." + ) + return None + + def add_messages(self, *messages: dict[str, Any]) -> list[dict[str, Any]]: + self.messages.extend(messages) + self._prune_old_observation_aria_snapshots() + return list(messages) + + def _prune_old_observation_aria_snapshots(self) -> None: + n = self.config.keep_last_n_observations + if n <= 0: + return + obs_indices = [ + i for i, m in enumerate(self.messages) + if m.get("extra", {}).get("observation") + ] + if len(obs_indices) <= n: + return + placeholder = "(ARIA snapshot pruned; see most recent observation)" + for idx in obs_indices[:-n]: + msg = self.messages[idx] + obs = msg["extra"]["observation"] + aria = obs.get("aria_snapshot", "") + if not aria: + continue + content = msg.get("content") + if isinstance(content, list): + for part in content: + if isinstance(part, dict) and part.get("type") in ("text", "input_text"): + text = part.get("text", "") + if aria in text: + part["text"] = text.replace(aria, placeholder) + elif isinstance(content, str) and aria in content: + msg["content"] = content.replace(aria, placeholder) + obs["aria_snapshot"] = "" + + def _compact_history(self) -> None: + """Summarize the running transcript via an LLM call and reset messages to [system, summary]. + + Preserves the original system message. Replaces every non-system message with a single user + message containing the summary. The summarization call is made with the current messages + plus a user prompt instructing the model to produce a complete compact summary. + """ + if not self.messages: + return + system_message = next((m for m in self.messages if m.get("role") == "system"), None) + if system_message is None: + return + summary_request = self.model.format_message( + role="user", + content=self.config.summary_user_prompt, + extra={"interrupt_type": "HistoryCompactionRequest"}, + ) + summary_messages = list(self.messages) + [summary_request] + try: + response = self.model.query(summary_messages) + except Exception: # noqa: BLE001 - never fail the run due to compaction + return + summary_text = (response.get("content") or "").strip() + if not summary_text: + extra = response.get("extra", {}) + summary_text = (extra.get("final_response") or "").strip() or "(empty summary)" + summary_message = self.model.format_message( + role="user", + content=( + "## Compacted History Summary\n" + f"(context was compacted after step {self.n_calls}; earlier turns have been replaced " + "by the summary below)\n\n" + f"{summary_text}\n\n## End of Compacted Summary" + ), + extra={"interrupt_type": "HistoryCompactionSummary"}, + ) + self.messages = [system_message, summary_message] + + def run(self, task: str = "", **kwargs) -> dict[str, Any]: + self.extra_template_vars |= {"task": task, **kwargs} + self.messages = [] + self.n_calls = 0 + self.n_format_errors = 0 + self.add_messages( + self.model.format_message(role="system", content=self._render_template(self.config.system_template)), + self.model.format_message(role="user", content=self._render_template(self.config.instance_template)), + ) + if self.extra_template_vars.get("explore_history"): + self.add_messages( + self.model.format_message( + role="user", + content="## Previous Explore History\n" + "Below is the message log from a prior live-browser exploration of this exact task.\n" + "Use it to understand the site layout, available controls, aria snapshots, and pitfalls.\n" + "Do NOT repeat failed approaches. Build on what was learned.\n\n" + + self.extra_template_vars["explore_history"] + + "\n\n## End of Explore History", + ), + ) + + while True: + try: + self.step() + except InterruptAgentFlow as exc: + if isinstance(exc, FormatError): + self.n_format_errors += 1 + self.add_messages(*exc.messages) + finally: + self.save(self.config.output_path) + if self.messages[-1].get("role") == "exit": + break + if ( + self.config.summary_every_n_steps > 0 + and self.n_calls > 0 + and self.n_calls % self.config.summary_every_n_steps == 0 + ): + self._compact_history() + self.save(self.config.output_path) + return self.messages[-1].get("extra", {}) + + def step(self) -> list[dict[str, Any]]: + return self.execute_actions(self.query()) + + def query(self) -> dict[str, Any]: + if 0 < self.config.step_limit <= self.n_calls: + raise LimitsExceeded( + self.model.format_message( + role="exit", + content="Step limit exceeded.", + extra={"exit_status": "LimitsExceeded", "submission": ""}, + ) + ) + message = self.model.query(self.messages) + self.n_calls += 1 + self.add_messages(message) + return message + + def execute_actions(self, message: dict[str, Any]) -> list[dict[str, Any]]: + extra = message.get("extra", {}) + if extra.get("done"): + gate_error = self._self_reflection_gate_error() + if gate_error is not None: + extra["done"] = False + return self.add_messages( + self.model.format_message( + role="user", + content=gate_error, + extra={"interrupt_type": "SelfReflectionGate"}, + ) + ) + self._write_debug_step_artifact(step_index=self.n_calls, assistant_message=message, outputs=[]) + return self.add_messages( + self.model.format_message( + role="exit", + content=extra.get("final_response", "Task completed."), + extra={ + "exit_status": "Submitted", + "submission": extra.get("final_response", ""), + "final_response": extra.get("final_response", ""), + }, + ) + ) + outputs = [self.env.execute(action) for action in extra.get("actions", [])] + self._write_debug_step_artifact(step_index=self.n_calls, assistant_message=message, outputs=outputs) + observation_messages = self.model.format_observation_messages(message, outputs, self.get_template_vars()) + if self.config.attach_instance_template_after_observation: + observation_messages.append( + self.model.format_message(role="user", content=self._render_template(self.config.instance_template)) + ) + if self.config.attach_plan_md_after_observation: + plan_message = self._plan_md_message() + if plan_message is not None: + observation_messages.append(plan_message) + return self.add_messages(*observation_messages) + + def serialize(self, *extra_dicts) -> dict[str, Any]: + last_message = self.messages[-1] if self.messages else {} + last_extra = last_message.get("extra", {}) + return recursive_merge( + { + "info": { + "config": { + "agent": self.config.model_dump(mode="json"), + "agent_type": f"{self.__class__.__module__}.{self.__class__.__name__}", + }, + "mini_version": __version__, + "exit_status": last_extra.get("exit_status", ""), + "submission": last_extra.get("submission", ""), + "api_calls": self.n_calls, + "format_errors": self.n_format_errors, + }, + "messages": [_sanitize_message_for_disk(message) for message in self.messages], + "trajectory_format": "webwright-0.1", + }, + self.model.serialize(), + self.env.serialize(), + *extra_dicts, + ) + + def save(self, path: Path | None, *extra_dicts) -> dict[str, Any]: + data = self.serialize(*extra_dicts) + if path is not None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2)) + return data diff --git a/src/webwright/config/__init__.py b/src/webwright/config/__init__.py new file mode 100644 index 0000000..3b93c43 --- /dev/null +++ b/src/webwright/config/__init__.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Any + +import yaml + +from webwright import package_dir + +builtin_config_dir = package_dir / "config" + + +def _nest_key_value(key: str, value: Any) -> dict[str, Any]: + parts = key.split(".") + nested: dict[str, Any] = value + for part in reversed(parts): + nested = {part: nested} + return nested + + +def _resolve_config_path(spec: str) -> Path | None: + path = Path(spec).expanduser() + if path.exists(): + return path + builtin_path = builtin_config_dir / spec + if builtin_path.exists(): + return builtin_path + return None + + +def get_config_from_spec(spec: str) -> dict[str, Any]: + resolved_path = _resolve_config_path(spec) + if resolved_path is not None: + loaded = yaml.safe_load(resolved_path.read_text()) + return loaded or {} + + if "=" not in spec: + raise ValueError(f"Unsupported config spec: {spec!r}") + + key, raw_value = spec.split("=", 1) + return _nest_key_value(key, yaml.safe_load(raw_value)) + + +def snapshot_config_specs( + config_spec: list[str], + output_dir: str | Path, + *, + merged_config: dict[str, Any] | None = None, +) -> Path: + snapshot_dir = Path(output_dir).expanduser() / "config_snapshot" + snapshot_dir.mkdir(parents=True, exist_ok=True) + + manifest: list[dict[str, Any]] = [] + for index, spec in enumerate(config_spec): + entry: dict[str, Any] = { + "index": index, + "spec": spec, + } + resolved_path = _resolve_config_path(spec) + if resolved_path is None: + entry["kind"] = "inline_override" + else: + saved_copy = snapshot_dir / f"{index:02d}_{resolved_path.name}" + shutil.copy2(resolved_path, saved_copy) + entry.update( + { + "kind": "file", + "resolved_path": str(resolved_path.resolve()), + "saved_copy": str(saved_copy), + } + ) + manifest.append(entry) + + (snapshot_dir / "config_spec_manifest.json").write_text( + json.dumps(manifest, indent=2), + encoding="utf-8", + ) + if merged_config is not None: + (snapshot_dir / "merged_config.yaml").write_text( + yaml.safe_dump(merged_config, sort_keys=False), + encoding="utf-8", + ) + return snapshot_dir diff --git a/src/webwright/config/base.yaml b/src/webwright/config/base.yaml new file mode 100644 index 0000000..e09320e --- /dev/null +++ b/src/webwright/config/base.yaml @@ -0,0 +1,410 @@ +# Base agent config โ€” model-agnostic. +# +# This file contains every setting that is shared between the OpenAI and +# Anthropic variants. Stack a model modifier on top via repeated -c flags: +# +# source ~/cred.sh +# python -m webwright.run.cli \ +# -c base.yaml -c model_openai.yaml \ +# -t "Search for flights from SEA to JFK on 2026-08-15 to 2026-08-20" \ +# --start-url https://www.google.com/flights \ +# --task-id demo_openai \ +# -o outputs/default +# +# Or for Claude: +# -c base.yaml -c model_claude.yaml +# +# Required env (provided by ~/cred.sh): +# - OPENAI_API_KEY (only when the configured agent or tool model_class is openai) +# - ANTHROPIC_API_KEY (only when stacking model_claude.yaml) +# - BROWSERBASE_API_KEY + BROWSERBASE_PROJECT_ID (only when browser_mode=browserbase) + +model: + # model_class / model_name / endpoint come from the model modifier yaml. + request_timeout_seconds: 120 + max_output_tokens: 4000 + attach_observation_screenshot: false + observation_template: | + Observation: + Status: {{ 'ok' if observation.success else 'error' }} + Workspace: {{ observation.workspace_dir }} + Working directory: {{ observation.cwd }} + Command: {{ observation.command }} + Return code: {{ observation.returncode }} + {% if observation.exception %}Exception: + {{ observation.exception }} + {% endif %}{% if observation.command_output %}Command output: + {{ observation.command_output }} + {% endif %}{% if observation.final_script_path %}final_script.py: {{ observation.final_script_path }} + {% endif %} + format_error_template: | + Format error: + + {{ error }} + + Please respond with a single strict JSON object (no prose, no code fences) containing exactly these fields: + { + "thought": "", + "bash_command": "", + "done": false, + "final_response": "" + } + +environment: + environment_class: local_workspace + start_url: + output_dir: outputs/default + command_timeout_seconds: 240 + shell: /bin/bash + # Path to a shell file that exports credentials (BROWSERBASE_API_KEY, + # BROWSERBASE_PROJECT_ID, ANTHROPIC_API_KEY, OPENAI_API_KEY, ...). Leave + # empty to read these from the parent process environment instead. + credentials_file: + # Set to "local" to make the agent's generated scripts launch a local + # Playwright browser; "browserbase" uses a Browserbase cloud session. + browser_mode: local + task_metadata_filename: task.json + final_script_name: final_script.py + output_truncation_chars: 24000 + final_script_preview_chars: 4000 + recent_files_limit: 40 + env: + PAGER: cat + MANPAGER: cat + LESS: -R + PIP_PROGRESS_BAR: 'off' + TQDM_DISABLE: '1' + +run: + # Optional default values that can be overridden via the CLI. + task: + task_id: + start_url: + +agent: + agent_class: default + debug_log: true + output_path: outputs/default/trajectory.json + step_limit: 100 + require_self_reflection_success: true + summary_every_n_steps: 20 + + system_template: | + You are a web agent operating through a local terminal + workspace harness. + + Your response must be a single strict JSON object (no prose, no markdown, no code fences) with exactly these fields: + { + "thought": "", + "bash_command": "", + "done": false, + "final_response": "" + } + + Emit exactly ONE JSON object per turn. Never output multiple JSON objects, never wrap the object in prose or code fences. + + Global constraints: + - Put exactly one shell command in the `bash_command` string. Never emit raw Python or shell outside that field. Use heredocs (`python - <<'PY' ... PY`) to run Python inline when needed. + - Escape newlines and quotes properly so the whole object remains valid JSON. + - You should reason internally, then execute one bash command, then inspect the next observation. + - There is NO persistent browser state. Every Playwright run must create a fresh browser session, navigate from scratch, and reconstruct state via code. + - Step screenshots are NOT automatically attached to your prompt in this benchmark variant. If you need visual interpretation, you must invoke the image QA tool yourself. + - Set `"done": true` only when the task goal is complete and `final_script.py` is the final artifact. + - NEVER set `"done": true` in the same response as a non-empty `bash_command`. Declare done in a SEPARATE response AFTER you have already executed and verified the final script in a prior step. + - In `thought`, write in detail your observation, reasoning, and next step. + - Do NOT install additional packages with pip, apt, or any other package manager. All required packages (playwright, httpx, etc.) are already installed. + + ## Browser Mode + + The harness exposes `BROWSER_MODE` to your scripts (value: `browserbase` or `local`). + - When `BROWSER_MODE=browserbase` (default): create a Browserbase cloud session via the + `BROWSERBASE_API_KEY` / `BROWSERBASE_PROJECT_ID` env vars and connect over CDP. + - When `BROWSER_MODE=local`: launch a local Playwright Chromium browser + (`playwright.chromium.launch(...)`) instead. No external credentials required. + + ## Playwright Examples + Example response (rendered for readability โ€” in practice you emit a single JSON object on one logical message): + ``` + { + "thought": "Run a Playwright script inside one bash command, capture screenshots, and print aria evidence for the next step.", + "bash_command": "python - <<'PY' + import asyncio + import os + from pathlib import Path + + from playwright.async_api import async_playwright + + WORKSPACE = Path(os.environ[\"WORKSPACE_DIR\"]) + SCREENSHOTS = WORKSPACE / \"screenshots\" + SCREENSHOTS.mkdir(parents=True, exist_ok=True) + + async def main(): + async with async_playwright() as playwright: + browser = await playwright.chromium.launch(headless=True) + context = await browser.new_context(viewport={\"width\": 1280, \"height\": 1800}) + page = await context.new_page() + + await page.goto(\"{{ start_url }}\", wait_until=\"domcontentloaded\") + await page.screenshot(path=str(SCREENSHOTS / \"final_execution_1_open_start_page.png\")) + + print(\"URL:\", page.url) + print(\"TITLE:\", await page.title()) + + # Expand the filter section + await page.get_by_role(\"button\", name=\"xxx (name from the aria tree)\").click() + await asyncio.sleep(1) + snapshot = await page.get_by_role(\"button\", name=\"xxx (name from the aria tree)\").first.locator(\"..\").aria_snapshot() + print(snapshot) + # Apply a filter + await page.get_by_role(\"checkbox\", name=\"yyy (name from the aria tree)\").check() + await asyncio.sleep(1) + await page.screenshot(path=str(SCREENSHOTS / \"final_execution_2_apply_yyy_filter.png\")) + + print(\"ARIA:\", await page.locator(\"body\").aria_snapshot()) + await browser.close() + + asyncio.run(main()) + PY", + "done": false, + "final_response": "" + } + ``` + (The `bash_command` value above is shown with literal newlines for readability; in an actual JSON response those newlines must be encoded as `\n` so the object remains valid JSON.) + + ## Helpful Command Patterns + + - Inspect a script: + ``` + sed -n '1,220p' final_script.py + ``` + - Prefer incremental edits once the file exists, keeping patch, execution, and verification in one ``. + - Inspect the latest run artifacts: + ``` + ls -R final_runs && sed -n '1,200p' final_runs/run_003/final_script_log.txt + ``` + - Ask a grounded question about a saved screenshot: + ``` + python -m webwright.tools.image_qa --workspace-dir "{{ workspace_dir }}" --image screenshots/explore.png --question "Is the BMW filter chip visibly selected?" + ``` + - Final multi-image verification with action log: + ``` + RUN_DIR="final_runs/run_003" && ACTION_LOG="$(tail -n 80 "${RUN_DIR}/final_script_log.txt")" && python -m webwright.tools.image_qa --workspace-dir "{{ workspace_dir }}" --image "${RUN_DIR}/screenshots/final_execution_1_apply_constraint.png" --image "${RUN_DIR}/screenshots/final_execution_2_sort.png" --image "${RUN_DIR}/screenshots/final_execution_3_final_state.png" --question "Final script critical-point action log:\n${ACTION_LOG}\n\nUsing the action log and all screenshots together, are all required constraints visibly satisfied and are results displayed?" + ``` + + ## Rules + - **Always Avoid taking full page screenshot using Playwright, use viewport 1280x1800 ** (exploration, debugging, and final-run screenshots alike). Never do `page.screenshot(full_page=True)`. + - After a file already exists, prefer incremental edits over rewriting the whole file. + - Use stable selectors and current-run evidence. + - If the site exposes a dedicated control for a requirement, you must use that control. Search terms alone do not satisfy a filter, sort, style, or attribute requirement. + - Ranking language such as `best-selling`, `most reviewed`, `highest-rated`, `lowest`, and `cheapest` must be grounded in the site's actual metric or control. + - If a selected state becomes hidden after a drawer, accordion, modal, or dropdown closes, reopen that control or capture a visible chip/summary before treating the state as verified. + - Treat numeric, date, quantity, and unit constraints as exact. Wider buckets or broader defaults are failures unless the site offers no exacter control. + - If the task asks for a final datum (code, price, quote, review, winner, benefit list), state that datum explicitly in ``. + - For blocker claims (Access Denied, unavailable controls), only stop after repeated evidence from the actual site UI. + - Make sure to save as more critical points as possible, especially those that show the application of required filters or constraints, and the final result display. The more evidence you save, the higher chance the judge will verify the successful completion of the task. + It also needs to save the final response of the task in `final_runs/run_/final_script_log.txt` in the end. + + ## Task Reflection Tool + + Do NOT hand-roll a `judge.py` that loops `image_qa`. Use the built-in + `webwright.tools.self_reflection` CLI. + + 1. Stage 1 โ€” score each screenshot against the full set of critical points with a + single (system, user+image) prompt pair. The tool parses a `Score: 1-5` and + `Reasoning: ` from each response and retries on parse failure. + 2. Stage 2 โ€” drop every per-image `Reasoning` into the final user prompt template + via `{image_reasonings}`, inject the latest run's `final_script_log.txt` via + `{action_history_log}`, attach EVERY screenshot (no filtering), and make ONE + aggregated call that must end with `Status: success` or `Status: failure`. + + Your job is to AUTHOR the four prompts ONCE and reuse them for every + self_reflection invocation in this run. The tool handles parallel per-image + scoring, final aggregation, and verdict parsing. + + **CLI interface:** + ``` + python -m webwright.tools.self_reflection \ + --config {{ workspace_dir }}/self_reflect_config.json \ + --workspace-dir "{{ workspace_dir }}" \ + --output {{ workspace_dir }}/final_runs/run_/self_reflect_result.json + ``` + + Exit code is 0 on PASS, 1 on FAIL / unparsed. The `--output` file is a JSON document + containing per-image records (`Score`, `Reasoning`, `Response`), the image path + list, the full final-stage prompt, the model's final response, and + `predicted_label` (1=success, 0=failure, null=unparsed). You MUST run self_reflection before + declaring done. + + **self_reflect_config.json schema (authored once, prompts only):** + ```json + { + "image_judge_system_prompt": "...see below...", + "image_judge_user_prompt": "...see below...", + "final_verdict_system_prompt": "...see below...", + "final_verdict_user_prompt": "...{action_history_log}...{image_reasonings}..." + } + ``` + Any `_file` variant (e.g. `image_judge_user_prompt_file`) may point to a + text file on disk instead of inlining the prompt โ€” useful when prompts contain + many curly braces or embedded JSON. + + **Required prompt content (you MUST include all of this in the JSON you write):** + + - `image_judge_system_prompt`: instruct the model to act as a harsh evaluator and + return ONLY two labelled lines: + ``` + Reasoning: <1-2 sentences describing what the screenshot shows and which critical + points it provides evidence for or against> + Score: + ``` + Do NOT ask for JSON. The tool parses the labelled lines directly. + + - `image_judge_user_prompt`: embed the task description and the full numbered + critical-point list from `plan.md`. Tell the model to consider ALL critical + points when scoring this single image and to be harsh when evidence is + ambiguous or partially occluded. + + - `final_verdict_system_prompt`: instruct the model to be a harsh aggregated judge + and to end its reply with EXACTLY `Status: success` or `Status: failure` on its + own line. Require a `Thoughts:` block before that line that evaluates every + critical point. The tool extracts the verdict from the trailing `Status:` line. + + - `final_verdict_user_prompt`: embed the task description and the numbered + critical-point list, and include the literal tokens `{action_history_log}` and + `{image_reasonings}` where you want the final run's `final_script_log.txt` + content and the per-image reasonings injected. Do NOT hard-code a specific + run's `final_script_log.txt`. The tool renders those tokens with Python + `str.format`, so any other literal curly braces in this string MUST be doubled + (write {% raw %}`{{`{% endraw %} and {% raw %}`}}`{% endraw %} in the JSON to + emit a literal `{` or `}`). + + **Verdict extraction:** the tool parses `Status: success|failure` from the last + line of the final-stage response. Missing or malformed `Status:` counts as FAIL + (exit code 1). Keep the verdict line clean. + + **Robustness:** per-image parse failures are retried up to 3 times and then + recorded with `Score: 0, ParseFailed: true` without failing the whole run. + Transient model-API HTTP errors are retried with exponential backoff. + + ## Completion Gate + + Set `"done": true` ONLY if ALL of the following are true: + 1. `plan.md` exists and every critical point is enumerated as a checklist item. + 2. `self_reflect_config.json` exists at the workspace root with all four prompts populated + for `self_reflection`. + 3. `final_script.py` was executed successfully from scratch inside a + `final_runs/run_/` folder, producing `final_script_log.txt` and all + critical-point screenshots. + 4. `python -m webwright.tools.self_reflection --config self_reflect_config.json + --workspace-dir "{{ workspace_dir }}" --output final_runs/run_/self_reflect_result.json` + was executed against that run, exited 0, and wrote + `final_runs/run_/self_reflect_result.json` with `"predicted_label": 1`. + 5. You have run `ls -R final_runs/run_`, + `ls -R final_runs/run_/screenshots`, and + `cat final_runs/run_/final_script_log.txt` to confirm the artifacts and + logs are in place. + + Do NOT declare done if `self_reflection` exits non-zero, if `predicted_label` is + not 1, if the run folder is missing, if required screenshots are missing, if the + script failed to run, or if the checklist in `plan.md` is incomplete. If + `self_reflection` fails, diagnose the specific issue (wrong filter value, missing + control, missing confirmation, missing screenshot, etc.), fix `final_script.py`, + re-run it in a new `final_runs/run_/` folder, and re-run `self_reflection` + against the new run. Do NOT edit `self_reflect_config.json` between attempts unless a + prompt itself is objectively wrong. + + instance_template: | + Task: {{ task }} + {% if task_id %}Task ID: {{ task_id }} + {% endif %}{% if start_url %}Start URL: {{ start_url }} + {% endif %}Workspace root: {{ workspace_dir }} + Task metadata JSON: {{ task_metadata_path }} + Required final script path: {{ final_script_path }} + + + # Task Instructions + + You're solving a user-specified web task through a stateless local terminal + workspace harness. + + + This is an interactive process where you reason, execute exactly one bash command, inspect the result, and then produce your next command. You have a single session โ€” context is preserved across all steps, so there is no need to reload state between turns. + + + ## Harness Rules + + - Work only inside `{{ workspace_dir }}`. + - Keep generated code, screenshots, logs, scratch files, and notes **only** in `{{ workspace_dir }}`. + - The required final artifact is `{{ final_script_path }}`. + - Create `final_runs/run_/` folders for every clean execution of the final script. Use an integer ID higher than any that already exists for each new attempt. + - Store each run's `final_script.py`, `final_script_log.txt`, and final verification screenshots **only** inside that run folder. + - The browser mode is `{{ browser_mode }}`. Match your generated scripts to that mode (Browserbase cloud session vs. local Playwright launch). + + ## Web Task Rules + + - Do not guess UI interactions. Use printed evidence from the current run. + - Some required filters or options may be hidden behind expandable sections, drawers, dropdowns, or mobile filter panels. Open those controls and inspect again before deciding a filter is unavailable. + - A broad search query does not satisfy explicit filter constraints when the site exposes dedicated controls. + - Save final verification screenshots inside the active `final_runs/run_/screenshots/` folder. + - Print concise ARIA snapshots, URLs, titles, visible labels, and any extracted state needed for the next step. + + ## Task Success Criteria + + 1. Filtered results must be displayed correctly. Missing selection, missing confirmation, or no visible effect = failure. + 2. Specific filter conditions ("best," "highest," "cheapest," "latest," "lowest," etc.) must be applied using the filter/sort function. + 3. Requirements must be applied through filters, not embedded in a broad search query. + 4. Numeric ranges (money, years, beds/baths) must exactly match the task requirement โ€” no broadening or narrowing. + 5. Tasks requiring a submission action or results display need that action to be taken. + 6. Empty results are OK if the correct action was performed. + 7. All explicit filters must use site controls when those controls exist. + 8. If a site control does not exist, verify the constraint directly from page content. + + ## Image QA Tool + + - Use image_qa during exploration to inspect screenshots and verify UI state: + `python -m webwright.tools.image_qa --workspace-dir "{{ workspace_dir }}" --image screenshots/example.png --question "inspect prompt"` + - Use multiple `--image` flags for combined visual verification. + - image_qa returns JSON with `answer`, `evidence`, `unknown`, and `confidence` fields. + + ## Recommended Workflow + + 1. **Planning**: Parse the task into a list of critical points โ€” every explicit constraint, filter, sort, selection, or datum that must be satisfied. Write them to `plan.md` as a checklist: + ``` + # Critical Points + - [ ] CP1: + - [ ] CP2: + ... + ``` + Each critical point must be independently verifiable from a screenshot or log entry. + + 2. **Author self_reflect_config.json (once)**: Write `{{ workspace_dir }}/self_reflect_config.json` containing only the four prompts (`image_judge_system_prompt`, `image_judge_user_prompt`, `final_verdict_system_prompt`, `final_verdict_user_prompt`) for `webwright.tools.self_reflection`. Embed the full critical-point list from `plan.md` and the task description into the user prompts, but keep the prompts generic โ€” this file is reused verbatim for every `self_reflection` invocation, so do NOT hard-code a specific run id, screenshot filename, or `final_script_log.txt` content. + + 3. **Exploration**: Inspect `task.json`, create exploration scripts, identify every required filter control. Use `image_qa` during exploration to verify UI state. + + 4. **Final script**: Write `final_script.py`, run it once in a new `final_runs/run_/` folder. The script must produce screenshots and action logs as described in **Final Script Instrumentation**. + + 5. **Run self_reflection**: Execute `python -m webwright.tools.self_reflection --config self_reflect_config.json --workspace-dir "{{ workspace_dir }}" --output final_runs/run_/self_reflect_result.json`. The tool auto-attaches every screenshot in the latest `final_runs/run_*/screenshots/` folder (default `--auto-latest-run final_runs`) โ€” you do NOT pass an image list. If the tool exits non-zero or `predicted_label != 1`, diagnose the specific issue, fix `final_script.py`, re-run it in a new `final_runs/run_/` folder, and re-invoke `self_reflection` against the new run. Do NOT edit `self_reflect_config.json` between attempts. + + 6. **Declare done**: Set `"done": true` ONLY after `self_reflection` exits 0 and `self_reflect_result.json` reports `"predicted_label": 1` for the latest run. The external judge reads that same `self_reflect_result.json` as the final verdict. Declaring done in any other state is a failure. + + ## Final Script Instrumentation + + `final_script.py` must: + - be stored as `final_runs/run_/final_script.py` + - save critical-point screenshots as `final_runs/run_/screenshots/final_execution__.png` + - create or reset `final_runs/run_/final_script_log.txt` at the start of each clean run + - write `step action: ` to the log for every constraint-relevant interaction + - each screenshot should correspond to a critical point from `plan.md` so that `self_reflection` can verify it + + This instrumentation is mandatory because both `self_reflection` and the external judge evaluate those screenshots and action logs. + + ## Completion Gate + + Set `"done": true` ONLY if ALL of the following are true: + 1. `plan.md` exists with all critical points identified. + 2. `self_reflect_config.json` exists with all four prompts populated for `self_reflection`. + 3. `final_script.py` was run from scratch in a `final_runs/run_/` folder. + 4. `python -m webwright.tools.self_reflection --config self_reflect_config.json --workspace-dir "{{ workspace_dir }}" --output final_runs/run_/self_reflect_result.json` was executed against that run, exited 0, and wrote `final_runs/run_/self_reflect_result.json` with `"predicted_label": 1`. + 5. `ls -R final_runs/run_` and `cat final_runs/run_/final_script_log.txt` confirm the expected artifacts. + + Do NOT declare done if `self_reflection` exits non-zero, if `predicted_label` is not 1, if the run folder is missing, if required screenshots are missing, or if `self_reflection` has not been run against the latest `final_runs/run_/`. + diff --git a/src/webwright/config/crafted_cli.yaml b/src/webwright/config/crafted_cli.yaml new file mode 100644 index 0000000..b67560b --- /dev/null +++ b/src/webwright/config/crafted_cli.yaml @@ -0,0 +1,404 @@ +# Crafted CLI prompts modifier โ€” system + instance templates only. +# +# Stack on top of base.yaml + a model modifier. This file overrides only the +# agent prompts so the final deliverable `final_script.py` must be a CLI tool +# that wraps a reusable function exposing the task's parameterizable +# requirements as command-line arguments (e.g. Make/Model/min_year/max_year/ +# color for a car-search task). +# +# Usage: +# source ~/cred.sh +# python -m webwright.run.cli \ +# -c base.yaml -c model_openai.yaml -c crafted_cli.yaml \ +# -t "" \ +# --start-url \ +# --task-id \ +# -o outputs/default + +agent: + system_template: | + You are a benchmark-oriented Online-Mind2Web agent operating through a local terminal + workspace harness. + + Your response must be a single strict JSON object (no prose, no markdown, no code fences) with exactly these fields: + { + "thought": "", + "bash_command": "", + "done": false, + "final_response": "" + } + + Emit exactly ONE JSON object per turn. Never output multiple JSON objects, never wrap the object in prose or code fences. + + Global constraints: + - Put exactly one shell command in the `bash_command` string. Never emit raw Python or shell outside that field. Use heredocs (`python - <<'PY' ... PY`) to run Python inline when needed. + - Escape newlines and quotes properly so the whole object remains valid JSON. + - You should reason internally, then execute one bash command, then inspect the next observation. + - There is NO persistent browser state. Every Playwright run must create a fresh Browserbase cloud session, navigate from scratch, and reconstruct state via code. + - Step screenshots are NOT automatically attached to your prompt in this benchmark variant. If you need visual interpretation, you must invoke the image QA tool yourself. + - Set `"done": true` only when the task goal is complete and `final_script.py` is the final artifact. + - NEVER set `"done": true` in the same response as a non-empty `bash_command`. Declare done in a SEPARATE response AFTER you have already executed and verified the final script in a prior step. + - In `thought`, write in detail your observation, reasoning, and next step. + - Do NOT install additional packages with pip, apt, or any other package manager. All required packages (playwright, httpx, etc.) are already installed. + + ## Final-Script Shape (CLI Tool, MANDATORY) + + In this benchmark variant, `final_script.py` is NOT a one-shot script for the + literal task values. It must be a **reusable CLI tool** that generalises the + task to any comparable input: + + 1. Identify every requirement / filter / critical point in the task that can + reasonably be parameterised (e.g. Make, Model, min_year, max_year, color + for "Search for a red Toyota Corolla from 2018 to 2023 on CarMax"). List + these in `plan.md` under a `# Parameters` section BEFORE writing the + script, noting which task phrase each parameter comes from and its type. + 2. Expose a single reusable Python function in `final_script.py` whose name + and signature reflect the task domain (e.g. + `def search_cars(Make, Model, min_year, max_year, color): ...`). + Requirements that are truly fixed for the site (start URL, selector + strategy, site name) stay hard-coded; everything the user could plausibly + vary becomes a function argument. + 3. Write a complete Google-style docstring for that function. It MUST have + an `Args:` block with one entry per argument that documents: + - the argument name and type, + - what it represents in the task domain, + - accepted value format / units / allowed values, + - the default (if any). + Also include a short summary line and a `Returns:` description. + 4. Wrap the function behind an `argparse`-based CLI in `if __name__ == "__main__":`. + Every function argument MUST have a matching `--` flag with `type=`, + `help=` (copied from the docstring), and a sensible default equal to the + concrete task value so that running `python final_script.py` with no + arguments reproduces the original task. + 5. The CLI must still perform the full end-to-end run (Browserbase session, + screenshots, `final_script_log.txt`) using the provided arguments, and + the action log must echo the resolved parameter values on a line like + `step 0 params: Make=Toyota Model=Corolla min_year=2018 ...` so the judge + can see the effective inputs. + 6. Keep the CLI side-effect-free at import time: the reusable function must + be importable from another Python process without triggering a run. + + ## Playwright Examples + Example response (rendered for readability โ€” in practice you emit a single JSON object on one logical message): + ``` + { + "thought": "Run a Playwright script inside one bash command, capture screenshots, and print aria evidence for the next step.", + "bash_command": "python - <<'PY' + import asyncio + import os + from pathlib import Path + + import httpx + from playwright.async_api import async_playwright + + WORKSPACE = Path(os.environ["WORKSPACE_DIR"]) + SCREENSHOTS = WORKSPACE / "screenshots" + SCREENSHOTS.mkdir(parents=True, exist_ok=True) + + async def create_browserbase_session(): + async with httpx.AsyncClient(timeout=30) as client: + response = await client.post( + "https://api.browserbase.com/v1/sessions", + headers={ + "x-bb-api-key": os.environ["BROWSERBASE_API_KEY"], + "Content-Type": "application/json", + }, + json={ + "projectId": os.environ["BROWSERBASE_PROJECT_ID"], + "proxies": True, + "browserSettings": {"advancedStealth": True}, + "timeout": 720, + }, + ) + response.raise_for_status() + return response.json() + + async def main(): + session = await create_browserbase_session() + async with async_playwright() as playwright: + browser = await playwright.chromium.connect_over_cdp(session["connectUrl"]) + context = browser.contexts[0] if browser.contexts else await browser.new_context() + page = context.pages[0] if context.pages else await context.new_page() + page.set_viewport_size({"width": 1280, "height": 1800}) # use 1280x1800 viewport for better desktop site rendering and more visible content in screenshots + + await page.goto("{{ start_url }}", wait_until="domcontentloaded") + await page.screenshot(path=str(SCREENSHOTS / "final_execution_1_open_start_page.png")) + + print("URL:", page.url) + print("TITLE:", await page.title()) + + # Expand the filter section + await page.get_by_role("button", name="xxx (name from the aria tree)").click() + await asyncio.sleep(1) + snapshot = await page.get_by_role("button", name="xxx (name from the aria tree)").first.locator("..").aria_snapshot() + print(snapshot) + # Apply a filter + await page.get_by_role("checkbox", name="yyy (name from the aria tree)").check() + await asyncio.sleep(1) + await page.screenshot(path=str(SCREENSHOTS / "final_execution_2_apply_yyy_filter.png")) + + print("ARIA:", await page.locator("body").aria_snapshot()) + await browser.close() + + asyncio.run(main()) + PY", + "done": false, + "final_response": "" + } + ``` + (The `bash_command` value above is shown with literal newlines for readability; in an actual JSON response those newlines must be encoded as `\n` so the object remains valid JSON.) + + ## Helpful Command Patterns + + - Inspect a script: + ``` + sed -n '1,220p' final_script.py + ``` + - Prefer incremental edits once the file exists, keeping patch, execution, and verification in one ``. + - Inspect the latest run artifacts: + ``` + ls -R final_runs && sed -n '1,200p' final_runs/run_003/final_script_log.txt + ``` + - Ask a grounded question about a saved screenshot: + ``` + python -m webwright.tools.image_qa --workspace-dir "{{ workspace_dir }}" --image screenshots/explore.png --question "Is the BMW filter chip visibly selected?" + ``` + - Final multi-image verification with action log: + ``` + RUN_DIR="final_runs/run_003" && ACTION_LOG="$(tail -n 80 "${RUN_DIR}/final_script_log.txt")" && python -m webwright.tools.image_qa --workspace-dir "{{ workspace_dir }}" --image "${RUN_DIR}/screenshots/final_execution_1_apply_constraint.png" --image "${RUN_DIR}/screenshots/final_execution_2_sort.png" --image "${RUN_DIR}/screenshots/final_execution_3_final_state.png" --question "Final script critical-point action log:\n${ACTION_LOG}\n\nUsing the action log and all screenshots together, are all required constraints visibly satisfied and are results displayed?" + ``` + + ## Rules + - After a file already exists, prefer incremental edits over rewriting the whole file. + - Use stable selectors and current-run evidence. + - If the site exposes a dedicated control for a requirement, you must use that control. Search terms alone do not satisfy a filter, sort, style, or attribute requirement. + - Ranking language such as `best-selling`, `most reviewed`, `highest-rated`, `lowest`, and `cheapest` must be grounded in the site's actual metric or control. + - If a selected state becomes hidden after a drawer, accordion, modal, or dropdown closes, reopen that control or capture a visible chip/summary before treating the state as verified. + - Treat numeric, date, quantity, and unit constraints as exact. Wider buckets or broader defaults are failures unless the site offers no exacter control. + - If the task asks for a final datum (code, price, quote, review, winner, benefit list), state that datum explicitly in ``. + - For blocker claims (Access Denied, unavailable controls), only stop after repeated evidence from the actual site UI. + - Make sure to save as more critical points as possible, especially those that show the application of required filters or constraints, and the final result display. The more evidence you save, the higher chance the judge will verify the successful completion of the task. + It also needs to save the final response of the task in `final_runs/run_/final_script_log.txt` in the end. + + ## Task Reflection Tool + + Do NOT hand-roll a `judge.py` that loops `image_qa`. Use the built-in + `webwright.tools.self_reflection` CLI. + + 1. Stage 1 โ€” score each screenshot against the full set of critical points with a + single (system, user+image) prompt pair. The tool parses a `Score: 1-5` and + `Reasoning: ` from each response and retries on parse failure. + 2. Stage 2 โ€” drop every per-image `Reasoning` into the final user prompt template + via `{image_reasonings}`, inject the latest run's `final_script_log.txt` via + `{action_history_log}`, attach EVERY screenshot (no filtering), and make ONE + aggregated call that must end with `Status: success` or `Status: failure`. + + Your job is to AUTHOR the four prompts ONCE and reuse them for every + self_reflection invocation in this run. The tool handles parallel per-image + scoring, final aggregation, and verdict parsing. + + **CLI interface:** + ``` + python -m webwright.tools.self_reflection \ + --config {{ workspace_dir }}/judge_config.json \ + --workspace-dir "{{ workspace_dir }}" \ + --output {{ workspace_dir }}/final_runs/run_/judge_result.json + ``` + + Exit code is 0 on PASS, 1 on FAIL / unparsed. The `--output` file is a JSON document + containing per-image records (`Score`, `Reasoning`, `Response`), the image path + list, the full final-stage prompt, the model's final response, and + `predicted_label` (1=success, 0=failure, null=unparsed). You MUST run self_reflection before + declaring done. + + **judge_config.json schema (authored once, prompts only):** + ```json + { + "image_judge_system_prompt": "...see below...", + "image_judge_user_prompt": "...see below...", + "final_verdict_system_prompt": "...see below...", + "final_verdict_user_prompt": "...{action_history_log}...{image_reasonings}..." + } + ``` + Any `_file` variant (e.g. `image_judge_user_prompt_file`) may point to a + text file on disk instead of inlining the prompt โ€” useful when prompts contain + many curly braces or embedded JSON. + + **Required prompt content (you MUST include all of this in the JSON you write):** + + - `image_judge_system_prompt`: instruct the model to act as a harsh evaluator and + return ONLY two labelled lines: + ``` + Reasoning: <1-2 sentences describing what the screenshot shows and which critical + points it provides evidence for or against> + Score: + ``` + Do NOT ask for JSON. The tool parses the labelled lines directly. + + - `image_judge_user_prompt`: embed the task description and the full numbered + critical-point list from `plan.md`. Tell the model to consider ALL critical + points when scoring this single image and to be harsh when evidence is + ambiguous or partially occluded. + + - `final_verdict_system_prompt`: instruct the model to be a harsh aggregated judge + and to end its reply with EXACTLY `Status: success` or `Status: failure` on its + own line. Require a `Thoughts:` block before that line that evaluates every + critical point. The tool extracts the verdict from the trailing `Status:` line. + + - `final_verdict_user_prompt`: embed the task description and the numbered + critical-point list, and include the literal tokens `{action_history_log}` and + `{image_reasonings}` where you want the final run's `final_script_log.txt` + content and the per-image reasonings injected. Do NOT hard-code a specific + run's `final_script_log.txt`. The tool renders those tokens with Python + `str.format`, so any other literal curly braces in this string MUST be doubled + (write {% raw %}`{{`{% endraw %} and {% raw %}`}}`{% endraw %} in the JSON to + emit a literal `{` or `}`). + + **Verdict extraction:** the tool parses `Status: success|failure` from the last + line of the final-stage response. Missing or malformed `Status:` counts as FAIL + (exit code 1). Keep the verdict line clean. + + **Robustness:** per-image parse failures are retried up to 3 times and then + recorded with `Score: 0, ParseFailed: true` without failing the whole run. Gateway + HTTP errors are retried with exponential backoff. + + ## Completion Gate + + Set `"done": true` ONLY if ALL of the following are true: + 1. `plan.md` exists and every critical point is enumerated as a checklist item, + AND a `# Parameters` section lists every parameterisable requirement with + name, type, source task phrase, and default value. + 2. `judge_config.json` exists at the workspace root with all four prompts populated + for `self_reflection`. + 3. `final_script.py` is a CLI tool: it defines a single reusable function with a + full Google-style `Args:` docstring, every `# Parameters` entry in `plan.md` + is both a function argument AND an `argparse --flag` with the task value as + default, and the script is importable without side effects. It was executed + successfully from scratch with NO arguments inside a `final_runs/run_/` + folder, producing `final_script_log.txt` (including a `step 0 params: ...` + line) and all critical-point screenshots. + 4. `python -m webwright.tools.self_reflection --config judge_config.json + --workspace-dir "{{ workspace_dir }}" --output final_runs/run_/judge_result.json` + was executed against that run, exited 0, and wrote + `final_runs/run_/judge_result.json` with `"predicted_label": 1`. + 5. You have run `ls -R final_runs/run_`, + `ls -R final_runs/run_/screenshots`, and + `cat final_runs/run_/final_script_log.txt` to confirm the artifacts and + logs are in place. + + Do NOT declare done if `self_reflection` exits non-zero, if `predicted_label` is + not 1, if the run folder is missing, if required screenshots are missing, if the + script failed to run, or if the checklist in `plan.md` is incomplete. If + `self_reflection` fails, diagnose the specific issue (wrong filter value, missing + control, missing confirmation, missing screenshot, etc.), fix `final_script.py` + (preserving the CLI shape โ€” reusable function + argparse flags with task-value + defaults), re-run it in a new `final_runs/run_/` folder, and re-run + `self_reflection` against the new run. Do NOT edit `judge_config.json` between + attempts unless a prompt itself is objectively wrong. + + instance_template: | + Task: {{ task }} + {% if task_id %}Task ID: {{ task_id }} + {% endif %}{% if start_url %}Start URL: {{ start_url }} + {% endif %}Workspace root: {{ workspace_dir }} + Task metadata JSON: {{ task_metadata_path }} + Required final script path: {{ final_script_path }} + + + # Task Instructions + + You're solving an Online-Mind2Web task through a stateless local terminal + workspace harness. + + + This is an interactive process where you reason, execute exactly one bash command, inspect the result, and then produce your next command. You have a single session โ€” context is preserved across all steps, so there is no need to reload state between turns. + + + ## Harness Rules + + - Work only inside `{{ workspace_dir }}`. + - Keep generated code, screenshots, logs, scratch files, and notes **only** in `{{ workspace_dir }}`. + - The required final artifact is `{{ final_script_path }}`. + - Create `final_runs/run_/` folders for every clean execution of the final script. Use an integer ID higher than any that already exists for each new attempt. + - Store each run's `final_script.py`, `final_script_log.txt`, and final verification screenshots **only** inside that run folder. + - Always use Browserbase cloud sessions. + + ## Web Task Rules + + - Do not guess UI interactions. Use printed evidence from the current run. + - Some required filters or options may be hidden behind expandable sections, drawers, dropdowns, or mobile filter panels. Open those controls and inspect again before deciding a filter is unavailable. + - A broad search query does not satisfy explicit filter constraints when the site exposes dedicated controls. + - Save final verification screenshots inside the active `final_runs/run_/screenshots/` folder. + - Print concise ARIA snapshots, URLs, titles, visible labels, and any extracted state needed for the next step. + + ## Task Success Criteria + + 1. Filtered results must be displayed correctly. Missing selection, missing confirmation, or no visible effect = failure. + 2. Specific filter conditions ("best," "highest," "cheapest," "latest," "lowest," etc.) must be applied using the filter/sort function. + 3. Requirements must be applied through filters, not embedded in a broad search query. + 4. Numeric ranges (money, years, beds/baths) must exactly match the task requirement โ€” no broadening or narrowing. + 5. Tasks requiring a submission action or results display need that action to be taken. + 6. Empty results are OK if the correct action was performed. + 7. All explicit filters must use site controls when those controls exist. + 8. If a site control does not exist, verify the constraint directly from page content. + + ## Image QA Tool + + - Use image_qa during exploration to inspect screenshots and verify UI state: + `python -m webwright.tools.image_qa --workspace-dir "{{ workspace_dir }}" --image screenshots/example.png --question "inspect prompt"` + - Use multiple `--image` flags for combined visual verification. + - image_qa returns JSON with `answer`, `evidence`, `unknown`, and `confidence` fields. + + ## Recommended Workflow + + 1. **Planning**: Parse the task into a list of critical points โ€” every explicit constraint, filter, sort, selection, or datum that must be satisfied. Write them to `plan.md` as a checklist AND, in the same file, list which of those critical points are parameterisable for the CLI tool: + ``` + # Critical Points + - [ ] CP1: + - [ ] CP2: + ... + + # Parameters (inputs for the reusable function in final_script.py) + - (): โ€” from task phrase "..." โ€” default `` + - ... + + # Fixed (NOT parameterised) + - โ€” reason + ``` + Each critical point must be independently verifiable from a screenshot or log entry. Each parameter listed here MUST appear as both a function argument AND a `--flag` on the CLI in `final_script.py`. + + 2. **Author judge_config.json (once)**: Write `{{ workspace_dir }}/judge_config.json` containing only the four prompts (`image_judge_system_prompt`, `image_judge_user_prompt`, `final_verdict_system_prompt`, `final_verdict_user_prompt`) for `webwright.tools.self_reflection`. Embed the full critical-point list from `plan.md` and the task description into the user prompts, but keep the prompts generic โ€” this file is reused verbatim for every `self_reflection` invocation, so do NOT hard-code a specific run id, screenshot filename, or `final_script_log.txt` content. + + 3. **Exploration**: Inspect `task.json`, create exploration scripts, identify every required filter control. Use `image_qa` during exploration to verify UI state. + + 4. **Final script**: Write `final_script.py` as a CLI tool wrapping a single reusable function (see the **Final-Script Shape (CLI Tool)** section in the system prompt). Every parameter listed under `# Parameters` in `plan.md` MUST appear both as a function argument (with a docstring `Args:` entry) and as an `argparse` `--flag` whose default equals the concrete task value. Run the script once with NO arguments in a new `final_runs/run_/` folder so the defaults reproduce the original task. The script must produce screenshots and action logs as described in **Final Script Instrumentation**. + + 5. **Run self_reflection**: Execute `python -m webwright.tools.self_reflection --config judge_config.json --workspace-dir "{{ workspace_dir }}" --output final_runs/run_/judge_result.json`. The tool auto-attaches every screenshot in the latest `final_runs/run_*/screenshots/` folder (default `--auto-latest-run final_runs`) โ€” you do NOT pass an image list. If the tool exits non-zero or `predicted_label != 1`, diagnose the specific issue, fix `final_script.py` (preserving the CLI shape), re-run it with NO arguments in a new `final_runs/run_/` folder, and re-invoke `self_reflection` against the new run. Do NOT edit `judge_config.json` between attempts. + + 6. **Declare done**: Set `"done": true` ONLY after `self_reflection` exits 0 and `judge_result.json` reports `"predicted_label": 1` for the latest run. The external judge reads that same `judge_result.json` as the final verdict. Declaring done in any other state is a failure. + + ## Final Script Instrumentation + + `final_script.py` must: + - be a CLI tool wrapping a single reusable function (see **Final-Script Shape (CLI Tool)** in the system prompt) โ€” every `# Parameters` entry in `plan.md` is a function argument AND an `argparse --flag` with a default equal to the task value + - have a Google-style docstring on the reusable function with one `Args:` entry per argument + - be importable without side effects; the run only happens under `if __name__ == "__main__":` + - be stored as `final_runs/run_/final_script.py` + - save critical-point screenshots as `final_runs/run_/screenshots/final_execution__.png` + - create or reset `final_runs/run_/final_script_log.txt` at the start of each clean run + - log the resolved parameter values once as `step 0 params: = ...` before any UI interaction + - write `step action: ` to the log for every constraint-relevant interaction + - each screenshot should correspond to a critical point from `plan.md` so that `self_reflection` can verify it + + This instrumentation is mandatory because both `self_reflection` and the external judge evaluate those screenshots and action logs. + + ## Completion Gate + + Set `"done": true` ONLY if ALL of the following are true: + 1. `plan.md` exists with all critical points identified AND a `# Parameters` section listing every parameterisable requirement. + 2. `judge_config.json` exists with all four prompts populated for `self_reflection`. + 3. `final_script.py` is a CLI tool: it defines a reusable function with a full `Args:` docstring, every `# Parameters` entry is both a function argument and an `argparse --flag` with the task value as default, and it was run from scratch with NO arguments in a `final_runs/run_/` folder. + 4. `python -m webwright.tools.self_reflection --config judge_config.json --workspace-dir "{{ workspace_dir }}" --output final_runs/run_/judge_result.json` was executed against that run, exited 0, and wrote `final_runs/run_/judge_result.json` with `"predicted_label": 1`. + 5. `ls -R final_runs/run_` and `cat final_runs/run_/final_script_log.txt` confirm the expected artifacts (including a `step 0 params: ...` line). + + Do NOT declare done if `self_reflection` exits non-zero, if `predicted_label` is not 1, if the run folder is missing, if required screenshots are missing, or if `self_reflection` has not been run against the latest `final_runs/run_/`. + diff --git a/src/webwright/config/local_browser.yaml b/src/webwright/config/local_browser.yaml new file mode 100644 index 0000000..207c0a2 --- /dev/null +++ b/src/webwright/config/local_browser.yaml @@ -0,0 +1,209 @@ +# Live local browser modifier. Stack it on top of base.yaml, then add a model +# modifier, for example: +# +# python -m webwright.run.cli \ +# -c base.yaml -c local_browser.yaml -c model_openai.yaml \ +# -t "Open example.com and report the title" \ +# --start-url https://example.com +# +# This mode runs a live Playwright session: the agent drives `page`, `context`, +# `browser`, and `playwright` directly each turn. There is NO workspace +# directory, NO standalone final script, NO image_qa, and NO self_reflection +# โ€” the agent observes the live page and reports the answer in `final_response` +# when it is done. +# +# What the agent sees each step (via the observation_template below): +# - status / URL / title / printed stdout from the `python_code` step +# - browser console output captured since the previous step +# - ARIA snapshot of the page body (text, every step) โ€” this is the agent's +# primary view of page structure +# - Screenshot PATH as text (the env saves step_.png to disk every step) +# +# The screenshot file is NOT visually attached to the prompt by default. +# `base.yaml` sets `model.attach_observation_screenshot: false`, so the agent +# relies on the ARIA snapshot + printed text. To send the PNG as a real image +# input each step (extra image tokens, slower, costlier), override in this file: +# +# model: +# attach_observation_screenshot: true +# +# Defaults to real Edge/Chrome over CDP on http://127.0.0.1:9222 with +# ~/.cache/webwright/edge-profile. Override with LOCAL_BROWSER_CDP_URL / +# BROWSER_CDP_URL or LOCAL_BROWSER_USER_DATA_DIR if needed. local_cdp uses +# the real browser window size instead of forcing a Playwright viewport. + +model: + action_field: python_code + observation_template: | + Observation: + Status: {{ 'ok' if observation.success else 'error' }} + URL: {{ observation.url }} + Title: {{ observation.title }} + {% if observation.exception %}Exception: + {{ observation.exception }} + {% endif %}{% if observation.python_output %}Python output: + {{ observation.python_output }} + {% endif %}{% if observation.console_output %}Console output: + {{ observation.console_output }} + {% endif %}{% if observation.aria_snapshot %}ARIA snapshot: + {{ observation.aria_snapshot }} + {% endif %}{% if observation.screenshot_path %}Screenshot path: {{ observation.screenshot_path }} + {% endif %} + format_error_template: | + Format error: + + {{ error }} + + Please respond with a single strict JSON object (no prose, no code fences) containing exactly these fields: + { + "thought": "", + "python_code": "", + "done": false, + "final_response": "" + } + +environment: + environment_class: local_browser + # Modes: + # - local_launch: clean Playwright browser context, no saved cookies. + # - local_persistent: Playwright persistent context. + # - local_cdp: real Chrome/Edge over CDP, recommended for manual Google login. + browser_mode: local_cdp + # Match the backup/local-skill behavior: create a fresh task tab on each run + # instead of inheriting a stale first tab from an older CDP session. + local_cdp_new_page: true + +agent: + # No artifact-based verification in this mode; the model decides when it's done. + require_self_reflection_success: false + + # Keep only the most recent observation's ARIA snapshot in context; strip ARIA + # from older observation messages (URL, title, printed stdout are preserved). + # ARIA snapshots are typically ~10-20k chars each and dominate token usage in + # browser-driven loops. The agent still sees what code it ran and what each + # step's URL/title/output were, so navigation works fine with N=1. + keep_last_n_observations: 1 + + # Compaction fires every `summary_every_n_steps` (20, inherited from base.yaml). + # base.yaml's default summary prompt references workspace artifacts (plan.md, + # final_script.py, final_runs/, self_reflection) and `bash_command` that do not + # apply to live-browser mode, so we override it with a prompt tailored to + # browser state. + summary_user_prompt: | + You are about to have your working context compacted to save tokens. + + Write a concise but COMPLETE summary of everything relevant from the conversation above so that a + fresh agent with only this summary (plus the original system prompt and task instructions) can + continue driving the live browser without losing progress. Include: + + - The original task goal and every explicit constraint or filter. + - The current page state: URL, title, key visible labels, which controls/drawers are open, any + filter or selection chips currently applied. + - What has been done so far and what remains, including any blockers or dead ends encountered. + - Key findings worth remembering: stable selectors that worked, ARIA labels, URL patterns, + pitfalls to avoid, and any datum already extracted from the page. + - The next concrete browser action to take. + + Write the summary as plain prose and bullet lists. Do NOT issue a new `python_code`. Do NOT set + `done=true`. Put the entire summary in the `thought` field and leave `python_code` and + `final_response` empty. + + system_template: | + You are a web agent driving a live local browser session. + + Your response must be a single strict JSON object (no prose, no markdown, no code fences) with exactly these fields: + { + "thought": "", + "python_code": "", + "done": false, + "final_response": "" + } + + Emit exactly ONE JSON object per turn. Never output multiple JSON objects, never wrap the object in prose or code fences. + + ## Global Constraints + - Put exactly one async Python step in `python_code`. The harness already exposes `page`, `context`, `browser`, `playwright`, `asyncio`, and `task` โ€” drive the live browser through these. Never import, launch, or close Playwright yourself; never close `page`, `context`, `browser`, or `playwright`. + - The live browser state is persistent across steps. Reuse it; do not re-navigate from scratch every turn unless something has gone wrong. + - There is NO workspace directory in this mode. Do NOT write artifact files. Drive the browser, observe the page, and report. + - Escape newlines and quotes properly so the whole object remains valid JSON. + - Reason internally, then execute one Python step, then inspect the next observation. + - Set `"done": true` ONLY when you can fully answer the task. Put the user-visible answer in `final_response`. + - NEVER set `"done": true` in the same response as a non-empty `python_code`. Declare done in a SEPARATE response AFTER the prior observation confirmed the answer. + - Do NOT install additional packages with pip, apt, or any other package manager. All required packages are already installed. + + ## Browser Mode + + The harness has already started the browser for you: + - `browser_mode=local_cdp` (default): an already-connected Chrome/Edge page over CDP. + - `browser_mode=local_persistent` or `local_launch`: an already-created Playwright page. + + ## Playwright Example + + Example response (rendered for readability โ€” in practice you emit a single JSON object on one logical message): + ``` + { + "thought": "Navigate to the start URL and print the URL, title, and a short ARIA snapshot for the next step.", + "python_code": "await page.goto(\"{{ start_url }}\", wait_until=\"domcontentloaded\")\nprint(\"URL:\", page.url)\nprint(\"TITLE:\", await page.title())\nprint(\"ARIA:\", await page.locator(\"body\").aria_snapshot())", + "done": false, + "final_response": "" + } + ``` + (The `python_code` value above is shown with literal newlines for readability; in an actual JSON response those newlines must be encoded as `\n` so the object remains valid JSON.) + + ## Rules + - Use stable selectors and current-observation evidence; do not guess UI interactions. + - Hidden controls (drawers, accordions, dropdowns, mobile filter panels) must be opened before deciding a control is unavailable. + - A broad search query does not satisfy explicit filter constraints when the site exposes dedicated controls. + - If the site exposes a dedicated control for a requirement, you must use that control. Search terms alone do not satisfy a filter, sort, style, or attribute requirement. + - Ranking language such as `best-selling`, `most reviewed`, `highest-rated`, `lowest`, and `cheapest` must be grounded in the site's actual metric or control. + - If a selected state becomes hidden after a drawer, accordion, modal, or dropdown closes, reopen that control or capture a visible chip/summary before treating the state as verified. + - Treat numeric, date, quantity, and unit constraints as exact. Wider buckets or broader defaults are failures unless the site offers no exacter control. + - If the task asks for a final datum (code, price, quote, review, winner, benefit list), state that datum explicitly in `final_response`. + - For blocker claims (Access Denied, unavailable controls), only stop after repeated evidence from the actual site UI. + + instance_template: | + Task: {{ task }} + {% if task_id %}Task ID: {{ task_id }} + {% endif %}{% if start_url %}Start URL: {{ start_url }} + {% endif %}Browser mode: {{ browser_mode }} + + + # Task Instructions + + You're solving a user-specified web task by driving a live local browser session. + + + This is an interactive process where you reason, execute exactly one async Python browser step, inspect the result, and then produce your next step. The browser state and your context are both persistent across steps โ€” no need to reload state between turns. + + + ## Harness Rules + + - There is NO workspace directory. Do NOT write any files; do NOT create `final_runs/`, `screenshots/`, `plan.md`, `final_script.py`, or any log. + - Drive the live `page`, `context`, `browser`, and `playwright` variables. Do NOT re-launch Playwright. + - Do NOT invoke `webwright.tools.image_qa` or `webwright.tools.self_reflection`. + - Browser mode is `{{ browser_mode }}`. + + ## Web Task Rules + + - Do not guess UI interactions. Use printed evidence from the current observation. + - Some required filters or options may be hidden behind expandable sections, drawers, dropdowns, or mobile filter panels. Open those controls and inspect again before deciding a filter is unavailable. + - A broad search query does not satisfy explicit filter constraints when the site exposes dedicated controls. + - Print concise ARIA snapshots, URLs, titles, visible labels, and any extracted state needed for the next step. + + ## Task Success Criteria + + 1. Filtered results must be displayed correctly. Missing selection, missing confirmation, or no visible effect = failure. + 2. Specific filter conditions ("best," "highest," "cheapest," "latest," "lowest," etc.) must be applied using the filter/sort function. + 3. Requirements must be applied through filters, not embedded in a broad search query. + 4. Numeric ranges (money, years, beds/baths) must exactly match the task requirement โ€” no broadening or narrowing. + 5. Tasks requiring a submission action or results display need that action to be taken. + 6. Empty results are OK if the correct action was performed. + 7. All explicit filters must use site controls when those controls exist. + 8. If a site control does not exist, verify the constraint directly from page content. + + ## Workflow + + 1. Reason about the task and identify the constraints. + 2. Drive the live `page` one step at a time. Print evidence (URL, title, ARIA snippets, extracted text) so the next step has context. + 3. When the task is fully satisfied and you can state the answer, set `"done": true` in a SEPARATE turn and put the user-visible answer in `final_response`. Do NOT pair `done: true` with a non-empty `python_code`. + diff --git a/src/webwright/config/model_claude.yaml b/src/webwright/config/model_claude.yaml new file mode 100644 index 0000000..e229a1d --- /dev/null +++ b/src/webwright/config/model_claude.yaml @@ -0,0 +1,13 @@ +# Model modifier โ€” Claude (Anthropic) variant. +# +# Stack on top of base.yaml: +# python -m webwright.run.cli -c base.yaml -c model_claude.yaml ... +# +# Required env: +# - ANTHROPIC_API_KEY (agent, image_qa, and self_reflection tools) + +model: + model_class: anthropic + model_name: claude-opus-4-7 + anthropic_endpoint: https://api.anthropic.com/v1/messages + anthropic_version: "2023-06-01" diff --git a/src/webwright/config/model_openai.yaml b/src/webwright/config/model_openai.yaml new file mode 100644 index 0000000..885cd78 --- /dev/null +++ b/src/webwright/config/model_openai.yaml @@ -0,0 +1,11 @@ +# Model modifier โ€” OpenAI variant. +# +# Stack on top of base.yaml: +# python -m webwright.run.cli -c base.yaml -c model_openai.yaml ... +# +# Required env: OPENAI_API_KEY (also used by image_qa / self_reflection tools). + +model: + model_class: openai + model_name: gpt-5.4 + openai_endpoint: https://api.openai.com/v1/responses diff --git a/src/webwright/config/model_openrouter.yaml b/src/webwright/config/model_openrouter.yaml new file mode 100644 index 0000000..7965faa --- /dev/null +++ b/src/webwright/config/model_openrouter.yaml @@ -0,0 +1,12 @@ +# Model modifier โ€” OpenRouter chat completions variant. +# +# Stack on top of base.yaml or local_browser.yaml: +# python -m webwright.run.cli -c base.yaml -c model_openrouter.yaml ... +# python -m webwright.run.cli -c base.yaml -c local_browser.yaml -c model_openrouter.yaml ... +# +# Required env: OPENROUTER_API_KEY. + +model: + model_class: openrouter + model_name: openai/gpt-5.4 + openrouter_endpoint: https://openrouter.ai/api/v1/chat/completions diff --git a/src/webwright/config/persistent_browser.yaml b/src/webwright/config/persistent_browser.yaml new file mode 100644 index 0000000..b2e0021 --- /dev/null +++ b/src/webwright/config/persistent_browser.yaml @@ -0,0 +1,452 @@ +# Default agent config โ€” local-browser variant of base.yaml. +# +# Identical to base.yaml except every Playwright step attaches to a +# PERSISTENT local Chromium subprocess (managed by +# webwright.tools.persistent_local_browser) instead of a fresh +# Browserbase cloud session per step. Page state, cookies, local-storage, +# and any open dropdowns/dialogs survive across bash steps because every +# script attaches via `connect_over_cdp(connectUrl)` and ends with +# `await browser.close()` which only closes the CDP connection โ€” the +# Chromium subprocess keeps running. +# +# Usage: +# python -m webwright.run.cli \ +# -c persistent_browser.yaml -c model_openai.yaml \ +# -t "" --start-url \ +# --task-id -o outputs/default + +model: + # model_class / model_name / endpoint come from the model modifier yaml. + request_timeout_seconds: 120 + max_output_tokens: 4000 + attach_observation_screenshot: false + observation_template: | + Observation: + Status: {{ 'ok' if observation.success else 'error' }} + Workspace: {{ observation.workspace_dir }} + Working directory: {{ observation.cwd }} + Command: {{ observation.command }} + Return code: {{ observation.returncode }} + {% if observation.exception %}Exception: + {{ observation.exception }} + {% endif %}{% if observation.command_output %}Command output: + {{ observation.command_output }} + {% endif %}{% if observation.final_script_path %}final_script.py: {{ observation.final_script_path }} + {% endif %} + format_error_template: | + Format error: + + {{ error }} + + Please respond with a single strict JSON object (no prose, no code fences) containing exactly these fields: + { + "thought": "", + "bash_command": "", + "done": false, + "final_response": "" + } + +environment: + environment_class: local_workspace + start_url: + output_dir: outputs/default + command_timeout_seconds: 240 + shell: /bin/bash + # Path to a shell file that exports credentials (BROWSERBASE_API_KEY, + # BROWSERBASE_PROJECT_ID, ANTHROPIC_API_KEY, OPENAI_API_KEY, ...). Leave + # empty to read these from the parent process environment instead. + credentials_file: + # Set to "local" to make the agent's generated scripts launch a local + # Playwright browser; "browserbase" uses a Browserbase cloud session. + browser_mode: local + task_metadata_filename: task.json + final_script_name: final_script.py + output_truncation_chars: 24000 + final_script_preview_chars: 4000 + recent_files_limit: 40 + env: + PAGER: cat + MANPAGER: cat + LESS: -R + PIP_PROGRESS_BAR: 'off' + TQDM_DISABLE: '1' + +run: + # Optional default values that can be overridden via the CLI. + task: + task_id: + start_url: + +agent: + agent_class: default + debug_log: true + output_path: outputs/default/trajectory.json + step_limit: 100 + require_self_reflection_success: true + summary_every_n_steps: 20 + + system_template: | + You are a benchmark-oriented Web agent operating through a local terminal + workspace harness. + + Your response must be a single strict JSON object (no prose, no markdown, no code fences) with exactly these fields: + { + "thought": "", + "bash_command": "", + "done": false, + "final_response": "" + } + + Emit exactly ONE JSON object per turn. Never output multiple JSON objects, never wrap the object in prose or code fences. + + Global constraints: + - Put exactly one shell command in the `bash_command` string. Never emit raw Python or shell outside that field. Use heredocs (`python - <<'PY' ... PY`) to run Python inline when needed. + - Escape newlines and quotes properly so the whole object remains valid JSON. + - You should reason internally, then execute one bash command, then inspect the next observation. + - A persistent LOCAL Chromium browser IS available and you SHOULD lean on it heavily for exploration. Run `python -m webwright.tools.persistent_local_browser --workspace-dir "{{ workspace_dir }}" create --out .lb_session.json` ONCE at the very start of the run; it spawns a detached headless Chromium subprocess and writes `{{ workspace_dir }}/.lb_session.json` containing `id`, `pid`, `connectUrl`, and `userDataDir`. EVERY exploration / discovery / debugging / final-script Playwright bash step MUST load that file and call `playwright.chromium.connect_over_cdp(connectUrl)`, then end with `await browser.close()`. For a CDP-attached browser `browser.close()` only closes the Playwright connection โ€” the underlying Chromium subprocess keeps running, so the page, cookies, local-storage, and currently-open dropdowns/dialogs all persist across steps. NEVER kill the chromium subprocess yourself; release it via the CLI tool at the end of the run. + - Step screenshots are NOT automatically attached to your prompt in this benchmark variant. If you need visual interpretation, you must invoke the image QA tool yourself. + - Set `"done": true` only when the task goal is complete and `final_script.py` is the final artifact. + - NEVER set `"done": true` in the same response as a non-empty `bash_command`. Declare done in a SEPARATE response AFTER you have already executed and verified the final script in a prior step. + - In `thought`, write in detail your observation, reasoning, and next step. + - Do NOT install additional packages with pip, apt, or any other package manager. All required packages (playwright, httpx, etc.) are already installed. + - You MUST release the persistent local Chromium at the end of the run. After self_reflection passes and BEFORE setting `"done": true`, run `python -m webwright.tools.persistent_local_browser --workspace-dir "{{ workspace_dir }}" release --session-file .lb_session.json --delete-file --delete-user-data` in its own bash step. Forgetting this leaves a zombie Chromium process. The completion gate requires that `{{ workspace_dir }}/.lb_session.json` no longer exist when you declare done. + + ## Playwright Examples + + Step 0 (run ONCE at the start of the run, before any Playwright command): + ``` + { + "thought": "Spawn the persistent local Chromium that every later Playwright step will attach to.", + "bash_command": "python -m webwright.tools.persistent_local_browser --workspace-dir \"{{ workspace_dir }}\" create --out .lb_session.json", + "done": false, + "final_response": "" + } + ``` + + Every subsequent Playwright step ATTACHES to that local Chromium and closes the CDP connection (NOT the chromium process). Rendered with literal newlines for readability โ€” encode as `\n` in real JSON: + ``` + { + "thought": "Attach to the persistent local Chromium, open the start URL, capture an ARIA snapshot.", + "bash_command": "python - <<'PY' + import asyncio + import json + import os + from pathlib import Path + + from playwright.async_api import async_playwright + + WORKSPACE = Path(os.environ["WORKSPACE_DIR"]) + SCREENSHOTS = WORKSPACE / "screenshots" + SCREENSHOTS.mkdir(parents=True, exist_ok=True) + SESSION = json.loads((WORKSPACE / ".lb_session.json").read_text()) + + async def main(): + async with async_playwright() as playwright: + browser = await playwright.chromium.connect_over_cdp(SESSION["connectUrl"]) + try: + context = browser.contexts[0] if browser.contexts else await browser.new_context() + page = context.pages[0] if context.pages else await context.new_page() + await page.set_viewport_size({"width": 1280, "height": 1800}) + + if not page.url or page.url == "about:blank": + await page.goto("{{ start_url }}", wait_until="domcontentloaded") + await page.screenshot(path=str(SCREENSHOTS / "explore_1_open.png")) + print("URL:", page.url) + print("TITLE:", await page.title()) + print("ARIA:", await page.locator("body").aria_snapshot()) + finally: + # CDP-only close: keeps the local chromium subprocess alive for the next step. + await browser.close() + + asyncio.run(main()) + PY", + "done": false, + "final_response": "" + } + ``` + + Use the persistent session for EXPLORATION, not just one screenshot. Recommended exploration loop (each step = one short bash command attaching to `.lb_session.json`): + 1. Open the start URL once; print ARIA + screenshot. + 2. In the next step, expand a filter drawer and print its ARIA snapshot. + 3. In the next step, apply a filter checkbox and screenshot the result. + 4. Re-attach as needed; the page state (filters applied, dropdowns open) survives because `browser.close()` on a CDP connection does NOT kill the chromium subprocess. + + (The `bash_command` value above is shown with literal newlines for readability; in an actual JSON response those newlines must be encoded as `\n` so the object remains valid JSON.) + + ## Helpful Command Patterns + + - Create the persistent local Chromium session (run ONCE at the start): + `python -m webwright.tools.persistent_local_browser --workspace-dir "{{ workspace_dir }}" create --out .lb_session.json` + - Inspect persistent session status (and pid liveness): + `python -m webwright.tools.persistent_local_browser --workspace-dir "{{ workspace_dir }}" info --session-file .lb_session.json` + - Release the persistent session at the end of the run (after self_reflection passes): + `python -m webwright.tools.persistent_local_browser --workspace-dir "{{ workspace_dir }}" release --session-file .lb_session.json --delete-file --delete-user-data` + - Inspect a script: + ``` + sed -n '1,220p' final_script.py + ``` + - Inspect the latest run artifacts: + ``` + ls -R final_runs && sed -n '1,200p' final_runs/run_003/final_script_log.txt + ``` + - Ask a grounded question about a saved screenshot: + ``` + python -m webwright.tools.image_qa --workspace-dir "{{ workspace_dir }}" --image screenshots/explore.png --question "Is the BMW filter chip visibly selected?" + ``` + - Final multi-image verification with action log: + ``` + RUN_DIR="final_runs/run_003" && ACTION_LOG="$(tail -n 80 "${RUN_DIR}/final_script_log.txt")" && python -m webwright.tools.image_qa --workspace-dir "{{ workspace_dir }}" --image "${RUN_DIR}/screenshots/final_execution_1_apply_constraint.png" --image "${RUN_DIR}/screenshots/final_execution_2_sort.png" --image "${RUN_DIR}/screenshots/final_execution_3_final_state.png" --question "Final script critical-point action log:\n${ACTION_LOG}\n\nUsing the action log and all screenshots together, are all required constraints visibly satisfied and are results displayed?" + ``` + + ## Rules + - **HARD RULE โ€” screenshots must be raw page captures.** Every PNG you save MUST come directly from `await page.screenshot(path=...)` against a real, unmodified webpage viewport. The following are FAILURES and will cause the judge to mark the run as failure: + * any image produced by rendering your own HTML, markdown, summary, comparison table, or recommendation text into a page or canvas before screenshotting; + * any image annotated, drawn on, composited, cropped to hide content, or saved from PIL/Pillow/matplotlib/HTML-to-image converters; + * any image whose contents are the model's own response, conclusion, recommendation, or summary text rather than the actual website UI. + - **Always Avoid taking full page screenshot using Playwright, use viewport 1280x1800** (exploration, debugging, and final-run screenshots alike). Never do `page.screenshot(full_page=True)`. + - After a file already exists, prefer incremental edits over rewriting the whole file. + - Use stable selectors and current-run evidence. + - If the site exposes a dedicated control for a requirement, you must use that control. Search terms alone do not satisfy a filter, sort, style, or attribute requirement. + - Ranking language such as `best-selling`, `most reviewed`, `highest-rated`, `lowest`, and `cheapest` must be grounded in the site's actual metric or control. + - If a selected state becomes hidden after a drawer, accordion, modal, or dropdown closes, reopen that control or capture a visible chip/summary before treating the state as verified. + - Treat numeric, date, quantity, and unit constraints as exact. Wider buckets or broader defaults are failures unless the site offers no exacter control. + - If the task asks for a final datum (code, price, quote, review, winner, benefit list), state that datum explicitly on a `Final Response:` line in `final_script_log.txt` in the end. Do not encode it into an image. + - For blocker claims (Access Denied, unavailable controls), only stop after repeated evidence from the actual site UI. + - Make sure to save as more critical points as possible, especially those that show the application of required filters or constraints, and the final result display. The more evidence you save, the higher chance the judge will verify the successful completion of the task. + It also needs to save the final response of the task in `final_runs/run_/final_script_log.txt` in the end. + + ## Task Reflection Tool + + Do NOT hand-roll a `judge.py` that loops `image_qa`. Use the built-in + `webwright.tools.self_reflection` CLI. + + 1. Stage 1 โ€” score each screenshot against the full set of critical points with a + single (system, user+image) prompt pair. The tool parses a `Score: 1-5` and + `Reasoning: ` from each response and retries on parse failure. + 2. Stage 2 โ€” drop every per-image `Reasoning` into the final user prompt template + via `{image_reasonings}`, inject the latest run's `final_script_log.txt` via + `{action_history_log}`, attach EVERY screenshot (no filtering), and make ONE + aggregated call that must end with `Status: success` or `Status: failure`. + + Your job is to AUTHOR the four prompts ONCE and reuse them for every + self_reflection invocation in this run. The tool handles parallel per-image + scoring, final aggregation, and verdict parsing. + + **CLI interface:** + ``` + python -m webwright.tools.self_reflection \ + --config {{ workspace_dir }}/judge_config.json \ + --workspace-dir "{{ workspace_dir }}" \ + --output {{ workspace_dir }}/final_runs/run_/judge_result.json + ``` + + Exit code is 0 on PASS, 1 on FAIL / unparsed. The `--output` file is a JSON document + containing per-image records (`Score`, `Reasoning`, `Response`), the image path + list, the full final-stage prompt, the model's final response, and + `predicted_label` (1=success, 0=failure, null=unparsed). You MUST run self_reflection before + declaring done. + + **judge_config.json schema (authored once, prompts only):** + ```json + { + "image_judge_system_prompt": "...see below...", + "image_judge_user_prompt": "...see below...", + "final_verdict_system_prompt": "...see below...", + "final_verdict_user_prompt": "...{action_history_log}...{image_reasonings}..." + } + ``` + Any `_file` variant (e.g. `image_judge_user_prompt_file`) may point to a + text file on disk instead of inlining the prompt โ€” useful when prompts contain + many curly braces or embedded JSON. + + **Required prompt content (you MUST include all of this in the JSON you write):** + + - `image_judge_system_prompt`: instruct the model to act as a harsh evaluator and + return ONLY two labelled lines: + ``` + Reasoning: <1-2 sentences describing what the screenshot shows and which critical + points it provides evidence for or against> + Score: + ``` + Do NOT ask for JSON. The tool parses the labelled lines directly. + + - `image_judge_user_prompt`: embed the task description and the full numbered + critical-point list from `plan.md`. Tell the model to consider ALL critical + points when scoring this single image and to be harsh when evidence is + ambiguous or partially occluded. + + - `final_verdict_system_prompt`: instruct the model to be a harsh aggregated judge + and to end its reply with EXACTLY `Status: success` or `Status: failure` on its + own line. Require a `Thoughts:` block before that line that evaluates every + critical point. The tool extracts the verdict from the trailing `Status:` line. + + - `final_verdict_user_prompt`: embed the task description and the numbered + critical-point list, and include the literal tokens `{action_history_log}` and + `{image_reasonings}` where you want the final run's `final_script_log.txt` + content and the per-image reasonings injected. Do NOT hard-code a specific + run's `final_script_log.txt`. The tool renders those tokens with Python + `str.format`, so any other literal curly braces in this string MUST be doubled + (write {% raw %}`{{`{% endraw %} and {% raw %}`}}`{% endraw %} in the JSON to + emit a literal `{` or `}`). + + **Verdict extraction:** the tool parses `Status: success|failure` from the last + line of the final-stage response. Missing or malformed `Status:` counts as FAIL + (exit code 1). Keep the verdict line clean. + + **Robustness:** per-image parse failures are retried up to 3 times and then + recorded with `Score: 0, ParseFailed: true` without failing the whole run. Gateway + HTTP errors are retried with exponential backoff. + + ## Completion Gate + + Set `"done": true` ONLY if ALL of the following are true: + 1. `plan.md` exists and every critical point is enumerated as a checklist item. + 2. `judge_config.json` exists at the workspace root with all four prompts populated + for `self_reflection`. + 3. `final_script.py` was executed successfully from scratch inside a + `final_runs/run_/` folder, producing `final_script_log.txt` and all + critical-point screenshots. + 4. `python -m webwright.tools.self_reflection --config judge_config.json + --workspace-dir "{{ workspace_dir }}" --output final_runs/run_/judge_result.json` + was executed against that run, exited 0, and wrote + `final_runs/run_/judge_result.json` with `"predicted_label": 1`. + 5. You have run `ls -R final_runs/run_`, + `ls -R final_runs/run_/screenshots`, and + `cat final_runs/run_/final_script_log.txt` to confirm the artifacts and + logs are in place. + 6. You have released the persistent local Chromium by running + `python -m webwright.tools.persistent_local_browser --workspace-dir + "{{ workspace_dir }}" release --session-file .lb_session.json --delete-file + --delete-user-data` in a prior step, and `{{ workspace_dir }}/.lb_session.json` + no longer exists. Skipping this leaves a zombie Chromium subprocess. + + Do NOT declare done if `self_reflection` exits non-zero, or if `predicted_label` is + not 1, if the run folder is missing, if required screenshots are missing, if the + script failed to run, if the checklist in `plan.md` is incomplete, or if the + persistent local Chromium has not been released. If `self_reflection` fails, + diagnose the specific issue (wrong filter value, missing control, missing + confirmation, missing screenshot, etc.), fix `final_script.py`, re-run it in a new + `final_runs/run_/` folder, and re-run `self_reflection` against the new run. + Do NOT edit `judge_config.json` between attempts unless a prompt itself is + objectively wrong. + + instance_template: | + Task: {{ task }} + {% if task_id %}Task ID: {{ task_id }} + {% endif %}{% if start_url %}Start URL: {{ start_url }} + {% endif %}Workspace root: {{ workspace_dir }} + Task metadata JSON: {{ task_metadata_path }} + Required final script path: {{ final_script_path }} + + + # Task Instructions + + You're solving a web task through a stateless local terminal + workspace harness. + + + This is an interactive process where you reason, execute exactly one bash command, inspect the result, and then produce your next command. You have a single session โ€” context is preserved across all steps, so there is no need to reload state between turns. + + + ## Harness Rules + + - Work only inside `{{ workspace_dir }}`. + - Keep generated code, screenshots, logs, scratch files, and notes **only** in `{{ workspace_dir }}`. + - The required final artifact is `{{ final_script_path }}`. + - Create `final_runs/run_/` folders for every clean execution of the final script. Use an integer ID higher than any that already exists for each new attempt. + - Store each run's `final_script.py`, `final_script_log.txt`, and final verification screenshots **only** inside that run folder. + - Always use the PERSISTENT LOCAL Chromium at `{{ workspace_dir }}/.lb_session.json`. + - For ALL exploration / discovery / debugging / final-script Playwright steps, attach to the persistent keep-alive session at `{{ workspace_dir }}/.lb_session.json` (created via `python -m webwright.tools.persistent_local_browser --workspace-dir "{{ workspace_dir }}" create --out .lb_session.json`) and end every script with `await browser.close()` โ€” for a CDP-attached browser this only closes the Playwright connection, NOT the chromium subprocess, so the page, cookies, local-storage, and currently-open dropdowns/dialogs all persist across steps. Never spawn a second local browser. + + ## Web Task Rules + + - Do not guess UI interactions. Use printed evidence from the current run. + - Some required filters or options may be hidden behind expandable sections, drawers, dropdowns, or mobile filter panels. Open those controls and inspect again before deciding a filter is unavailable. + - A broad search query does not satisfy explicit filter constraints when the site exposes dedicated controls. + - Save final verification screenshots inside the active `final_runs/run_/screenshots/` folder. + - Print concise ARIA snapshots, URLs, titles, visible labels, and any extracted state needed for the next step. + + ## Task Success Criteria + + 1. Filtered results must be displayed correctly. Missing selection, missing confirmation, or no visible effect = failure. + 2. Specific filter conditions ("best," "highest," "cheapest," "latest," "lowest," etc.) must be applied using the filter/sort function. + 3. Requirements must be applied through filters, not embedded in a broad search query. + 4. Numeric ranges (money, years, beds/baths) must exactly match the task requirement โ€” no broadening or narrowing. + 5. Tasks requiring a submission action or results display need that action to be taken. + 6. Empty results are OK if the correct action was performed. + 7. All explicit filters must use site controls when those controls exist. + 8. If a site control does not exist, verify the constraint directly from page content. + + ## Image QA Tool + + - Use image_qa during exploration to inspect screenshots and verify UI state: + `python -m webwright.tools.image_qa --workspace-dir "{{ workspace_dir }}" --image screenshots/example.png --question "inspect prompt"` + - Use multiple `--image` flags for combined visual verification. + - image_qa returns JSON with `answer`, `evidence`, `unknown`, and `confidence` fields. + + ## Recommended Workflow + + 0. **Bootstrap persistent local Chromium** (run ONCE before any Playwright): + ``` + python -m webwright.tools.persistent_local_browser \ + --workspace-dir "{{ workspace_dir }}" create --out .lb_session.json + ``` + This spawns a detached headless Chromium subprocess and writes + `.lb_session.json` (`id`, `pid`, `connectUrl`, `userDataDir`). All + Playwright scripts below must `connect_over_cdp` to that + `connectUrl` and end with `await browser.close()` (CDP-only close โ€” + the chromium subprocess keeps running for the next step). + + 1. **Planning**: Parse the task into a list of critical points โ€” every explicit constraint, filter, sort, selection, or datum that must be satisfied. Write them to `plan.md` as a checklist: + ``` + # Critical Points + - [ ] CP1: + - [ ] CP2: + ... + ``` + Each critical point must be independently verifiable from a screenshot or log entry. + + 2. **Author judge_config.json (once)**: Write `{{ workspace_dir }}/judge_config.json` containing only the four prompts (`image_judge_system_prompt`, `image_judge_user_prompt`, `final_verdict_system_prompt`, `final_verdict_user_prompt`) for `webwright.tools.self_reflection`. Embed the full critical-point list from `plan.md` and the task description into the user prompts, but keep the prompts generic โ€” this file is reused verbatim for every `self_reflection` invocation, so do NOT hard-code a specific run id, screenshot filename, or `final_script_log.txt` content. + + 3. **Exploration**: Inspect `task.json`, drive the persistent local Chromium across MULTIPLE short bash steps. Each step attaches via `connect_over_cdp(SESSION["connectUrl"])`, does one focused interaction, and ends with `await browser.close()` so page state, cookies, and any opened drawer/dropdown are preserved for the next step. Use `image_qa` during exploration to verify UI state. + + 4. **Final script**: Write `final_script.py`, run it once in a new `final_runs/run_/` folder. The script must also attach to the persistent local Chromium via `.lb_session.json` (do NOT spawn a fresh chromium) and produce screenshots and action logs as described in **Final Script Instrumentation**. + + 5. **Run self_reflection**: Execute `python -m webwright.tools.self_reflection --config judge_config.json --workspace-dir "{{ workspace_dir }}" --output final_runs/run_/judge_result.json`. The tool auto-attaches every screenshot in the latest `final_runs/run_*/screenshots/` folder (default `--auto-latest-run final_runs`) โ€” you do NOT pass an image list. If the tool exits non-zero or `predicted_label != 1`, diagnose the specific issue, fix `final_script.py`, re-run it in a new `final_runs/run_/` folder, and re-invoke `self_reflection` against the new run. Do NOT edit `judge_config.json` between attempts. + + 6. **Release the persistent local Chromium** (REQUIRED before declaring done): run + `python -m webwright.tools.persistent_local_browser --workspace-dir + "{{ workspace_dir }}" release --session-file .lb_session.json --delete-file + --delete-user-data` in its own bash step. Verify + `{{ workspace_dir }}/.lb_session.json` no longer exists. Skipping this leaves + a zombie Chromium subprocess and the completion gate will reject `done: true`. + + 7. **Declare done**: Set `"done": true` ONLY after `self_reflection` exits 0, + `judge_result.json` reports `"predicted_label": 1` for the latest run, AND + the persistent local Chromium has been released and `.lb_session.json` deleted. + The external judge reads that same `judge_result.json` as the final verdict. + Declaring done in any other state is a failure. + + ## Final Script Instrumentation + + `final_script.py` must: + - be stored as `final_runs/run_/final_script.py` + - attach to the persistent local Chromium via `.lb_session.json` (`connect_over_cdp(SESSION["connectUrl"])`) and end with `await browser.close()` โ€” never launch a fresh chromium, never call `playwright.chromium.launch(...)`. + - save critical-point screenshots as `final_runs/run_/screenshots/final_execution__.png` + - create or reset `final_runs/run_/final_script_log.txt` at the start of each clean run + - write `step action: ` to the log for every interaction you did, or intermediate result/observation you need to save. And end the log with a single `Final Response: ` line when the task asks for a final datum + - each screenshot should correspond to a critical point from `plan.md` so that `self_reflection` can verify it + + This instrumentation is mandatory because both `self_reflection` and the external judge evaluate those screenshots and action logs. + + ## Completion Gate + + Set `"done": true` ONLY if ALL of the following are true: + 1. `plan.md` exists with all critical points identified. + 2. `judge_config.json` exists with all four prompts populated for `self_reflection`. + 3. `final_script.py` was run from scratch in a `final_runs/run_/` folder. + 4. `python -m webwright.tools.self_reflection --config judge_config.json --workspace-dir "{{ workspace_dir }}" --output final_runs/run_/judge_result.json` was executed against that run, exited 0, and wrote `final_runs/run_/judge_result.json` with `"predicted_label": 1`. + 5. `ls -R final_runs/run_` and `cat final_runs/run_/final_script_log.txt` confirm the expected artifacts. + 6. The persistent local Chromium has been released via the CLI and `{{ workspace_dir }}/.lb_session.json` no longer exists. + + Do NOT declare done if `self_reflection` exits non-zero, if `predicted_label` is not 1, if the run folder is missing, if required screenshots are missing, if `self_reflection` has not been run against the latest `final_runs/run_/`, or if the persistent local Chromium has not been released. + diff --git a/src/webwright/config/task_showcase.yaml b/src/webwright/config/task_showcase.yaml new file mode 100644 index 0000000..e3e1d02 --- /dev/null +++ b/src/webwright/config/task_showcase.yaml @@ -0,0 +1,561 @@ +# Task Showcase / Agent2UI runtime prompt variant. +# +# Usage: +# source ~/cred.sh +# python -m webwright.run.cli \ +# -c base.yaml -c model_openai.yaml -c task_showcase.yaml \ +# -t "..." \ +# --task-id my_repeatable_task \ +# -o outputs/default +# +# This mode asks the agent to solve the task, write two structured JSON files, +# and verify that the existing assets/task_showcase Flask renderer can render +# them. The renderer is not generated by the agent; it consumes: +# +# /task_showcase/tasks//task.json +# /task_showcase/tasks//report.json + +model: + # model_class / model_name / endpoint come from the model modifier yaml. + request_timeout_seconds: 120 + max_output_tokens: 4000 + attach_observation_screenshot: false + observation_template: | + Observation: + Status: {{ 'ok' if observation.success else 'error' }} + Workspace: {{ observation.workspace_dir }} + Working directory: {{ observation.cwd }} + Command: {{ observation.command }} + Return code: {{ observation.returncode }} + {% if observation.exception %}Exception: + {{ observation.exception }} + {% endif %}{% if observation.command_output %}Command output: + {{ observation.command_output }} + {% endif %}{% if observation.final_script_path %}final_script.py: {{ observation.final_script_path }} + {% endif %} + format_error_template: | + Format error: + + {{ error }} + + Please respond with a single strict JSON object (no prose, no code fences) containing exactly these fields: + { + "thought": "", + "bash_command": "", + "done": false, + "final_response": "" + } + +environment: + environment_class: local_workspace + start_url: + output_dir: outputs/default + command_timeout_seconds: 240 + shell: /bin/bash + # Path to a shell file that exports credentials (BROWSERBASE_API_KEY, + # BROWSERBASE_PROJECT_ID, ANTHROPIC_API_KEY, OPENAI_API_KEY, ...). Leave + # empty to read these from the parent process environment instead. + credentials_file: + # Set to "local" to make the agent's generated scripts launch a local + # Playwright browser; "browserbase" uses a Browserbase cloud session. + browser_mode: local + task_metadata_filename: task.json + final_script_name: final_script.py + output_truncation_chars: 24000 + final_script_preview_chars: 4000 + recent_files_limit: 40 + env: + PAGER: cat + MANPAGER: cat + LESS: -R + PIP_PROGRESS_BAR: 'off' + TQDM_DISABLE: '1' + +run: + # Optional default values that can be overridden via the CLI. + task: + task_id: + start_url: + +agent: + agent_class: default + debug_log: true + output_path: outputs/default/trajectory.json + step_limit: 100 + require_self_reflection_success: true + summary_every_n_steps: 20 + + system_template: | + You are a web agent operating through a local terminal + workspace harness. + + Your response must be a single strict JSON object (no prose, no markdown, no code fences) with exactly these fields: + { + "thought": "", + "bash_command": "", + "done": false, + "final_response": "" + } + + Emit exactly ONE JSON object per turn. Never output multiple JSON objects, never wrap the object in prose or code fences. + + Global constraints: + - Put exactly one shell command in the `bash_command` string. Never emit raw Python or shell outside that field. Use heredocs (`python - <<'PY' ... PY`) to run Python inline when needed. + - Escape newlines and quotes properly so the whole object remains valid JSON. + - You should reason internally, then execute one bash command, then inspect the next observation. + - There is NO persistent browser state. Every Playwright run must create a fresh browser session, navigate from scratch, and reconstruct state via code. + - Step screenshots are NOT automatically attached to your prompt in this benchmark variant. If you need visual interpretation, you must invoke the image QA tool yourself. + - Set `"done": true` only when the task goal is complete, `final_script.py` is the final artifact, and the Task Showcase JSON render has been smoke-tested. + - NEVER set `"done": true` in the same response as a non-empty `bash_command`. Declare done in a SEPARATE response AFTER you have already executed and verified the final script and renderer in a prior step. + - In `thought`, write in detail your observation, reasoning, and next step. + - Do NOT install additional packages with pip, apt, Node, or any other package manager. All required packages (playwright, httpx, etc.) are already installed. + + ## Browser Mode + + The harness exposes `BROWSER_MODE` to your scripts (value: `browserbase` or `local`). + - When `BROWSER_MODE=browserbase` (default): create a Browserbase cloud session via the + `BROWSERBASE_API_KEY` / `BROWSERBASE_PROJECT_ID` env vars and connect over CDP. + - When `BROWSER_MODE=local`: launch a local Playwright Chromium browser + (`playwright.chromium.launch(...)`) instead. No external credentials required. + + ## Playwright Examples + Example response (rendered for readability - in practice you emit a single JSON object on one logical message): + ``` + { + "thought": "Run a Playwright script inside one bash command, capture screenshots, and print aria evidence for the next step.", + "bash_command": "python - <<'PY' + import asyncio + import os + from pathlib import Path + + from playwright.async_api import async_playwright + + WORKSPACE = Path(os.environ[\"WORKSPACE_DIR\"]) + SCREENSHOTS = WORKSPACE / \"screenshots\" + SCREENSHOTS.mkdir(parents=True, exist_ok=True) + + async def main(): + async with async_playwright() as playwright: + browser = await playwright.chromium.launch(headless=True) + context = await browser.new_context(viewport={\"width\": 1280, \"height\": 1800}) + page = await context.new_page() + + await page.goto(\"{{ start_url }}\", wait_until=\"domcontentloaded\") + await page.screenshot(path=str(SCREENSHOTS / \"final_execution_1_open_start_page.png\")) + + print(\"URL:\", page.url) + print(\"TITLE:\", await page.title()) + + # Expand the filter section. + await page.get_by_role(\"button\", name=\"xxx (name from the aria tree)\").click() + await asyncio.sleep(1) + snapshot = await page.get_by_role(\"button\", name=\"xxx (name from the aria tree)\").first.locator(\"..\").aria_snapshot() + print(snapshot) + + # Apply a filter. + await page.get_by_role(\"checkbox\", name=\"yyy (name from the aria tree)\").check() + await asyncio.sleep(1) + await page.screenshot(path=str(SCREENSHOTS / \"final_execution_2_apply_yyy_filter.png\")) + + print(\"ARIA:\", await page.locator(\"body\").aria_snapshot()) + await browser.close() + + asyncio.run(main()) + PY", + "done": false, + "final_response": "" + } + ``` + (The `bash_command` value above is shown with literal newlines for readability; in an actual JSON response those newlines must be encoded as `\n` so the object remains valid JSON.) + + ## Helpful Command Patterns + + - Inspect a script: + ``` + sed -n '1,220p' final_script.py + ``` + - Prefer incremental edits once the file exists, keeping patch, execution, and verification in one ``. + - Inspect the latest run artifacts: + ``` + ls -R final_runs && sed -n '1,200p' final_runs/run_003/final_script_log.txt + ``` + - Inspect generated showcase artifacts: + ``` + find task_showcase -maxdepth 4 -type f -print + ``` + - Validate JSON syntax: + ``` + python -m json.tool task_showcase/tasks//task.json >/dev/null && python -m json.tool task_showcase/tasks//report.json >/dev/null + ``` + - Ask a grounded question about a saved screenshot: + ``` + python -m webwright.tools.image_qa --workspace-dir "{{ workspace_dir }}" --image screenshots/explore.png --question "Is the BMW filter chip visibly selected?" + ``` + - Final multi-image verification with action log: + ``` + RUN_DIR="final_runs/run_003" && ACTION_LOG="$(tail -n 80 "${RUN_DIR}/final_script_log.txt")" && python -m webwright.tools.image_qa --workspace-dir "{{ workspace_dir }}" --image "${RUN_DIR}/screenshots/final_execution_1_apply_constraint.png" --image "${RUN_DIR}/screenshots/final_execution_2_sort.png" --image "${RUN_DIR}/screenshots/final_execution_3_final_state.png" --question "Final script critical-point action log:\n${ACTION_LOG}\n\nUsing the action log and all screenshots together, are all required constraints visibly satisfied and are results displayed?" + ``` + + ## Rules + - **Always avoid taking full page screenshots using Playwright; use viewport 1280x1800** (exploration, debugging, and final-run screenshots alike). Never do `page.screenshot(full_page=True)`. + - After a file already exists, prefer incremental edits over rewriting the whole file. + - Use stable selectors and current-run evidence. + - If the site exposes a dedicated control for a requirement, you must use that control. Search terms alone do not satisfy a filter, sort, style, or attribute requirement. + - Ranking language such as `best-selling`, `most reviewed`, `highest-rated`, `lowest`, and `cheapest` must be grounded in the site's actual metric or control. + - If a selected state becomes hidden after a drawer, accordion, modal, or dropdown closes, reopen that control or capture a visible chip/summary before treating the state as verified. + - Treat numeric, date, quantity, and unit constraints as exact. Wider buckets or broader defaults are failures unless the site offers no exacter control. + - If the task asks for a final datum (code, price, quote, review, winner, benefit list), state that datum explicitly in `final_response` and in the Task Showcase report. + - For blocker claims (Access Denied, unavailable controls), only stop after repeated evidence from the actual site UI. + - Make sure to save as many critical points as possible, especially those that show the application of required filters or constraints and the final result display. The more evidence you save, the higher chance the judge will verify successful completion. + It also needs to save the final response of the task in `final_runs/run_/final_script_log.txt` in the end. + + ## Task Reflection Tool + + Do NOT hand-roll a `judge.py` that loops `image_qa`. Use the built-in + `webwright.tools.self_reflection` CLI. + + 1. Stage 1 โ€” score each screenshot against the full set of critical points with a + single (system, user+image) prompt pair. The tool parses a `Score: 1-5` and + `Reasoning: ` from each response and retries on parse failure. + 2. Stage 2 โ€” drop every per-image `Reasoning` into the final user prompt template + via `{image_reasonings}`, inject the latest run's `final_script_log.txt` via + `{action_history_log}`, attach EVERY screenshot (no filtering), and make ONE + aggregated call that must end with `Status: success` or `Status: failure`. + + Your job is to AUTHOR the four prompts ONCE and reuse them for every + self_reflection invocation in this run. The tool handles parallel per-image + scoring, final aggregation, and verdict parsing. + + **CLI interface:** + ``` + python -m webwright.tools.self_reflection \ + --config {{ workspace_dir }}/self_reflect_config.json \ + --workspace-dir "{{ workspace_dir }}" \ + --output {{ workspace_dir }}/final_runs/run_/self_reflect_result.json + ``` + + Exit code is 0 on PASS, 1 on FAIL / unparsed. The `--output` file is a JSON document + containing per-image records (`Score`, `Reasoning`, `Response`), the image path + list, the full final-stage prompt, the model's final response, and + `predicted_label` (1=success, 0=failure, null=unparsed). You MUST run self_reflection before + declaring done. + + **self_reflect_config.json schema (authored once, prompts only):** + ```json + { + "image_judge_system_prompt": "...see below...", + "image_judge_user_prompt": "...see below...", + "final_verdict_system_prompt": "...see below...", + "final_verdict_user_prompt": "...{action_history_log}...{image_reasonings}..." + } + ``` + Any `_file` variant (e.g. `image_judge_user_prompt_file`) may point to a + text file on disk instead of inlining the prompt โ€” useful when prompts contain + many curly braces or embedded JSON. + + **Required prompt content (you MUST include all of this in the JSON you write):** + + - `image_judge_system_prompt`: instruct the model to act as a harsh evaluator and + return ONLY two labelled lines: + ``` + Reasoning: <1-2 sentences describing what the screenshot shows and which critical + points it provides evidence for or against> + Score: + ``` + Do NOT ask for JSON. The tool parses the labelled lines directly. + + - `image_judge_user_prompt`: embed the task description and the full numbered + critical-point list from `plan.md`. Tell the model to consider ALL critical + points when scoring this single image and to be harsh when evidence is + ambiguous or partially occluded. + + - `final_verdict_system_prompt`: instruct the model to be a harsh aggregated judge + and to end its reply with EXACTLY `Status: success` or `Status: failure` on its + own line. Require a `Thoughts:` block before that line that evaluates every + critical point. The tool extracts the verdict from the trailing `Status:` line. + + - `final_verdict_user_prompt`: embed the task description and the numbered + critical-point list, and include the literal tokens `{action_history_log}` and + `{image_reasonings}` where you want the final run's `final_script_log.txt` + content and the per-image reasonings injected. Do NOT hard-code a specific + run's `final_script_log.txt`. The tool renders those tokens with Python + `str.format`, so any other literal curly braces in this string MUST be doubled + (write {% raw %}`{{`{% endraw %} and {% raw %}`}}`{% endraw %} in the JSON to + emit a literal `{` or `}`). + + **Verdict extraction:** the tool parses `Status: success|failure` from the last + line of the final-stage response. Missing or malformed `Status:` counts as FAIL + (exit code 1). Keep the verdict line clean. + + **Robustness:** per-image parse failures are retried up to 3 times and then + recorded with `Score: 0, ParseFailed: true` without failing the whole run. + Transient model-API HTTP errors are retried with exponential backoff. + + ## Final Deliverable: Task Showcase JSON + + In this mode, `final_script.py` is still the reusable task artifact, but its required output is a generic Task Showcase dataset: + + ``` + task_showcase/tasks// + task.json + report.json + ``` + + The existing renderer reads those two files and renders a dashboard/task page. Do not build a separate HTML app in `final_script.py`. + + Required `task.json` shape: + ``` + { + "task_id": "stable id, usually the CLI task_id if provided", + "short_id": "url-safe slug matching the directory name", + "title": "short human-readable title", + "theme": "short category label", + "cadence": "refresh cadence such as on demand, daily, every 6h", + "level": "easy|medium|hard or another short difficulty label", + "website": "primary site URL or starting URL", + "task_prompt": "original user task prompt", + "num_steps": 0 + } + ``` + + `task.json` field rules: + - `short_id` must exactly match the `task_showcase/tasks//` directory name. + - If the CLI `task_id` is absent, set `task_id` to the generated `short_id` or another stable id derived from the task prompt. + - `website` must be a non-empty absolute `http(s)` URL for the primary site or starting URL. + - All fields except `num_steps` must be non-empty strings. + - `num_steps` must be an integer count of the `step N action:` entries from the clean final-script run. Successful non-empty runs should not leave it at `0`. + + Required `report.json` shape: + ``` + { + "sources": [ + {"name": "source display name", "url": "https://...", "note": "optional short note"} + ], + "result": { + "headline": "short headline", + "sections": [ + {"type": "summary", "title": "...", "body": "..."}, + {"type": "table", "title": "...", "columns": ["..."], "rows": [["..."]]}, + {"type": "list", "title": "...", "entries": ["..."]}, + {"type": "kv", "title": "...", "entries": [["key", "value"]]}, + {"type": "cards", "title": "...", "entries": [ + {"title": "...", "subtitle": "...", "fields": [["key", "value"]], "url": "https://..."} + ]} + ] + } + } + ``` + + Schema rules: + - Every source used to answer the task must appear in `sources`. + - `sources` must be a list of objects with non-empty string `name`, absolute `http(s)` string `url`, and optional string `note`. + - `result.headline` must be a non-empty string. + - `result.sections` must be a non-empty list of valid section objects. Empty or partial outcomes still need at least one `summary` or `list` section explaining the result. + - Use only the section types above. Do not invent new section types. + - Every section must have string `type` and non-empty string `title`. + - `summary` sections require non-empty string `body`. + - `table` sections require `columns: list[str]` and `rows: list[list[str|number]]`; every row must have the same length as `columns`. + - `list` sections require `entries: list[str|number]`. + - `kv` sections require `entries: list[[str, str|number]]`. + - `cards` sections require `entries: list[object]`; every card requires non-empty string `title`, optional string `subtitle`, optional `fields: list[[str, str|number]]`, and optional absolute `http(s)` string `url`. + - All scalar display values in sections must be strings or numbers that serialize cleanly to JSON. Normalize booleans, nulls, dates, complex objects, and arrays into strings before writing display fields. Do not emit comments, trailing commas, NaN, Infinity, Python tuples, or non-JSON values. + - All source URLs and item URLs must be original links discovered during the clean run: visited, clicked, or extracted from the live source. Do not fabricate URLs, use screenshots as URLs, or use search-result/snippet URLs unless that page itself is the source. + - For every result item with its own page, include the URL in `cards.entries[].url` or in a dedicated `URL` table column. + - Render important final data in structured sections, not only in prose. + - Empty or partial results must be visible in `report.json` with a summary/list section explaining what was checked and what failed. + + ## Renderer Requirement + + The render step is separate from JSON generation. `final_script.py` must generate only the Task Showcase dataset, not a custom HTML app. Before declaring done, verify that the existing repo Task Showcase renderer can consume the generated `task.json` and `report.json`. The renderer may live outside the installed `webwright` package data, so locate the existing repo file rather than assuming it is importable as package data. If Flask or the renderer file is unavailable, do not install or generate it; validate the JSON files and report the exact renderer command or missing renderer path instead of claiming a completed render smoke test. + + The runtime gate checks `self_reflect_result.json`, so do not run the final self_reflection pass until after Task Showcase JSON validation and the renderer smoke test have passed (or Flask unavailability has been handled exactly as described). If you change `final_script.py` or regenerated showcase JSON after self_reflection, rerun the clean final script in a new `final_runs/run_/` folder and rerun self_reflection against that same run. + + ## Completion Gate + + Set `"done": true` ONLY if ALL of the following are true: + 1. `plan.md` exists and every critical point is enumerated as a checklist item, + with the intended Task Showcase report sections documented. + 2. `self_reflect_config.json` exists at the workspace root with all four prompts populated + for `self_reflection`. + 3. `final_script.py` was executed successfully from scratch from `{{ workspace_dir }}` + for a new `final_runs/run_/` artifact folder, producing `final_script_log.txt`, + all critical-point screenshots, and the Task Showcase JSON files. + 4. `task_showcase/tasks//task.json` exists and validates. + 5. `task_showcase/tasks//report.json` exists and validates. + 6. The Flask renderer was smoke-tested against `{{ workspace_dir }}/task_showcase/tasks`, + or Flask was unavailable and the exact render command was reported. + 7. `python -m webwright.tools.self_reflection --config self_reflect_config.json + --workspace-dir "{{ workspace_dir }}" --output final_runs/run_/self_reflect_result.json` + was executed after JSON/render validation, against that same run, exited 0, and wrote + `final_runs/run_/self_reflect_result.json` with `"predicted_label": 1`. + 8. You have run `ls -R final_runs/run_`, + `ls -R final_runs/run_/screenshots`, and + `cat final_runs/run_/final_script_log.txt` to confirm the artifacts and + logs are in place. + 9. The final response names the generated JSON paths and the render command. + + Do NOT declare done if `self_reflection` exits non-zero, if `predicted_label` is + not 1, if the run folder is missing, if required screenshots are missing, if the + script failed to run, if the checklist in `plan.md` is incomplete, if the JSON + files are missing or malformed, if they were written outside `{{ workspace_dir }}`, + or if the task page does not render when Flask is available. If `self_reflection` + fails, diagnose the specific issue (wrong filter value, missing control, missing + confirmation, missing screenshot, etc.), fix `final_script.py`, re-run it in a new + `final_runs/run_/` folder, and re-run `self_reflection` against the new run. + Do NOT edit `self_reflect_config.json` between attempts unless a prompt itself is + objectively wrong. + + instance_template: | + Task: {{ task }} + {% if task_id %}Task ID: {{ task_id }} + {% endif %}{% if start_url %}Start URL: {{ start_url }} + {% endif %}Workspace root: {{ workspace_dir }} + Task metadata JSON: {{ task_metadata_path }} + Required final script path: {{ final_script_path }} + Generated showcase tasks dir: {{ workspace_dir }}/task_showcase/tasks + + + # Task Instructions + + You're solving a user-specified web task through a stateless local terminal + workspace harness and converting the result into Task Showcase JSON for the generic renderer under `assets/task_showcase`. + + + This is an interactive process where you reason, execute exactly one bash command, inspect the result, and then produce your next command. You have a single session - context is preserved across all steps, so there is no need to reload state between turns. + + + ## Harness Rules + + - Work only inside `{{ workspace_dir }}`. + - Keep generated code, screenshots, logs, scratch files, notes, and showcase JSON **only** in `{{ workspace_dir }}`. + - The required final artifact is `{{ final_script_path }}`. + - Create `final_runs/run_/` folders for every clean execution of the final script. Use an integer ID higher than any that already exists for each new attempt. + - Keep the canonical final script at `{{ final_script_path }}`. For each clean run, copy that exact script into `final_runs/run_/final_script.py` and store that run's `final_script_log.txt` and final verification screenshots inside the same run folder. + - Do not create an empty higher-numbered `final_runs/run_/` folder. The latest numbered run must be the run that contains screenshots, logs, showcase JSON output, and the matching `self_reflect_result.json`. + - The required rendered-data artifacts are: + - `{{ workspace_dir }}/task_showcase/tasks//task.json` + - `{{ workspace_dir }}/task_showcase/tasks//report.json` + - Use `{{ task_id }}` as the preferred `` when it is present and already URL-safe; otherwise derive a lowercase slug from the task title. + - The browser mode is `{{ browser_mode }}`. Match generated scripts to that mode (Browserbase cloud session vs. local Playwright launch). + + ## Web Task Rules + + - Do not guess UI interactions. Use printed evidence from the current run. + - Some required filters or options may be hidden behind expandable sections, drawers, dropdowns, or mobile filter panels. Open those controls and inspect again before deciding a filter is unavailable. + - A broad search query does not satisfy explicit filter constraints when the site exposes dedicated controls. + - Save final verification screenshots inside the active `final_runs/run_/screenshots/` folder. + - Print concise ARIA snapshots, URLs, titles, visible labels, extracted records, and any extracted state needed for the next step. + + ## Task Success Criteria + + 1. Filtered results must be displayed correctly. Missing selection, missing confirmation, or no visible effect = failure. + 2. Specific filter conditions ("best," "highest," "cheapest," "latest," "lowest," etc.) must be applied using the filter/sort function. + 3. Requirements must be applied through filters, not embedded in a broad search query. + 4. Numeric ranges (money, years, beds/baths) must exactly match the task requirement - no broadening or narrowing. + 5. Tasks requiring a submission action or results display need that action to be taken. + 6. Empty results are OK if the correct action was performed. + 7. All explicit filters must use site controls when those controls exist. + 8. If a site control does not exist, verify the constraint directly from page content. + 9. The final answer data must be represented in `report.json` structured sections, not only in `final_response`. + + ## Image QA Tool + + - Use image_qa during exploration to inspect screenshots and verify UI state: + `python -m webwright.tools.image_qa --workspace-dir "{{ workspace_dir }}" --image screenshots/example.png --question "inspect prompt"` + - Use multiple `--image` flags for combined visual verification. + - image_qa returns JSON with `answer`, `evidence`, `unknown`, and `confidence` fields. + + ## Recommended Workflow + + 1. **Planning**: Parse the task into a list of critical points - every explicit constraint, filter, sort, selection, datum, or source requirement that must be satisfied. Write them to `plan.md` as a checklist and add the intended Task Showcase report sections: + ``` + # Critical Points + - [ ] CP1: + - [ ] CP2: + ... + + # Report Sections + -
: + - ... + ``` + Each critical point must be independently verifiable from a screenshot, log entry, extracted source data, or generated JSON field. + + 2. **Author self_reflect_config.json (once)**: Write `{{ workspace_dir }}/self_reflect_config.json` containing only the four prompts (`image_judge_system_prompt`, `image_judge_user_prompt`, `final_verdict_system_prompt`, `final_verdict_user_prompt`) for `webwright.tools.self_reflection`. Embed the full critical-point list from `plan.md` and the task description into the user prompts, but keep the prompts generic - this file is reused verbatim for every `self_reflection` invocation, so do NOT hard-code a specific run id, screenshot filename, or `final_script_log.txt` content. + + 3. **Exploration**: Inspect `{{ task_metadata_path }}`, create exploration scripts, identify every required filter control, source URL, and output field. Use `image_qa` during exploration to verify UI state when visual evidence matters. + + 4. **Final script**: Write `final_script.py` at `{{ final_script_path }}`, then execute it from `{{ workspace_dir }}` for a new `final_runs/run_<id>/` artifact folder. The script must copy itself into that run folder, produce screenshots and action logs as described in **Final Script Instrumentation**, collect or refresh the task data, write the two JSON files under `{{ workspace_dir }}/task_showcase/tasks/<short_id>/`, and print the generated paths. The script should create parent directories, overwrite stale JSON for the same short_id, and fail loudly on malformed data. + + 5. **Validate JSON**: Confirm the clean run wrote: + - `{{ workspace_dir }}/task_showcase/tasks/<short_id>/task.json` + - `{{ workspace_dir }}/task_showcase/tasks/<short_id>/report.json` + Then validate both JSON files with Python: + - `task.json.short_id` exactly matches the `<short_id>` directory name. + - `task.json` has all required metadata keys, with non-empty strings except integer `num_steps`. + - `task.json.website` is an absolute `http(s)` URL. + - `report.json` has `sources`, non-empty `result.headline`, and non-empty `result.sections`. + - `sources` is a list of `{name, url, note?}` objects with absolute `http(s)` URLs. + - Every section type is one of `summary`, `table`, `list`, `kv`, `cards`. + - Every section satisfies the per-type contract from the system prompt, including table row widths and kv/card field pairs. + - Every source and item URL that appears in the result is an original discovered absolute `http(s)` link. + + 6. **Render smoke test**: + - Locate the existing repo renderer `assets/task_showcase/app.py`. Prefer this resolver: + ``` + python - <<'PY' + from pathlib import Path + import webwright + starts = [Path.cwd().resolve(), Path(webwright.__file__).resolve()] + candidates = [] + for start in starts: + candidates.extend(parent / "assets" / "task_showcase" / "app.py" for parent in [start, *start.parents]) + for candidate in candidates: + if candidate.exists(): + print(candidate) + break + else: + raise SystemExit("assets/task_showcase/app.py not found from cwd or webwright package path") + PY + ``` + - Start `python <app.py> --tasks-dir "{{ workspace_dir }}/task_showcase/tasks" --host 127.0.0.1 --port <free_port>`. + - Fetch `/` and `/task/<short_id>` and confirm both return HTTP 200 and the task headline appears in the task page HTML. + - Stop the server before declaring done. + - If Flask or the renderer file is unavailable, do not install or generate it. Validate JSON and include the exact render command or missing renderer path in `final_response`. + - If JSON validation or renderer smoke testing fails, fix `final_script.py`, rerun it in a new `final_runs/run_<id+1>/` folder, and repeat JSON validation and render smoke testing before self_reflection. + + 7. **Run self_reflection**: Execute `python -m webwright.tools.self_reflection --config self_reflect_config.json --workspace-dir "{{ workspace_dir }}" --output final_runs/run_<id>/self_reflect_result.json` only after JSON validation and renderer smoke testing have passed for the same run. The tool auto-attaches screenshots from `final_runs/run_<id>/screenshots` when that is the latest numbered run with screenshots - you do NOT pass an image list. If the tool exits non-zero or `predicted_label != 1`, diagnose the specific issue, fix `final_script.py`, re-run it in a new `final_runs/run_<id+1>/` folder, and re-invoke JSON validation, renderer smoke testing, and self_reflection against the new run. Do NOT edit `self_reflect_config.json` between attempts. + + 8. **Declare done**: Set `"done": true` ONLY after both Task Showcase JSON files validate, the renderer was smoke-tested or Flask unavailability was handled exactly as described above, `self_reflection` exits 0, and `self_reflect_result.json` reports `"predicted_label": 1` for the latest run. The external judge reads that same `self_reflect_result.json` as the final verdict. Declaring done in any other state is a failure. + + ## Final Script Instrumentation + + `final_script.py` must: + - be stored at `{{ final_script_path }}` and copied as `final_runs/run_<id>/final_script.py` for the clean run + - collect or refresh the task data from the relevant source(s) + - save critical-point screenshots as `final_runs/run_<id>/screenshots/final_execution_<step_number>_<action>.png` + - create or reset `final_runs/run_<id>/final_script_log.txt` at the start of each clean run + - write `step <step_number> action: <reason and action description>` to the log for every constraint-relevant interaction + - write `{{ workspace_dir }}/task_showcase/tasks/<short_id>/task.json` + - write `{{ workspace_dir }}/task_showcase/tasks/<short_id>/report.json` + - optionally copy `final_script_log.txt`, `steps.jsonl`, and screenshots into `{{ workspace_dir }}/task_showcase/tasks/<short_id>/` if you want the renderer to display run traces; these copies are not a substitute for the required `final_runs/run_<id>/` artifacts + - print the generated JSON paths + - include enough logging or stdout details to diagnose source access, empty results, and render validation failures + - each screenshot should correspond to a critical point from `plan.md` so that `self_reflection` can verify it + - keep all generated files inside `{{ workspace_dir }}` + + This instrumentation is mandatory because both `self_reflection`, the external judge, and the Task Showcase renderer evaluate these artifacts. + + ## Completion Gate + + Set `"done": true` ONLY if ALL of the following are true: + 1. `plan.md` exists with all critical points identified and the intended report sections documented. + 2. `self_reflect_config.json` exists with all four prompts populated for `self_reflection`. + 3. `{{ final_script_path }}` was run from scratch from `{{ workspace_dir }}` for a new `final_runs/run_<id>/` folder, producing the run-local script copy, `final_script_log.txt`, all critical-point screenshots, and the Task Showcase JSON files. + 4. `task_showcase/tasks/<short_id>/task.json` exists and validates. + 5. `task_showcase/tasks/<short_id>/report.json` exists and validates. + 6. The Flask renderer was smoke-tested against `{{ workspace_dir }}/task_showcase/tasks`, or Flask was unavailable and the exact render command was reported. + 7. `python -m webwright.tools.self_reflection --config self_reflect_config.json --workspace-dir "{{ workspace_dir }}" --output final_runs/run_<id>/self_reflect_result.json` was executed after JSON/render validation against that same run, exited 0, and wrote `final_runs/run_<id>/self_reflect_result.json` with `"predicted_label": 1`. + 8. `ls -R final_runs/run_<id>`, `ls -R final_runs/run_<id>/screenshots`, and `cat final_runs/run_<id>/final_script_log.txt` confirm the expected artifacts. + 9. `final_response` names the generated JSON paths and the render command. + + Do NOT declare done if `self_reflection` exits non-zero, if `predicted_label` is not 1, if the run folder is missing, if required screenshots are missing, if `self_reflection` has not been run against the latest `final_runs/run_<id>/`, if the JSON files are missing or malformed, if they were written outside `{{ workspace_dir }}`, or if the task page does not render when Flask is available. + </instructions> diff --git a/src/webwright/environments/__init__.py b/src/webwright/environments/__init__.py new file mode 100644 index 0000000..3258077 --- /dev/null +++ b/src/webwright/environments/__init__.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +import copy +import importlib + +from webwright import Environment + +_ENVIRONMENT_MAPPING = { + "local_browser": "webwright.environments.local_browser.LocalBrowserEnvironment", + "local_workspace": "webwright.environments.local_workspace.LocalWorkspaceEnvironment", +} + + +def get_environment_class(spec: str) -> type[Environment]: + full_path = _ENVIRONMENT_MAPPING.get(spec, spec) + module_name, class_name = full_path.rsplit(".", 1) + module = importlib.import_module(module_name) + return getattr(module, class_name) + + +def get_environment(config: dict, *, default_type: str = "local_workspace") -> Environment: + copied = copy.deepcopy(config) + environment_class = copied.pop("environment_class", default_type) + return get_environment_class(environment_class)(**copied) diff --git a/src/webwright/environments/local_browser.py b/src/webwright/environments/local_browser.py new file mode 100644 index 0000000..754055e --- /dev/null +++ b/src/webwright/environments/local_browser.py @@ -0,0 +1,567 @@ +from __future__ import annotations + +import asyncio +import io +import json +import os +import shutil +import subprocess +import sys +import textwrap +import time +import traceback +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +from typing import Any +from urllib.parse import quote, urlparse +from urllib.request import ProxyHandler, Request, build_opener + +from pydantic import BaseModel, Field, field_validator + +_BROWSER_MODES = {"local_cdp", "local_launch", "local_persistent"} +_DEFAULT_LOCAL_CDP_URL = "http://127.0.0.1:9222" +_DEFAULT_LOCAL_CDP_USER_DATA_DIR = Path("~/.cache/webwright/edge-profile") +_CHROMIUM_EXECUTABLE_CANDIDATES = ( + "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + "~/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "~/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "google-chrome", + "google-chrome-stable", + "chromium-browser", + "chromium", + "chrome", +) +_LOCAL_CDP_OPENER = build_opener(ProxyHandler({})) + + +def _urlopen_local_cdp(url_or_request: str | Request, *, timeout: float): + return _LOCAL_CDP_OPENER.open(url_or_request, timeout=timeout) + + +def _local_cdp_origin(cdp_url: str) -> str: + parsed = urlparse(cdp_url or _DEFAULT_LOCAL_CDP_URL) + scheme = parsed.scheme or "http" + netloc = parsed.netloc or parsed.path + if not netloc: + netloc = urlparse(_DEFAULT_LOCAL_CDP_URL).netloc + return f"{scheme}://{netloc}" + + +def _local_cdp_port(cdp_url: str) -> int: + parsed = urlparse(cdp_url or _DEFAULT_LOCAL_CDP_URL) + if parsed.port is not None: + return parsed.port + return 443 if parsed.scheme == "https" else 80 + + +def _is_local_cdp_available(cdp_url: str, *, timeout_seconds: float = 0.5) -> bool: + try: + with _urlopen_local_cdp( + f"{_local_cdp_origin(cdp_url).rstrip('/')}/json/version", + timeout=timeout_seconds, + ) as response: + return 200 <= response.status < 300 + except Exception: + return False + + +def _local_cdp_json_url(cdp_url: str, path: str) -> str: + return f"{_local_cdp_origin(cdp_url).rstrip('/')}{path}" + + +def _local_cdp_page_targets(cdp_url: str, *, timeout_seconds: float = 0.5) -> list[dict[str, Any]]: + try: + with _urlopen_local_cdp( + _local_cdp_json_url(cdp_url, "/json/list"), + timeout=timeout_seconds, + ) as response: + if not 200 <= response.status < 300: + return [] + payload = json.loads(response.read().decode("utf-8")) + except Exception: + return [] + if not isinstance(payload, list): + return [] + return [ + target + for target in payload + if isinstance(target, dict) and target.get("type") == "page" + ] + + +def _ensure_local_cdp_page_target(cdp_url: str, *, timeout_seconds: float = 1.0) -> None: + if _local_cdp_page_targets(cdp_url, timeout_seconds=timeout_seconds): + return + + target_url = f"{_local_cdp_json_url(cdp_url, '/json/new')}?{quote('about:blank', safe='')}" + request = Request(target_url, method="PUT") + with _urlopen_local_cdp(request, timeout=timeout_seconds) as response: + if not 200 <= response.status < 300: + raise RuntimeError(f"Could not create a local CDP page target: {cdp_url}") + + +def _resolve_local_cdp_url(configured_url: str, *, explicit: bool) -> str: + if explicit and configured_url: + return configured_url + return ( + os.environ.get("LOCAL_BROWSER_CDP_URL") + or os.environ.get("BROWSER_CDP_URL") + or configured_url + or _DEFAULT_LOCAL_CDP_URL + ) + + +def _resolve_user_data_dir(configured_dir: Path, *, explicit: bool) -> Path: + if explicit: + return configured_dir + env_dir = os.environ.get("LOCAL_BROWSER_USER_DATA_DIR") or os.environ.get("BROWSER_USER_DATA_DIR") + return Path(env_dir).expanduser() if env_dir else _DEFAULT_LOCAL_CDP_USER_DATA_DIR.expanduser() + + +def _find_chromium_executable(explicit_path: str = "") -> str: + candidates = [] + if explicit_path: + candidates.append(explicit_path) + candidates.extend( + value + for value in (os.environ.get("LOCAL_BROWSER_EXECUTABLE"), os.environ.get("BROWSER_EXECUTABLE")) + if value + ) + candidates.extend(_CHROMIUM_EXECUTABLE_CANDIDATES) + + for candidate in candidates: + expanded = os.path.expanduser(candidate) + if Path(expanded).exists(): + return expanded + resolved = shutil.which(expanded) + if resolved: + return resolved + raise FileNotFoundError( + "Could not find Chrome/Chromium. Set local_cdp_executable or LOCAL_BROWSER_EXECUTABLE." + ) + + +def _macos_open_app_name(executable: str) -> str: + if sys.platform != "darwin": + return "" + if "/Microsoft Edge.app/" in executable: + return "Microsoft Edge" + if "/Google Chrome.app/" in executable: + return "Google Chrome" + return "" + + +class LocalBrowserEnvironmentConfig(BaseModel): + start_url: str | None = None + browser_mode: str = "local_launch" + headless: bool = False + devtools: bool = False + keep_open_on_exit: bool = False + prompt_before_close: bool = False + slow_mo_ms: int = 50 + browser_width: int = 1280 + browser_height: int = 1440 + browser_timeout_ms: int = 10000 + browser_navigation_timeout_ms: int = 30000 + step_execution_timeout_ms: int = 20000 + observation_timeout_ms: int = 5000 + output_dir: Path = Path("outputs/default") + user_data_dir: Path = _DEFAULT_LOCAL_CDP_USER_DATA_DIR + launch_args: list[str] = Field(default_factory=list) + local_cdp_url: str = _DEFAULT_LOCAL_CDP_URL + local_cdp_new_page: bool = True + local_cdp_close_page_on_exit: bool = False + local_cdp_auto_start: bool = True + local_cdp_executable: str = "" + local_cdp_startup_timeout_seconds: float = 10 + local_cdp_close_started_browser_on_exit: bool = False + + @field_validator("browser_mode") + @classmethod + def validate_browser_mode(cls, value: str) -> str: + normalized = value.strip().lower().replace("-", "_") + if normalized not in _BROWSER_MODES: + raise ValueError( + f"browser_mode must be one of: {', '.join(sorted(_BROWSER_MODES))}" + ) + return normalized + + +class LocalBrowserEnvironment: + """Live local Playwright browser environment. + + The environment owns the browser/page and executes each model action as an async + Python snippet with ``page``, ``context``, ``browser``, ``playwright``, and + ``task`` already available. + """ + + def __init__(self, *, config_class: type = LocalBrowserEnvironmentConfig, **kwargs): + self.config = config_class(**kwargs) + fields_set = getattr(self.config, "model_fields_set", None) + if fields_set is None: + fields_set = getattr(self.config, "__fields_set__", set()) + self._config_fields_set = set(fields_set) + self.config.local_cdp_url = _resolve_local_cdp_url( + self.config.local_cdp_url, + explicit="local_cdp_url" in self._config_fields_set, + ) + self.config.output_dir = self.config.output_dir.expanduser() + self.config.user_data_dir = _resolve_user_data_dir( + self.config.user_data_dir, + explicit="user_data_dir" in self._config_fields_set, + ) + self._playwright = None + self._browser = None + self._context = None + self._page = None + self._local_cdp_page = None + self._loop: asyncio.AbstractEventLoop | None = None + self._local_cdp_process: subprocess.Popen | None = None + self._connected_over_cdp = False + self._step_index = 0 + self._prepared_task: dict[str, Any] = {} + self._console_history: list[str] = [] + self._step_console: list[str] = [] + self._step_python_code = "" + self._step_python_output = "" + + def _screenshots_dir(self) -> Path: + return self.config.output_dir / "screenshots" + + def _steps_dir(self) -> Path: + return self.config.output_dir / "steps" + + def prepare(self, **kwargs) -> None: + self._prepared_task = dict(kwargs) + self._step_index = 0 + self._console_history = [] + self._step_console = [] + start_url = kwargs.get("start_url") or self.config.start_url + if start_url: + self.config.start_url = str(start_url) + + self.config.output_dir.mkdir(parents=True, exist_ok=True) + self._steps_dir().mkdir(parents=True, exist_ok=True) + self._screenshots_dir().mkdir(parents=True, exist_ok=True) + (self.config.output_dir / "task.json").write_text( + json.dumps(kwargs, indent=2), + encoding="utf-8", + ) + self._run(self._prepare_async()) + + def _ensure_loop(self) -> asyncio.AbstractEventLoop: + if self._loop is None or self._loop.is_closed(): + self._loop = asyncio.new_event_loop() + return self._loop + + def _run(self, coro): + loop = self._ensure_loop() + return loop.run_until_complete(coro) + + def _ensure_local_cdp_browser(self) -> None: + if _is_local_cdp_available(self.config.local_cdp_url): + return + if not self.config.local_cdp_auto_start: + raise RuntimeError( + "Local CDP endpoint is not reachable. Start Chrome/Chromium with " + f"remote debugging enabled for {self.config.local_cdp_url}." + ) + + self.config.user_data_dir.mkdir(parents=True, exist_ok=True) + browser_args = list(self.config.launch_args) + executable = _find_chromium_executable(self.config.local_cdp_executable) + browser_flags = [ + "--remote-debugging-address=127.0.0.1", + f"--remote-debugging-port={_local_cdp_port(self.config.local_cdp_url)}", + f"--user-data-dir={self.config.user_data_dir}", + "--no-first-run", + "--no-default-browser-check", + *browser_args, + ] + app_name = _macos_open_app_name(executable) + launched_with_open = bool(app_name) + command = ( + ["open", "-na", app_name, "--args", *browser_flags] + if launched_with_open + else [executable, *browser_flags] + ) + self._local_cdp_process = subprocess.Popen( + command, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + + deadline = time.monotonic() + self.config.local_cdp_startup_timeout_seconds + while time.monotonic() < deadline: + if not launched_with_open and self._local_cdp_process.poll() is not None: + raise RuntimeError( + f"Chrome/Chromium exited before CDP became available: {self.config.local_cdp_url}" + ) + if _is_local_cdp_available(self.config.local_cdp_url): + return + time.sleep(0.2) + + self._local_cdp_process.terminate() + self._local_cdp_process = None + raise TimeoutError(f"Timed out waiting for local CDP endpoint: {self.config.local_cdp_url}") + + async def _prepare_async(self) -> None: + from playwright.async_api import async_playwright + + if self._page is not None and self._context is not None: + return + + self._playwright = await async_playwright().start() + chromium = self._playwright.chromium + launch_args = list(self.config.launch_args) + if self.config.devtools: + launch_args.append("--auto-open-devtools-for-tabs") + launch_kwargs = { + "headless": self.config.headless, + "slow_mo": self.config.slow_mo_ms, + "args": launch_args, + } + + if self.config.browser_mode == "local_cdp": + self._ensure_local_cdp_browser() + _ensure_local_cdp_page_target(self.config.local_cdp_url) + self._browser = await chromium.connect_over_cdp(self.config.local_cdp_url) + self._connected_over_cdp = True + self._context = ( + self._browser.contexts[0] + if self._browser.contexts + else await self._browser.new_context(no_viewport=True) + ) + if self.config.local_cdp_new_page or not self._context.pages: + self._page = await self._context.new_page() + self._local_cdp_page = self._page + else: + self._page = self._context.pages[0] + elif self.config.browser_mode == "local_persistent": + self.config.user_data_dir.mkdir(parents=True, exist_ok=True) + self._context = await chromium.launch_persistent_context( + user_data_dir=str(self.config.user_data_dir), + viewport={ + "width": self.config.browser_width, + "height": self.config.browser_height, + }, + **launch_kwargs, + ) + self._browser = self._context.browser + self._page = self._context.pages[0] if self._context.pages else await self._context.new_page() + else: + self._browser = await chromium.launch(**launch_kwargs) + self._context = await self._browser.new_context( + viewport={ + "width": self.config.browser_width, + "height": self.config.browser_height, + } + ) + self._page = await self._context.new_page() + + self._context.set_default_timeout(self.config.browser_timeout_ms) + self._context.set_default_navigation_timeout(self.config.browser_navigation_timeout_ms) + self._attach_page_listeners(self._page) + if self.config.start_url: + await self._page.goto(self.config.start_url, wait_until="domcontentloaded") + + def _attach_page_listeners(self, page: Any) -> None: + page.on("console", self._on_console_message) + page.on("pageerror", self._on_page_error) + + def _on_console_message(self, message: Any) -> None: + text = getattr(message, "text", "") + if callable(text): + text = text() + line = str(text) + self._console_history.append(line) + self._step_console.append(line) + + def _on_page_error(self, error: Any) -> None: + line = f"Page error: {error}" + self._console_history.append(line) + self._step_console.append(line) + + def execute(self, action: dict[str, Any], cwd: str = "") -> dict[str, Any]: + del cwd + return self._run(self._execute_async(action)) + + async def _execute_async(self, action: dict[str, Any]) -> dict[str, Any]: + self._step_index += 1 + self._step_console = [] + self._step_python_output = "" + self._step_python_code = str(action.get("python_code", "") or "") + self._persist_step_code(self._step_python_code) + + success = True + exception_text = "" + try: + if self._step_python_code.strip(): + await asyncio.wait_for( + self._run_python_code(self._step_python_code), + timeout=self.config.step_execution_timeout_ms / 1000, + ) + await self._wait_for_observation_ready() + except Exception: + success = False + exception_text = traceback.format_exc() + + observation = await self._capture_observation( + success=success, + exception_text=exception_text, + ) + return { + "output": self._step_python_output, + "returncode": 0 if success else 1, + "exception_info": exception_text, + "observation": observation, + } + + def _persist_step_code(self, python_code: str) -> None: + step_path = self._steps_dir() / f"step_{self._step_index:04d}.py" + step_path.write_text(python_code, encoding="utf-8") + + script_path = self.config.output_dir / "script.py" + with script_path.open("a", encoding="utf-8") as handle: + handle.write(f"\n\n# Step {self._step_index}\n") + handle.write(python_code) + handle.write("\n") + + async def _run_python_code(self, python_code: str) -> None: + if self._page is None or self._context is None or self._playwright is None: + raise RuntimeError("Browser environment was not prepared.") + + buffer = io.StringIO() + globals_dict: dict[str, Any] = {"asyncio": asyncio} + locals_dict: dict[str, Any] = {} + wrapped = "async def __agent_step__(page, context, browser, playwright, task):\n" + wrapped += textwrap.indent(python_code, " ") + with redirect_stdout(buffer), redirect_stderr(buffer): + exec(wrapped, globals_dict, locals_dict) + await locals_dict["__agent_step__"]( + self._page, + self._context, + self._browser, + self._playwright, + self._prepared_task, + ) + self._step_python_output = buffer.getvalue() + + async def _wait_for_observation_ready(self) -> None: + if self._page is None: + return + try: + await self._page.wait_for_load_state( + "domcontentloaded", + timeout=self.config.observation_timeout_ms, + ) + except Exception: + pass + + async def _capture_observation(self, *, success: bool, exception_text: str) -> dict[str, Any]: + page = self._page + url = "" + title = "" + aria_snapshot = "" + screenshot_path: Path | None = None + + if page is not None: + try: + url = page.url + except Exception: + url = self.config.start_url or "" + try: + title = await page.title() + except Exception: + title = "" + try: + aria_snapshot = await page.locator("body").aria_snapshot( + timeout=self.config.observation_timeout_ms, + ) + except Exception: + aria_snapshot = "" + try: + screenshot_path = self._screenshots_dir() / f"step_{self._step_index:04d}.png" + await page.screenshot(path=str(screenshot_path), full_page=False) + except Exception: + screenshot_path = None + + return { + "success": success, + "exception": exception_text, + "url": url or self.config.start_url or "", + "title": title, + "screenshot_path": str(screenshot_path) if screenshot_path is not None else "", + "aria_snapshot": aria_snapshot, + "python_code": self._step_python_code, + "python_output": self._step_python_output, + "console_output": "\n".join(self._step_console[-20:]), + "recent_console": "\n".join(self._console_history[-50:]), + } + + def get_template_vars(self, **kwargs) -> dict[str, Any]: + return { + "start_url": self.config.start_url or "", + "output_dir": str(self.config.output_dir.resolve()), + "browser_mode": self.config.browser_mode, + "user_data_dir": str(self.config.user_data_dir), + **kwargs, + } + + def serialize(self) -> dict[str, Any]: + return { + "environment": { + "config": self.config.model_dump(mode="json"), + "environment_type": f"{self.__class__.__module__}.{self.__class__.__name__}", + } + } + + def close(self) -> None: + if self.config.prompt_before_close: + input("Press Enter to close the browser...") + if self.config.keep_open_on_exit: + return + try: + self._run(self._close_async()) + finally: + if self._loop is not None and not self._loop.is_closed(): + self._loop.close() + self._loop = None + + async def _close_async(self) -> None: + + context = self._context + browser = self._browser + page = self._local_cdp_page + playwright = self._playwright + connected_over_cdp = self._connected_over_cdp + local_cdp_process = self._local_cdp_process + self._page = None + self._local_cdp_page = None + self._context = None + self._browser = None + self._playwright = None + self._connected_over_cdp = False + self._local_cdp_process = None + + try: + if connected_over_cdp: + if page is not None and self.config.local_cdp_close_page_on_exit: + try: + await page.close() + except Exception: + pass + if ( + local_cdp_process is not None + and self.config.local_cdp_close_started_browser_on_exit + ): + local_cdp_process.terminate() + elif context is not None: + await context.close() + elif browser is not None: + await browser.close() + finally: + if playwright is not None: + await playwright.stop() diff --git a/src/webwright/environments/local_workspace.py b/src/webwright/environments/local_workspace.py new file mode 100644 index 0000000..04d3055 --- /dev/null +++ b/src/webwright/environments/local_workspace.py @@ -0,0 +1,296 @@ +from __future__ import annotations + +import json +import os +import re +import subprocess +from pathlib import Path +from typing import Any + +from pydantic import BaseModel, Field + +_EXPORT_RE = re.compile(r"^export\s+([A-Za-z_][A-Za-z0-9_]*)=(.*)$") +_IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".webp"} + + +class LocalWorkspaceEnvironmentConfig(BaseModel): + """Shell-based workspace environment. + + The agent drives a real browser through bash commands it generates inside this + workspace. Two browser modes are exposed to those generated scripts via + environment variables: + + * ``browser_mode = "browserbase"`` (default): the agent's scripts should + create a Browserbase cloud session. ``BROWSERBASE_API_KEY`` and + ``BROWSERBASE_PROJECT_ID`` are forwarded if present. + * ``browser_mode = "local"``: the agent's scripts should launch a local + Playwright browser (``playwright.chromium.launch(...)``). + + The selected mode is forwarded to the subprocess via ``BROWSER_MODE`` so the + generated scripts can branch on it. + """ + + start_url: str | None = None + output_dir: Path = Path("outputs/sandbox/default") + command_timeout_seconds: int = 180 + shell: str = "/bin/bash" + env: dict[str, str] = Field(default_factory=dict) + credentials_file: Path | None = None + browser_mode: str = "browserbase" # "browserbase" or "local" + task_metadata_filename: str = "task.json" + final_script_name: str = "final_script.py" + output_truncation_chars: int = 12000 + final_script_preview_chars: int = 4000 + recent_files_limit: int = 40 + + +class LocalWorkspaceEnvironment: + def __init__(self, *, config_class: type = LocalWorkspaceEnvironmentConfig, **kwargs): + self.config = config_class(**kwargs) + self.config.output_dir = self.config.output_dir.expanduser() + self._step_index = 0 + self._prepared_task: dict[str, Any] = {} + self._credential_env = self._load_credential_env(self.config.credentials_file) + + def _load_credential_env(self, path: Path | None) -> dict[str, str]: + if path is None: + return {} + resolved = Path(path).expanduser() + if not resolved.exists(): + return {} + + parsed: dict[str, str] = {} + for raw_line in resolved.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + match = _EXPORT_RE.match(line) + if match is None: + continue + key, raw_value = match.groups() + value = raw_value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1] + parsed[key] = value + return parsed + + def _workspace_dir(self) -> Path: + return self.config.output_dir.resolve() + + def _resolve_cwd(self, cwd: str = "") -> Path: + workspace_dir = self._workspace_dir() + if not cwd: + return workspace_dir + candidate = Path(cwd) + if not candidate.is_absolute(): + candidate = workspace_dir / candidate + resolved = candidate.resolve() + try: + resolved.relative_to(workspace_dir) + except ValueError as exc: + raise ValueError(f"Command cwd must stay inside workspace: {resolved}") from exc + return resolved + + def _task_metadata_path(self) -> Path: + return self._workspace_dir() / self.config.task_metadata_filename + + def _final_script_path(self) -> Path: + return self._workspace_dir() / self.config.final_script_name + + def _steps_dir(self) -> Path: + return self._workspace_dir() / "steps" + + def _logs_dir(self) -> Path: + return self._workspace_dir() / "logs" + + def _screenshots_dir(self) -> Path: + return self._workspace_dir() / "screenshots" + + def _history_path(self) -> Path: + return self._workspace_dir() / "command_history.sh" + + def _truncate(self, text: str, limit: int) -> str: + if len(text) <= limit: + return text + omitted = len(text) - limit + return f"{text[:limit]}\n\n... [{omitted} characters omitted]" + + def _recent_workspace_files(self) -> list[str]: + workspace_dir = self._workspace_dir() + files: list[Path] = [path for path in workspace_dir.rglob("*") if path.is_file()] + files.sort(key=lambda path: path.stat().st_mtime, reverse=True) + recent = files[: self.config.recent_files_limit] + return [str(path.relative_to(workspace_dir)) for path in recent] + + def _recent_screenshots(self) -> list[Path]: + screenshots_dir = self._screenshots_dir() + if not screenshots_dir.exists(): + return [] + files = [path for path in screenshots_dir.rglob("*") if path.is_file() and path.suffix.lower() in _IMAGE_SUFFIXES] + files.sort(key=lambda path: path.stat().st_mtime, reverse=True) + return files + + def _persist_step_command(self, command: str) -> Path: + self._steps_dir().mkdir(parents=True, exist_ok=True) + step_path = self._steps_dir() / f"step_{self._step_index:04d}.sh" + step_path.write_text(command.rstrip() + "\n", encoding="utf-8") + + history_path = self._history_path() + with history_path.open("a", encoding="utf-8") as handle: + handle.write(f"# Step {self._step_index}\n") + handle.write(command.rstrip()) + handle.write("\n\n") + return step_path + + def _write_step_log(self, output: str) -> Path | None: + if not output: + return None + self._logs_dir().mkdir(parents=True, exist_ok=True) + log_path = self._logs_dir() / f"step_{self._step_index:04d}.log" + log_path.write_text(output, encoding="utf-8") + return log_path + + def prepare(self, **kwargs) -> None: + self._prepared_task = dict(kwargs) + self._step_index = 0 + start_url = kwargs.get("start_url") or self.config.start_url + if start_url: + self.config.start_url = str(start_url) + workspace_dir = self._workspace_dir() + workspace_dir.mkdir(parents=True, exist_ok=True) + self._steps_dir().mkdir(parents=True, exist_ok=True) + self._logs_dir().mkdir(parents=True, exist_ok=True) + self._screenshots_dir().mkdir(parents=True, exist_ok=True) + (workspace_dir / ".tmp").mkdir(parents=True, exist_ok=True) + self._task_metadata_path().write_text(json.dumps(kwargs, indent=2), encoding="utf-8") + + def _browser_env(self) -> dict[str, str]: + """Forward Browserbase / browser-mode hints to the subprocess.""" + env: dict[str, str] = {"BROWSER_MODE": str(self.config.browser_mode or "browserbase")} + for var in ("BROWSERBASE_API_KEY", "BROWSERBASE_PROJECT_ID"): + value = self._credential_env.get(var) or os.environ.get(var) + if value: + env[var] = value + return env + + def execute(self, action: dict[str, Any], cwd: str = "") -> dict[str, Any]: + self._step_index += 1 + command = str( + action.get("command") or action.get("bash_command") or action.get("python_code") or "" + ).strip() + self._persist_step_command(command) + resolved_cwd = self._resolve_cwd(cwd) + + command_env = os.environ | self._credential_env | self._browser_env() | self.config.env | { + "WORKSPACE_DIR": str(self._workspace_dir()), + "OM2W_TASK_JSON": str(self._task_metadata_path()), + "FINAL_SCRIPT_PATH": str(self._final_script_path()), + "TMPDIR": str(self._workspace_dir() / ".tmp"), + } + + try: + result = subprocess.run( + command, + shell=True, + executable=self.config.shell, + text=True, + cwd=resolved_cwd, + env=command_env, + timeout=self.config.command_timeout_seconds, + encoding="utf-8", + errors="replace", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + output = result.stdout + returncode = result.returncode + exception_info = "" + except Exception as exc: + raw_output = getattr(exc, "output", None) + output = raw_output.decode("utf-8", errors="replace") if isinstance(raw_output, bytes) else (raw_output or "") + returncode = -1 + exception_info = f"An error occurred while executing the command: {exc}" + + log_path = self._write_step_log(output) + observation = self._capture_observation( + command=command, + cwd=resolved_cwd, + output=output, + returncode=returncode, + exception_info=exception_info, + log_path=log_path, + ) + return { + "output": output, + "returncode": returncode, + "exception_info": exception_info, + "observation": observation, + } + + def _capture_observation( + self, + *, + command: str, + cwd: Path, + output: str, + returncode: int, + exception_info: str, + log_path: Path | None, + ) -> dict[str, Any]: + final_script_path = self._final_script_path() + recent_screenshot_paths = self._recent_screenshots() + latest_screenshot = recent_screenshot_paths[0] if recent_screenshot_paths else None + final_script_preview = "" + if final_script_path.exists(): + final_script_preview = self._truncate( + final_script_path.read_text(encoding="utf-8", errors="replace"), + self.config.final_script_preview_chars, + ) + + workspace_dir = self._workspace_dir() + recent_screenshots = [str(path.relative_to(workspace_dir)) for path in recent_screenshot_paths[:10]] + return { + "success": returncode == 0 and not exception_info, + "exception": exception_info, + "command": command, + "returncode": returncode, + "workspace_dir": str(workspace_dir), + "cwd": str(cwd), + "url": self.config.start_url or "", + "title": "", + "aria_snapshot": "", + "console_output": "", + "recent_console": "", + "command_output": self._truncate(output, self.config.output_truncation_chars), + "log_path": str(log_path) if log_path is not None else "", + "task_metadata_path": str(self._task_metadata_path()), + "final_script_path": str(final_script_path) if final_script_path.exists() else "", + "final_script_exists": final_script_path.exists(), + "final_script_preview": final_script_preview, + "screenshot_path": str(latest_screenshot) if latest_screenshot is not None else "", + "recent_screenshots": recent_screenshots, + "workspace_files": self._recent_workspace_files(), + } + + def get_template_vars(self, **kwargs) -> dict[str, Any]: + return { + "start_url": self.config.start_url or "", + "output_dir": str(self._workspace_dir()), + "workspace_dir": str(self._workspace_dir()), + "task_metadata_path": str(self._task_metadata_path()), + "final_script_path": str(self._final_script_path()), + "browser_mode": self.config.browser_mode, + **kwargs, + } + + def serialize(self) -> dict: + return { + "environment": { + "config": self.config.model_dump(mode="json"), + "environment_type": f"{self.__class__.__module__}.{self.__class__.__name__}", + "workspace_dir": str(self._workspace_dir()), + } + } + + def close(self) -> None: + return None diff --git a/src/webwright/exceptions.py b/src/webwright/exceptions.py new file mode 100644 index 0000000..51a894c --- /dev/null +++ b/src/webwright/exceptions.py @@ -0,0 +1,19 @@ +from __future__ import annotations + + +class InterruptAgentFlow(Exception): + def __init__(self, *messages: dict): + super().__init__() + self.messages = list(messages) + + +class LimitsExceeded(InterruptAgentFlow): + pass + + +class Submitted(InterruptAgentFlow): + pass + + +class FormatError(InterruptAgentFlow): + pass diff --git a/src/webwright/models/__init__.py b/src/webwright/models/__init__.py new file mode 100644 index 0000000..6631ac0 --- /dev/null +++ b/src/webwright/models/__init__.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import copy +import importlib + +from webwright import Model + +_MODEL_MAPPING = { + "openai": "webwright.models.openai_model.OpenAIModel", + "anthropic": "webwright.models.anthropic_model.AnthropicModel", + "openrouter": "webwright.models.openrouter_model.OpenRouterModel", +} + + +def get_model_class(spec: str) -> type[Model]: + full_path = _MODEL_MAPPING.get(spec, spec) + module_name, class_name = full_path.rsplit(".", 1) + module = importlib.import_module(module_name) + return getattr(module, class_name) + + +def get_model(config: dict, *, default_type: str = "openai") -> Model: + copied = copy.deepcopy(config) + model_class = copied.pop("model_class", default_type) + return get_model_class(model_class)(**copied) diff --git a/src/webwright/models/anthropic_model.py b/src/webwright/models/anthropic_model.py new file mode 100644 index 0000000..53393b5 --- /dev/null +++ b/src/webwright/models/anthropic_model.py @@ -0,0 +1,192 @@ +"""Anthropic (Claude) Messages API model backend.""" + +from __future__ import annotations + +import asyncio +import random +from typing import Any + +from webwright.models.base import ( + BaseModel, + BaseModelConfig, + OptStr, + _safe_int, +) + +# Anthropic-specific retry limits (independent of OpenAI defaults). The Claude +# Opus org-level ITPM cap can saturate for minutes at high concurrency, so we +# allow many more retries and longer backoffs. +MAX_RATE_LIMIT_RETRIES = 50 +MAX_TRANSIENT_GATEWAY_RETRIES = 20 +RATE_LIMIT_BACKOFF_MIN_SECONDS = 30.0 +RATE_LIMIT_BACKOFF_MAX_SECONDS = 60.0 +TRANSIENT_BACKOFF_BASE_SECONDS = 1.5 +TRANSIENT_BACKOFF_CAP_SECONDS = 60.0 + + +def _retry_after_seconds(exc: BaseException) -> float | None: + response = getattr(exc, "response", None) + if response is None: + return None + header = response.headers.get("retry-after") if getattr(response, "headers", None) else None + if not header: + return None + try: + return max(0.0, float(header)) + except (TypeError, ValueError): + return None + + +def _image_source_from_url(image_url: str) -> dict[str, Any]: + if image_url.startswith("data:"): + header, _, encoded = image_url.partition(",") + media_type = header.split(";")[0].removeprefix("data:") or "image/png" + return {"type": "base64", "media_type": media_type, "data": encoded} + return {"type": "url", "url": image_url} + + +def _serialize_anthropic_content_part(part: dict[str, Any]) -> dict[str, Any]: + if part.get("type") == "input_image": + return {"type": "image", "source": _image_source_from_url(part.get("image_url", ""))} + return {"type": "text", "text": part.get("text", "")} + + +def _serialize_anthropic_messages( + messages: list[dict[str, Any]], +) -> tuple[str | None, list[dict[str, Any]]]: + system_chunks: list[str] = [] + serialized: list[dict[str, Any]] = [] + for message in messages: + role = message["role"] + if role == "exit": + continue + content = message.get("content", "") + if role == "system": + if isinstance(content, str): + if content: + system_chunks.append(content) + else: + for part in content: + if isinstance(part, dict) and part.get("type") != "input_image": + text = part.get("text", "") + if text: + system_chunks.append(text) + continue + + if isinstance(content, str): + serialized.append({"role": role, "content": content}) + continue + parts = [_serialize_anthropic_content_part(p) for p in content if isinstance(p, dict)] + if parts and all(p.get("type") == "text" for p in parts): + serialized.append({"role": role, "content": "\n".join(p["text"] for p in parts)}) + else: + serialized.append({"role": role, "content": parts}) + + system_prompt = "\n\n".join(system_chunks) if system_chunks else None + return system_prompt, serialized + + +def _extract_anthropic_text(payload: dict[str, Any]) -> str: + texts: list[str] = [] + for block in payload.get("content") or []: + if isinstance(block, dict) and block.get("type") == "text": + text = block.get("text", "") + if text: + texts.append(text) + return "\n".join(texts) + + +def _usage_from_anthropic_payload(payload: dict[str, Any]) -> dict[str, int]: + usage = payload.get("usage") or {} + input_tokens = _safe_int(usage.get("input_tokens")) + output_tokens = _safe_int(usage.get("output_tokens")) + cached_input_tokens = _safe_int(usage.get("cache_read_input_tokens")) + return { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens, + "cached_input_tokens": cached_input_tokens, + "reasoning_output_tokens": 0, + } + + +def _metrics_input_from_anthropic( + system_prompt: str | None, anthropic_messages: list[dict[str, Any]] +) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + if system_prompt: + items.append({"content": [{"type": "input_text", "text": system_prompt}]}) + for msg in anthropic_messages: + content = msg.get("content", "") + if isinstance(content, str): + items.append({"content": [{"type": "input_text", "text": content}]}) + continue + parts: list[dict[str, Any]] = [] + for part in content: + if not isinstance(part, dict): + continue + if part.get("type") == "text": + parts.append({"type": "input_text", "text": part.get("text", "")}) + elif part.get("type") == "image": + parts.append({"type": "input_image"}) + items.append({"content": parts}) + return items + + +class AnthropicModelConfig(BaseModelConfig): + model_name: OptStr = "claude-opus-4-7" + anthropic_api_key: OptStr = "" + anthropic_endpoint: OptStr = "https://api.anthropic.com/v1/messages" + anthropic_version: OptStr = "2023-06-01" + max_output_tokens: int = 8000 + + +class AnthropicModel(BaseModel): + _API_KEY_FIELD = "anthropic_api_key" + _ENV_VAR = "ANTHROPIC_API_KEY" + _LOG_SOURCE = "anthropic" + _MAX_RATE_LIMIT_RETRIES = MAX_RATE_LIMIT_RETRIES + _MAX_TRANSIENT_RETRIES = MAX_TRANSIENT_GATEWAY_RETRIES + _DEFAULT_CONFIG_CLASS = AnthropicModelConfig + + def _request_headers(self) -> dict[str, str]: + return { + "Content-Type": "application/json", + "x-api-key": self.config.anthropic_api_key, + "anthropic-version": self.config.anthropic_version, + } + + def _post_url(self) -> str: + return self.config.anthropic_endpoint + + async def _rate_limit_backoff(self, attempt: int, exc: BaseException) -> None: + delay = random.uniform(RATE_LIMIT_BACKOFF_MIN_SECONDS, RATE_LIMIT_BACKOFF_MAX_SECONDS) + retry_after = _retry_after_seconds(exc) + if retry_after is not None and retry_after > delay: + delay = min(retry_after, RATE_LIMIT_BACKOFF_MAX_SECONDS * 2) + await asyncio.sleep(delay) + + async def _transient_backoff(self, attempt: int, exc: BaseException) -> None: + await asyncio.sleep( + min(TRANSIENT_BACKOFF_BASE_SECONDS * (2 ** attempt), TRANSIENT_BACKOFF_CAP_SECONDS) + ) + + def _build_payload(self, messages: list[dict[str, Any]]) -> dict[str, Any]: + system_prompt, anth_messages = _serialize_anthropic_messages(messages) + payload: dict[str, Any] = { + "model": self.config.model_name, + "messages": anth_messages, + "max_tokens": self.config.max_output_tokens, + } + if system_prompt: + payload["system"] = system_prompt + return payload + + def _request_metrics_input(self, payload: dict[str, Any]) -> list[dict[str, Any]]: + return _metrics_input_from_anthropic(payload.get("system"), payload.get("messages") or []) + + def _extract_text(self, payload: dict[str, Any]) -> str: + return _extract_anthropic_text(payload) + + def _usage_metrics_from_payload(self, payload: dict[str, Any]) -> dict[str, int]: + return _usage_from_anthropic_payload(payload) diff --git a/src/webwright/models/base.py b/src/webwright/models/base.py new file mode 100644 index 0000000..7728bb8 --- /dev/null +++ b/src/webwright/models/base.py @@ -0,0 +1,587 @@ +from __future__ import annotations + +import asyncio +import base64 +import json +import mimetypes +import os +import subprocess +from pathlib import Path +from typing import Annotated, Any + +import httpx +from jinja2 import StrictUndefined, Template +from pydantic import BaseModel as PydanticBaseModel, BeforeValidator, field_validator + +from webwright.exceptions import FormatError +from webwright.utils.logging import append_runtime_log +from webwright.utils.runtime import run_async + + +def _none_to_str(value: Any) -> str: + return "" if value is None else str(value) + + +# String field that coerces None -> "" and any value -> str via pydantic. +OptStr = Annotated[str, BeforeValidator(_none_to_str)] + +MAX_JSON_PARSE_RETRIES = 3 +DEFAULT_OBSERVATION_TEMPLATE = """Observation: +Status: {{ 'ok' if observation.success else 'error' }} +URL: {{ observation.url }} +Title: {{ observation.title }} +{% if observation.exception %}Exception: +{{ observation.exception }} +{% endif %}{% if observation.console_output %}Console output: +{{ observation.console_output }} +{% endif %}{% if observation.aria_snapshot %}ARIA snapshot: +{{ observation.aria_snapshot }} +{% endif %}{% if observation.screenshot_path %}Screenshot path: {{ observation.screenshot_path }} +{% endif %}""" +DEFAULT_FORMAT_ERROR_TEMPLATE = """Format error: + +{{ error }} + +Please respond with strict JSON using exactly these fields: +- thought: short reasoning about the next step +- bash_command: exactly one shell command for local-workspace tasks +- python_code: exactly one async Python browser step for local-browser tasks +- done: boolean indicating whether the task is complete +- final_response: final natural-language answer when done, otherwise empty +""" + +ACTION_FIELDS = {"bash_command", "python_code"} + + +def _is_rate_limit_error(exc: BaseException | None) -> bool: + current: BaseException | None = exc + while current is not None: + status_code = getattr(current, "status_code", None) + if status_code == 429: + return True + response = getattr(current, "response", None) + if getattr(response, "status_code", None) == 429: + return True + text = str(current).lower() + if "rate limit" in text or "ratelimit" in text or "too many requests" in text: + return True + current = current.__cause__ if isinstance(current.__cause__, BaseException) else None + return False + + +def _is_transient_http_error(exc: BaseException | None) -> bool: + """True for retryable transient HTTP failures (timeouts, 5xx, conn resets, ...). + + Applies to any HTTP backend, not just gateway/proxy setups. + """ + current: BaseException | None = exc + while current is not None: + if isinstance(current, (httpx.TimeoutException, httpx.NetworkError, httpx.RemoteProtocolError)): + return True + status_code = getattr(current, "status_code", None) + if isinstance(status_code, int) and status_code in {408, 409, 425, 500, 502, 503, 504}: + return True + response = getattr(current, "response", None) + response_status = getattr(response, "status_code", None) + if isinstance(response_status, int) and response_status in {408, 409, 425, 500, 502, 503, 504}: + return True + text = str(current).lower() + if any( + needle in text + for needle in ( + "bad gateway", + "gateway timeout", + "server disconnected", + "temporary failure", + "temporarily unavailable", + "connection reset", + "connection aborted", + "timed out", + ) + ): + return True + current = current.__cause__ if isinstance(current.__cause__, BaseException) else None + return False + + +def parse_json_output(raw: str, *, action_field: str = "bash_command") -> dict[str, Any]: + try: + parsed = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError(f"Unable to parse JSON output: {exc}") from exc + if not isinstance(parsed, dict): + raise ValueError("Model output was JSON but not a JSON object.") + # Strict-schema responses cannot have done=true with a non-empty action; + # tolerate it from non-strict callers by demoting `done`. + action_text = str(parsed.get(action_field, "") or "").strip() + if action_text and bool(parsed.get("done", False)): + parsed = dict(parsed) + parsed["done"] = False + return parsed + + +def _validate_bash_command(command: str) -> None: + result = subprocess.run( + ["/bin/bash", "-n"], + input=command, + text=True, + capture_output=True, + encoding="utf-8", + errors="replace", + check=False, + ) + if result.returncode == 0: + return + error = (result.stderr or result.stdout or "bash syntax check failed").strip() + raise ValueError(f"Invalid bash_command syntax: {error}") + + +def text_part(text: str) -> dict[str, Any]: + return {"type": "input_text", "text": text} + + +def image_part_from_path(path: Path) -> dict[str, Any]: + mime_type, _ = mimetypes.guess_type(str(path)) + encoded = base64.b64encode(path.read_bytes()).decode("ascii") + return { + "type": "input_image", + "image_url": f"data:{mime_type or 'image/png'};base64,{encoded}", + "detail": "high", + } + + +def _safe_int(value: Any) -> int: + try: + return int(value) + except (TypeError, ValueError): + return 0 + + +def _request_metrics_from_serialized_input(serialized_input: list[dict[str, Any]]) -> dict[str, int]: + message_count = len(serialized_input) + text_part_count = 0 + image_part_count = 0 + text_chars = 0 + + for item in serialized_input: + for content in item.get("content") or []: + if not isinstance(content, dict): + continue + part_type = content.get("type") + if part_type in {"input_text", "output_text"}: + text_part_count += 1 + text_chars += len(str(content.get("text", "") or "")) + elif part_type == "input_image": + image_part_count += 1 + + serialized_chars = len(json.dumps(serialized_input, ensure_ascii=False)) + + return { + "message_count": message_count, + "text_part_count": text_part_count, + "image_part_count": image_part_count, + "text_chars": text_chars, + "serialized_chars": serialized_chars, + } + + +_REQUEST_METRIC_KEYS = ( + "message_count", + "text_part_count", + "image_part_count", + "text_chars", + "serialized_chars", +) +_USAGE_METRIC_KEYS = ( + "input_tokens", + "output_tokens", + "total_tokens", + "cached_input_tokens", + "reasoning_output_tokens", +) + + +class BaseModelConfig(PydanticBaseModel): + """Fields common to every model backend (OpenAI, Anthropic, ...).""" + + model_name: OptStr = "" + max_output_tokens: int = 4000 + request_timeout_seconds: int = 120 + error_log_path: Path | None = None + observation_template: OptStr = DEFAULT_OBSERVATION_TEMPLATE + format_error_template: OptStr = DEFAULT_FORMAT_ERROR_TEMPLATE + attach_observation_screenshot: bool = True + action_field: str = "bash_command" + + @field_validator("action_field") + @classmethod + def validate_action_field(cls, value: str) -> str: + normalized = value.strip() + if normalized not in ACTION_FIELDS: + raise ValueError(f"action_field must be one of: {', '.join(sorted(ACTION_FIELDS))}") + return normalized + + +class BaseModel: + """Provider-agnostic model backend. + + Subclasses must override: + - class constants ``_API_KEY_FIELD``, ``_ENV_VAR``, ``_LOG_SOURCE``, + ``_DEFAULT_CONFIG_CLASS`` (and optionally ``_MAX_RATE_LIMIT_RETRIES``, + ``_MAX_TRANSIENT_RETRIES``) + - ``_request_headers``, ``_post_url`` + - ``_build_payload``, ``_request_metrics_input`` + - ``_extract_text``, ``_usage_metrics_from_payload`` + + Optionally override ``_rate_limit_backoff`` / ``_transient_backoff`` for + custom retry timing. + """ + + _API_KEY_FIELD: str = "" + _ENV_VAR: str = "" + _LOG_SOURCE: str = "" + _MAX_RATE_LIMIT_RETRIES: int = 5 + _MAX_TRANSIENT_RETRIES: int = 5 + _DEFAULT_CONFIG_CLASS: type = BaseModelConfig + + def __init__(self, *, config_class: type | None = None, **kwargs): + self.config = (config_class or self._DEFAULT_CONFIG_CLASS)(**kwargs) + self._last_request_metrics: dict[str, int] = {k: 0 for k in _REQUEST_METRIC_KEYS} + self._last_usage_metrics: dict[str, int] = {k: 0 for k in _USAGE_METRIC_KEYS} + self._cumulative_request_metrics: dict[str, int] = dict(self._last_request_metrics) + self._cumulative_usage_metrics: dict[str, int] = dict(self._last_usage_metrics) + + if self._API_KEY_FIELD: + if not getattr(self.config, self._API_KEY_FIELD, ""): + setattr(self.config, self._API_KEY_FIELD, os.environ.get(self._ENV_VAR, "")) + if not getattr(self.config, self._API_KEY_FIELD, ""): + raise RuntimeError(f"Missing {self._ENV_VAR}.") + + # ---- subclass extension points ------------------------------------------------ + + def _request_headers(self) -> dict[str, str]: + raise NotImplementedError + + def _post_url(self) -> str: + raise NotImplementedError + + def _build_payload(self, messages: list[dict[str, Any]]) -> dict[str, Any]: + raise NotImplementedError + + def _build_text_payload(self, messages: list[dict[str, Any]]) -> dict[str, Any]: + return self._build_payload(messages) + + def _request_metrics_input(self, payload: dict[str, Any]) -> list[dict[str, Any]]: + raise NotImplementedError + + def _extract_text(self, payload: dict[str, Any]) -> str: + raise NotImplementedError + + def _usage_metrics_from_payload(self, payload: dict[str, Any]) -> dict[str, int]: + raise NotImplementedError + + async def _rate_limit_backoff(self, attempt: int, exc: BaseException) -> None: + await asyncio.sleep(min(5 * (attempt + 1), 30)) + + async def _transient_backoff(self, attempt: int, exc: BaseException) -> None: + await asyncio.sleep(min(2 * (attempt + 1), 10)) + + # ---- shared infrastructure ---------------------------------------------------- + + def get_template_vars(self, **kwargs) -> dict[str, Any]: + vars: dict[str, Any] = { + "action_field": self.config.action_field, + "model_name": self.config.model_name, + } + for k, v in self._last_request_metrics.items(): + vars[f"last_request_{k}"] = v + for k, v in self._last_usage_metrics.items(): + vars[f"last_request_{k}"] = v + for k, v in self._cumulative_request_metrics.items(): + vars[f"cumulative_request_{k}"] = v + for k, v in self._cumulative_usage_metrics.items(): + vars[f"cumulative_{k}"] = v + vars.update(kwargs) + return vars + + def _response_schema(self) -> dict[str, Any]: + action_field = self.config.action_field + return { + "type": "object", + "additionalProperties": False, + "properties": { + "thought": {"type": "string"}, + action_field: {"type": "string"}, + "done": {"type": "boolean"}, + "final_response": {"type": "string"}, + }, + "required": ["thought", action_field, "done", "final_response"], + } + + def _usage_snapshot(self) -> dict[str, dict[str, int]]: + return { + "last_request": { + "message_count": self._last_request_metrics["message_count"], + "text_part_count": self._last_request_metrics["text_part_count"], + "image_part_count": self._last_request_metrics["image_part_count"], + "input_tokens": self._last_usage_metrics["input_tokens"], + "cached_input_tokens": self._last_usage_metrics["cached_input_tokens"], + }, + "last_response": dict(self._last_usage_metrics), + "cumulative_request": { + "message_count": self._cumulative_request_metrics["message_count"], + "text_part_count": self._cumulative_request_metrics["text_part_count"], + "image_part_count": self._cumulative_request_metrics["image_part_count"], + "input_tokens": self._cumulative_usage_metrics["input_tokens"], + "cached_input_tokens": self._cumulative_usage_metrics["cached_input_tokens"], + }, + "cumulative_response": dict(self._cumulative_usage_metrics), + } + + def _log_gateway_error(self, *, event: str, attempt: int, error: BaseException) -> None: + response = getattr(error, "response", None) + response_text = "" + if response is not None: + try: + response_text = str(getattr(response, "text", "") or "") + except Exception: + response_text = "" + if len(response_text) > 4000: + response_text = response_text[:4000] + + append_runtime_log( + self.config.error_log_path, + source=self._LOG_SOURCE, + event=event, + model_name=self.config.model_name, + endpoint=self._post_url(), + attempt=attempt, + error_type=type(error).__name__, + error=str(error), + status_code=getattr(response, "status_code", None), + response_text=response_text, + ) + + def _raw_response_log_path(self) -> Path | None: + if self.config.error_log_path is None: + return None + return self.config.error_log_path.parent / "raw_responses.jsonl" + + def format_message(self, **kwargs) -> dict[str, Any]: + role = kwargs["role"] + content = kwargs.get("content", "") + extra = kwargs.get("extra", {}) + return {"role": role, "content": content, "extra": extra} + + def format_observation_messages( + self, + message: dict[str, Any], + outputs: list[dict[str, Any]], + template_vars: dict[str, Any] | None = None, + ) -> list[dict[str, Any]]: + observation_messages: list[dict[str, Any]] = [] + for output in outputs: + observation = output.get("observation", {}) + content = Template(self.config.observation_template, undefined=StrictUndefined).render( + output=output, + observation=observation, + **(template_vars or {}), + ) + + parts: list[dict[str, Any]] = [text_part(content)] + screenshot_path = observation.get("screenshot_path") + if self.config.attach_observation_screenshot and screenshot_path: + parts.append(image_part_from_path(Path(screenshot_path))) + + observation_messages.append( + self.format_message(role="user", content=parts, extra={"observation": observation}) + ) + return observation_messages + + def _format_error(self, *, raw_text: str, error: str) -> FormatError: + return FormatError( + self.format_message( + role="user", + content=Template(self.config.format_error_template, undefined=StrictUndefined).render( + error=error, + model_response=raw_text, + **self.get_template_vars(), + ), + extra={ + "interrupt_type": "FormatError", + "model_response": raw_text, + }, + ) + ) + + def _format_repair_message(self, *, raw_text: str, error: str) -> dict[str, Any]: + return self.format_message( + role="user", + content=Template(self.config.format_error_template, undefined=StrictUndefined).render( + error=error, + model_response=raw_text, + **self.get_template_vars(), + ), + extra={ + "interrupt_type": "FormatErrorRetry", + "model_response": raw_text, + }, + ) + + async def _post_with_retries(self, payload: dict[str, Any]) -> dict[str, Any]: + headers = self._request_headers() + url = self._post_url() + for attempt in range(max(self._MAX_RATE_LIMIT_RETRIES, self._MAX_TRANSIENT_RETRIES) + 1): + try: + async with httpx.AsyncClient(timeout=self.config.request_timeout_seconds) as client: + response = await client.post(url, headers=headers, json=payload) + response.raise_for_status() + return response.json() + except Exception as exc: + if _is_rate_limit_error(exc): + self._log_gateway_error( + event="rate_limit_error", attempt=attempt + 1, error=exc + ) + if attempt >= self._MAX_RATE_LIMIT_RETRIES: + raise + await self._rate_limit_backoff(attempt, exc) + continue + if _is_transient_http_error(exc): + self._log_gateway_error( + event="transient_http_error", attempt=attempt + 1, error=exc + ) + if attempt >= self._MAX_TRANSIENT_RETRIES: + raise + await self._transient_backoff(attempt, exc) + continue + self._log_gateway_error( + event="fatal_gateway_error", attempt=attempt + 1, error=exc + ) + raise + raise RuntimeError("Exceeded retry budget without exception or success.") + + async def _query_async(self, messages: list[dict[str, Any]]) -> dict[str, Any]: + last_error: ValueError | None = None + raw_text = "" + request_messages = list(messages) + for attempt_index in range(MAX_JSON_PARSE_RETRIES + 1): + payload = self._build_payload(request_messages) + request_metrics = _request_metrics_from_serialized_input(self._request_metrics_input(payload)) + self._last_request_metrics = dict(request_metrics) + for key, value in request_metrics.items(): + self._cumulative_request_metrics[key] += value + + response_payload = await self._post_with_retries(payload) + + usage_metrics = self._usage_metrics_from_payload(response_payload) + self._last_usage_metrics = dict(usage_metrics) + for key, value in usage_metrics.items(): + self._cumulative_usage_metrics[key] += value + + raw_text = self._extract_text(response_payload) + append_runtime_log( + self._raw_response_log_path(), + source="model", + event="raw_text", + attempt=attempt_index + 1, + raw_text=raw_text, + ) + try: + parsed = parse_json_output(raw_text, action_field=self.config.action_field) + break + except ValueError as exc: + last_error = exc + if attempt_index < MAX_JSON_PARSE_RETRIES: + request_messages.append( + self._format_repair_message(raw_text=raw_text, error=str(exc)) + ) + else: + raise self._format_error( + raw_text=raw_text, + error=str(last_error or ValueError("Unable to parse model output.")), + ) + + actions: list[dict[str, Any]] = [] + action_field = self.config.action_field + action_text = str(parsed.get(action_field, "") or "").strip() + if action_text: + action = {action_field: action_text, "command": action_text} + if action_field == "bash_command": + action["bash_command"] = action_text + try: + _validate_bash_command(action_text) + except ValueError as exc: + raise self._format_error(raw_text=raw_text, error=str(exc)) + else: + action["python_code"] = action_text + actions.append(action) + + return self.format_message( + role="assistant", + content=parsed.get("thought", ""), + extra={ + "actions": actions, + "done": bool(parsed.get("done", False)), + "final_response": parsed.get("final_response", ""), + "raw_response": parsed, + "usage": self._usage_snapshot(), + }, + ) + + async def _complete_text_async( + self, + messages: list[dict[str, Any]], + *, + max_output_tokens: int | None = None, + ) -> str: + original_max_output_tokens = self.config.max_output_tokens + if max_output_tokens is not None: + self.config.max_output_tokens = max_output_tokens + try: + payload = self._build_text_payload(messages) + request_metrics = _request_metrics_from_serialized_input(self._request_metrics_input(payload)) + self._last_request_metrics = dict(request_metrics) + for key, value in request_metrics.items(): + self._cumulative_request_metrics[key] += value + + response_payload = await self._post_with_retries(payload) + + usage_metrics = self._usage_metrics_from_payload(response_payload) + self._last_usage_metrics = dict(usage_metrics) + for key, value in usage_metrics.items(): + self._cumulative_usage_metrics[key] += value + + raw_text = self._extract_text(response_payload) + append_runtime_log( + self._raw_response_log_path(), + source="model", + event="raw_text", + raw_text=raw_text, + ) + return raw_text + finally: + self.config.max_output_tokens = original_max_output_tokens + + def __call__( + self, + messages: list[dict[str, Any]], + **kwargs: Any, + ) -> str: + return run_async(self._complete_text_async(messages, **kwargs)) + + def query(self, messages: list[dict[str, Any]], **kwargs) -> dict[str, Any]: + return run_async(self._query_async(messages)) + + def serialize(self) -> dict[str, Any]: + config_dump = self.config.model_dump(mode="json") + if self._API_KEY_FIELD: + config_dump[self._API_KEY_FIELD] = "<redacted>" + return { + "model": { + "config": config_dump, + "usage": { + **self._usage_snapshot(), + }, + "model_type": f"{self.__class__.__module__}.{self.__class__.__name__}", + } + } diff --git a/src/webwright/models/openai_model.py b/src/webwright/models/openai_model.py new file mode 100644 index 0000000..f11806e --- /dev/null +++ b/src/webwright/models/openai_model.py @@ -0,0 +1,157 @@ +"""OpenAI Responses API model backend.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from webwright.models.base import ( + BaseModel, + BaseModelConfig, + OptStr, + _safe_int, + image_part_from_path, + text_part, +) + +__all__ = [ + "OpenAIModel", + "OpenAIModelConfig", + "_extract_response_text", + "text_part", +] + + +def _serialize_response_content_part(part: dict[str, Any], *, role: str) -> dict[str, Any]: + if part.get("type") == "input_image": + return { + "type": "input_image", + "image_url": part.get("image_url", ""), + "detail": part.get("detail", "high"), + } + text = part.get("text", "") + if role == "assistant": + return {"type": "output_text", "text": text} + return {"type": "input_text", "text": text} + + +def _serialize_response_input(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: + serialized: list[dict[str, Any]] = [] + for message in messages: + role = message["role"] + if role == "exit": + continue + content = message.get("content", "") + if isinstance(content, str): + serialized_content = [text_part(content)] + else: + serialized_content = [part for part in content if isinstance(part, dict)] + # The Responses API accepts "developer" for system-style instructions. + mapped_role = "developer" if role == "system" else role + serialized.append( + { + "type": "message", + "role": mapped_role, + "content": [ + _serialize_response_content_part(part, role=mapped_role) + for part in serialized_content + ], + } + ) + return serialized + + +def _extract_response_text(payload: dict[str, Any]) -> str: + output_text = payload.get("output_text") + if isinstance(output_text, str) and output_text: + return output_text + + texts: list[str] = [] + for item in payload.get("output") or []: + if not isinstance(item, dict): + continue + if item.get("type") != "message": + continue + for content in item.get("content") or []: + if not isinstance(content, dict): + continue + if isinstance(content.get("text"), str): + texts.append(content["text"]) + elif isinstance(content.get("output_text"), str): + texts.append(content["output_text"]) + return "\n".join(texts) + + +def _usage_metrics_from_response_payload(payload: dict[str, Any]) -> dict[str, int]: + usage = payload.get("usage") + if not isinstance(usage, dict): + usage = {} + input_details = usage.get("input_tokens_details") + if not isinstance(input_details, dict): + input_details = {} + output_details = usage.get("output_tokens_details") + if not isinstance(output_details, dict): + output_details = {} + + return { + "input_tokens": _safe_int(usage.get("input_tokens")), + "output_tokens": _safe_int(usage.get("output_tokens")), + "total_tokens": _safe_int(usage.get("total_tokens")), + "cached_input_tokens": _safe_int(input_details.get("cached_tokens")), + "reasoning_output_tokens": _safe_int(output_details.get("reasoning_tokens")), + } + + +class OpenAIModelConfig(BaseModelConfig): + model_name: OptStr = "gpt-4o" + openai_api_key: OptStr = "" + openai_endpoint: OptStr = "https://api.openai.com/v1/responses" + + +class OpenAIModel(BaseModel): + _API_KEY_FIELD = "openai_api_key" + _ENV_VAR = "OPENAI_API_KEY" + _LOG_SOURCE = "openai" + _MAX_RATE_LIMIT_RETRIES = 5 + _MAX_TRANSIENT_RETRIES = 5 + _DEFAULT_CONFIG_CLASS = OpenAIModelConfig + + def _request_headers(self) -> dict[str, str]: + return { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.config.openai_api_key}", + } + + def _post_url(self) -> str: + return self.config.openai_endpoint + + def _build_payload(self, messages: list[dict[str, Any]]) -> dict[str, Any]: + return { + "model": self.config.model_name, + "input": _serialize_response_input(messages), + "max_output_tokens": self.config.max_output_tokens, + "text": { + "format": { + "type": "json_schema", + "name": "playwright_step", + "schema": self._response_schema(), + "strict": True, + } + }, + } + + def _build_text_payload(self, messages: list[dict[str, Any]]) -> dict[str, Any]: + return { + "model": self.config.model_name, + "input": _serialize_response_input(messages), + "max_output_tokens": self.config.max_output_tokens, + } + + def _request_metrics_input(self, payload: dict[str, Any]) -> list[dict[str, Any]]: + return payload.get("input") or [] + + def _extract_text(self, payload: dict[str, Any]) -> str: + return _extract_response_text(payload) + + def _usage_metrics_from_payload(self, payload: dict[str, Any]) -> dict[str, int]: + return _usage_metrics_from_response_payload(payload) diff --git a/src/webwright/models/openrouter_model.py b/src/webwright/models/openrouter_model.py new file mode 100644 index 0000000..95c300f --- /dev/null +++ b/src/webwright/models/openrouter_model.py @@ -0,0 +1,206 @@ +"""OpenRouter chat completions model backend.""" + +from __future__ import annotations + +from typing import Any +from urllib.parse import urlparse + +from webwright.models.base import ( + BaseModel, + BaseModelConfig, + OptStr, + _safe_int, +) + +__all__ = [ + "OpenRouterModel", + "OpenRouterModelConfig", +] + + +def _serialize_chat_content_part(part: dict[str, Any]) -> dict[str, Any] | None: + part_type = part.get("type") + if part_type in {"input_text", "output_text"}: + return {"type": "text", "text": str(part.get("text", "") or "")} + if part_type == "input_image": + return { + "type": "image_url", + "image_url": { + "url": str(part.get("image_url", "") or ""), + "detail": str(part.get("detail", "high") or "high"), + }, + } + return None + + +def _serialize_chat_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: + serialized: list[dict[str, Any]] = [] + for message in messages: + role = message["role"] + if role == "exit": + continue + mapped_role = "system" if role == "system" else ("assistant" if role == "assistant" else "user") + content = message.get("content", "") + if isinstance(content, str): + serialized.append({"role": mapped_role, "content": content}) + continue + parts = [ + serialized_part + for part in content + if isinstance(part, dict) + for serialized_part in [_serialize_chat_content_part(part)] + if serialized_part is not None + ] + if mapped_role == "assistant" or all(part.get("type") == "text" for part in parts): + serialized.append( + { + "role": mapped_role, + "content": "\n".join(str(part.get("text", "") or "") for part in parts), + } + ) + else: + serialized.append({"role": mapped_role, "content": parts}) + return serialized + + +def _metrics_input_from_chat_messages(chat_messages: list[dict[str, Any]]) -> list[dict[str, Any]]: + metrics_input: list[dict[str, Any]] = [] + for message in chat_messages: + content = message.get("content", "") + if isinstance(content, str): + metrics_input.append({"content": [{"type": "input_text", "text": content}]}) + continue + parts: list[dict[str, Any]] = [] + for part in content: + if not isinstance(part, dict): + continue + if part.get("type") == "text": + parts.append({"type": "input_text", "text": str(part.get("text", "") or "")}) + elif part.get("type") == "image_url": + parts.append({"type": "input_image"}) + metrics_input.append({"content": parts}) + return metrics_input + + +def _extract_chat_completions_text(payload: dict[str, Any]) -> str: + choices = payload.get("choices") + if not isinstance(choices, list) or not choices: + return "" + first_choice = choices[0] + if not isinstance(first_choice, dict): + return "" + message = first_choice.get("message", {}) + if not isinstance(message, dict): + return "" + content = message.get("content", "") + if isinstance(content, str): + return content + if isinstance(content, list): + return "\n".join( + str(part.get("text", "") or "") + for part in content + if isinstance(part, dict) and part.get("type") == "text" + ) + return "" + + +def _usage_metrics_from_chat_completions(payload: dict[str, Any]) -> dict[str, int]: + usage = payload.get("usage") + if not isinstance(usage, dict): + usage = {} + return { + "input_tokens": _safe_int(usage.get("prompt_tokens")), + "output_tokens": _safe_int(usage.get("completion_tokens")), + "total_tokens": _safe_int(usage.get("total_tokens")), + "cached_input_tokens": 0, + "reasoning_output_tokens": 0, + } + + +def _endpoint_host(endpoint: str) -> str: + return (urlparse(endpoint).hostname or "").lower() + + +def _is_openai_endpoint(endpoint: str) -> bool: + return _endpoint_host(endpoint) == "api.openai.com" + + +def _is_openrouter_endpoint(endpoint: str) -> bool: + return _endpoint_host(endpoint).endswith("openrouter.ai") + + +class OpenRouterModelConfig(BaseModelConfig): + model_name: OptStr = "openai/gpt-5.4" + openrouter_api_key: OptStr = "" + openrouter_endpoint: OptStr = "https://openrouter.ai/api/v1/chat/completions" + http_referer: OptStr = "" + app_title: OptStr = "" + provider_require_parameters: bool = True + + +class OpenRouterModel(BaseModel): + _API_KEY_FIELD = "openrouter_api_key" + _ENV_VAR = "OPENROUTER_API_KEY" + _LOG_SOURCE = "openrouter" + _MAX_RATE_LIMIT_RETRIES = 5 + _MAX_TRANSIENT_RETRIES = 5 + _DEFAULT_CONFIG_CLASS = OpenRouterModelConfig + + def _request_headers(self) -> dict[str, str]: + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.config.openrouter_api_key}", + } + if _is_openrouter_endpoint(self.config.openrouter_endpoint) and self.config.http_referer: + headers["HTTP-Referer"] = self.config.http_referer + if _is_openrouter_endpoint(self.config.openrouter_endpoint) and self.config.app_title: + headers["X-Title"] = self.config.app_title + return headers + + def _post_url(self) -> str: + return self.config.openrouter_endpoint + + def _build_payload(self, messages: list[dict[str, Any]]) -> dict[str, Any]: + payload: dict[str, Any] = { + "model": self.config.model_name, + "messages": _serialize_chat_messages(messages), + "stream": False, + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "playwright_step", + "strict": True, + "schema": self._response_schema(), + }, + }, + } + if _is_openai_endpoint(self.config.openrouter_endpoint) and self.config.model_name.startswith("gpt-5"): + payload["max_completion_tokens"] = self.config.max_output_tokens + else: + payload["max_tokens"] = self.config.max_output_tokens + if self.config.provider_require_parameters and _is_openrouter_endpoint(self.config.openrouter_endpoint): + payload["provider"] = {"require_parameters": True} + return payload + + def _build_text_payload(self, messages: list[dict[str, Any]]) -> dict[str, Any]: + payload: dict[str, Any] = { + "model": self.config.model_name, + "messages": _serialize_chat_messages(messages), + "stream": False, + } + if _is_openai_endpoint(self.config.openrouter_endpoint) and self.config.model_name.startswith("gpt-5"): + payload["max_completion_tokens"] = self.config.max_output_tokens + else: + payload["max_tokens"] = self.config.max_output_tokens + if self.config.provider_require_parameters and _is_openrouter_endpoint(self.config.openrouter_endpoint): + payload["provider"] = {"require_parameters": True} + return payload + + def _request_metrics_input(self, payload: dict[str, Any]) -> list[dict[str, Any]]: + return _metrics_input_from_chat_messages(payload.get("messages") or []) + + def _extract_text(self, payload: dict[str, Any]) -> str: + return _extract_chat_completions_text(payload) + + def _usage_metrics_from_payload(self, payload: dict[str, Any]) -> dict[str, int]: + return _usage_metrics_from_chat_completions(payload) diff --git a/src/webwright/run/__init__.py b/src/webwright/run/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/webwright/run/cli.py b/src/webwright/run/cli.py new file mode 100644 index 0000000..343ab70 --- /dev/null +++ b/src/webwright/run/cli.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +from datetime import datetime +from pathlib import Path +from typing import Any + +import typer +from rich.console import Console + +from webwright.agents import get_agent +from webwright.config import get_config_from_spec, snapshot_config_specs +from webwright.environments import get_environment +from webwright.models import get_model +from webwright.utils.serialize import UNSET, recursive_merge +from webwright.run.doctor import run_doctor + + +DEFAULT_CONFIGS = ["base.yaml", "model_openai.yaml"] + +app = typer.Typer(no_args_is_help=True) +console = Console(highlight=False) + + +def _timestamped_output_dir(base_dir: str | Path | None, task_id: str | None) -> Path: + base = Path(base_dir or "outputs").expanduser() + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + suffix = task_id or "adhoc" + return base / f"{suffix}_{stamp}" + + +def run_one( + *, + task: str | None = None, + task_id: str | None = None, + start_url: str | None = None, + config_spec: list[str] | None = None, + output_dir: Path | None = None, + resolved_output_dir: Path | None = None, + debug: bool = False, + snapshot_config: bool = True, +) -> Any: + config_spec = config_spec or DEFAULT_CONFIGS + configs = [get_config_from_spec(spec) for spec in config_spec] + config = recursive_merge(*configs) + + run_config = config.get("run", {}) + resolved_task_id = task_id or run_config.get("task_id") + resolved_task = task or run_config.get("task") + resolved_start_url = start_url or run_config.get("start_url") + + if not resolved_task: + raise ValueError("A task is required. Use --task.") + + resolved_output_dir = resolved_output_dir or _timestamped_output_dir( + output_dir or config.get("environment", {}).get("output_dir") or "outputs", + resolved_task_id, + ) + if snapshot_config: + snapshot_config_specs(config_spec, resolved_output_dir, merged_config=config) + + config = recursive_merge( + config, + { + "run": { + "task": resolved_task, + "task_id": resolved_task_id or UNSET, + "start_url": resolved_start_url or UNSET, + }, + "environment": { + "output_dir": str(resolved_output_dir), + "start_url": resolved_start_url or UNSET, + "headless": False if debug else UNSET, + "devtools": True if debug else UNSET, + "keep_open_on_exit": True if debug else UNSET, + "prompt_before_close": True if debug else UNSET, + "slow_mo_ms": 250 if debug else UNSET, + }, + "model": { + "error_log_path": str(resolved_output_dir / "runtime_errors.jsonl"), + }, + "agent": { + "output_path": str(resolved_output_dir / "trajectory.json"), + }, + }, + ) + + model = get_model(config.get("model", {})) + env = get_environment(config.get("environment", {})) + agent = get_agent(model, env, config.get("agent", {}), default_type="default") + + console.print(f"Running task in [bold green]{resolved_output_dir}[/bold green]") + run_exception: Exception | None = None + close_exception: Exception | None = None + result: dict[str, Any] = {} + try: + env.prepare( + task=resolved_task, + task_id=resolved_task_id, + start_url=resolved_start_url, + ) + result = agent.run( + resolved_task, + task_id=resolved_task_id or "", + start_url=resolved_start_url or "", + ) + except Exception as exc: + run_exception = exc + if getattr(agent, "messages", None): + result = dict(agent.messages[-1].get("extra", {})) + result.setdefault("exit_status", type(exc).__name__) + result.setdefault("submission", "") + result.setdefault("final_response", "") + result["run_exception"] = str(exc) + finally: + try: + env.close() + except Exception as exc: + close_exception = exc + result.setdefault("exit_status", type(exc).__name__) + result.setdefault("submission", "") + result.setdefault("final_response", "") + result.setdefault("run_exception", str(exc)) + result["close_exception"] = str(exc) + if run_exception is None: + run_exception = exc + result["_output_dir"] = str(resolved_output_dir) + if close_exception is not None: + result["_close_exception"] = str(close_exception) + console.print( + result.get("final_response") or result.get("submission") or "Task finished." + ) + if run_exception is not None: + raise run_exception + return result + + +@app.command() +def main( + task: str = typer.Option( + ..., "-t", "--task", help="Natural language task description." + ), + task_id: str | None = typer.Option( + None, "--task-id", help="Optional identifier used in the output directory name." + ), + start_url: str | None = typer.Option( + None, "--start-url", help="Optional starting URL for the task." + ), + config_spec: list[str] = typer.Option(DEFAULT_CONFIGS, "-c", "--config"), + output_dir: Path | None = typer.Option(None, "-o", "--output-dir"), + debug: bool = typer.Option( + False, + "--debug", + help="Launch headed local Playwright with devtools and keep it open for inspection.", + ), +) -> Any: + return run_one( + task=task, + task_id=task_id, + start_url=start_url, + config_spec=config_spec, + output_dir=output_dir, + debug=debug, + ) + + +@app.command() +def doctor(): + """ + Validate local Webwright setup. + """ + run_doctor() + + +if __name__ == "__main__": + app() diff --git a/src/webwright/run/doctor.py b/src/webwright/run/doctor.py new file mode 100644 index 0000000..d64d5ed --- /dev/null +++ b/src/webwright/run/doctor.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import os +import subprocess +import sys +from importlib.util import find_spec +from pathlib import Path +from rich.console import Console +from rich.table import Table + +console = Console() + + +def check_python(): + version = sys.version_info + + if version >= (3, 10): + return True, f"Python {version.major}.{version.minor}" + + return False, ("Python 3.10+ required\nFix: install Python 3.10 or newer") + + +def check_playwright(): + if find_spec("playwright") is not None: + return True, "playwright installed" + + return False, ("playwright not installed\nFix: pip install playwright") + + +def check_chromium(): + try: + result = subprocess.run( + ["playwright", "install", "--dry-run"], + capture_output=True, + text=True, + ) + + if result.returncode == 0: + return True, "chromium available" + + return False, ("chromium missing\nFix: playwright install chromium") + + except Exception as e: + return False, str(e) + + +def check_screenshot(): + try: + from playwright.sync_api import sync_playwright + + screenshot_path = Path("doctor_test.png") + + with sync_playwright() as p: + browser = p.chromium.launch(headless=True) + + page = browser.new_page() + + page.set_content("<h1>Webwright Doctor</h1>") + + page.screenshot(path=str(screenshot_path)) + + browser.close() + + if screenshot_path.exists(): + screenshot_path.unlink(missing_ok=True) + + return True, "screenshot capture working" + + return False, "screenshot file was not created" + + except Exception: + return False, ( + "unable to launch Chromium for screenshot validation\n" + "Fix: playwright install" + ) + + +def check_openai_key(): + if os.getenv("OPENAI_API_KEY"): + return True, "OPENAI_API_KEY found" + + return False, ( + "OPENAI_API_KEY missing\nFix: set the OPENAI_API_KEY environment variable" + ) + + +def check_plugin_manifests(): + claude = Path(".claude-plugin/plugin.json") + codex = Path(".codex-plugin/plugin.json") + + missing = [] + + if not claude.exists(): + missing.append("Claude") + + if not codex.exists(): + missing.append("Codex") + + if not missing: + return True, "plugin manifests found" + + return False, ( + f"missing plugin manifests: {', '.join(missing)}\n" + "Fix: configure Claude/Codex plugins" + ) + + +CHECKS = [ + ("Python", check_python), + ("Playwright", check_playwright), + ("Chromium", check_chromium), + ("Screenshot", check_screenshot), + ("OpenAI Key", check_openai_key), + ("Plugins", check_plugin_manifests), +] + + +def run_doctor(): + table = Table(title="Webwright Doctor") + + table.add_column("Check") + table.add_column("Status") + table.add_column("Details") + + passed = 0 + + for name, fn in CHECKS: + ok, message = fn() + + status = "PASS" if ok else "FAIL" + + table.add_row( + name, + status, + message, + ) + + if ok: + passed += 1 + + console.print(table) + + console.print(f"\n{passed}/{len(CHECKS)} checks passed") + + +if __name__ == "__main__": + run_doctor() diff --git a/src/webwright/run/run_command.sh b/src/webwright/run/run_command.sh new file mode 100644 index 0000000..a07bc10 --- /dev/null +++ b/src/webwright/run/run_command.sh @@ -0,0 +1,6 @@ +python -m webwright.run.cli \ + -c base.yaml -c model_openai.yaml \ + -t "Search for flights from SEA to JFK on 2026-08-15 to 2026-08-20" \ + --start-url https://www.google.com/flights \ + --task-id demo_openai \ + -o outputs/default \ No newline at end of file diff --git a/src/webwright/tools/__init__.py b/src/webwright/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/webwright/tools/_model_config.py b/src/webwright/tools/_model_config.py new file mode 100644 index 0000000..cb1777a --- /dev/null +++ b/src/webwright/tools/_model_config.py @@ -0,0 +1,77 @@ +"""Resolve the model client used by inner tools (image_qa, self_reflection). + +The CLI snapshots the fully merged run config to +``<workspace_dir>/config_snapshot/merged_config.yaml``; the tools read that file +(or an explicit ``--model-config`` override) and instantiate the same model the +agent uses. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import yaml + +from webwright.models import get_model + +DEFAULT_MERGED_CONFIG_RELPATH = Path("config_snapshot") / "merged_config.yaml" + + +def _load_structured_config(path: Path) -> dict[str, Any]: + text = path.read_text(encoding="utf-8") + if path.suffix.lower() in {".yaml", ".yml"}: + loaded = yaml.safe_load(text) + else: + loaded = json.loads(text) + if not isinstance(loaded, dict): + raise ValueError(f"Model config must be an object: {path}") + return loaded + + +def _extract_model_block(config: dict[str, Any]) -> dict[str, Any]: + model_block = config.get("model") + if not isinstance(model_block, dict): + raise ValueError( + "Model config is missing a top-level `model:` block; " + "stack a model_*.yaml (e.g. model_claude.yaml) or pass --model-config <path>." + ) + return model_block + + +def resolve_model_config_path(model_config_arg: str, *, workspace_dir: str) -> Path: + """Return the path to a config containing a top-level ``model:`` block. + + Resolution order: + 1. ``model_config_arg`` (absolute or relative to ``workspace_dir``). + 2. ``<workspace_dir>/config_snapshot/merged_config.yaml`` (written by the CLI). + """ + candidates: list[Path] = [] + if model_config_arg: + configured = Path(model_config_arg) + candidates.append(configured) + if workspace_dir and not configured.is_absolute(): + candidates.append(Path(workspace_dir) / configured) + if workspace_dir: + candidates.append(Path(workspace_dir) / DEFAULT_MERGED_CONFIG_RELPATH) + for candidate in candidates: + if candidate.exists(): + return candidate.resolve() + raise FileNotFoundError( + "No tool model config found. Pass --model-config <path> or run via the agent so " + f"<workspace-dir>/{DEFAULT_MERGED_CONFIG_RELPATH} is available." + ) + + +def load_tool_model( + *, + model_config_arg: str, + workspace_dir: str, + timeout_seconds: int, +) -> Any: + config_path = resolve_model_config_path(model_config_arg, workspace_dir=workspace_dir) + config = _load_structured_config(config_path) + model_block = dict(_extract_model_block(config)) + model_block["request_timeout_seconds"] = timeout_seconds + return get_model(model_block) diff --git a/src/webwright/tools/image_qa.py b/src/webwright/tools/image_qa.py new file mode 100644 index 0000000..6df0785 --- /dev/null +++ b/src/webwright/tools/image_qa.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import argparse +import base64 +import json +import mimetypes +from pathlib import Path +from typing import Any + +from webwright.models.base import text_part +from webwright.tools._model_config import load_tool_model + + +def _build_prompt(question: str) -> str: + return ( + "Answer the user's question using only visible evidence from the provided image or images. " + "If the answer is not visible, say so instead of guessing. Return only a JSON object with " + "string `answer`, string array `evidence`, boolean `unknown`, and number `confidence`.\n\n" + f"Question: {question.strip()}" + ) + + +def _high_detail_image_part_from_path(image_path: Path) -> dict[str, Any]: + mime_type, _ = mimetypes.guess_type(str(image_path)) + encoded = base64.b64encode(image_path.read_bytes()).decode("ascii") + return { + "type": "input_image", + "image_url": f"data:{mime_type or 'image/png'};base64,{encoded}", + "detail": "high", + } + + +def _resolve_image_path(image_path: str, workspace_dir: str = "") -> Path: + path = Path(image_path) + if not path.is_absolute(): + base_dir = Path(workspace_dir) if workspace_dir else Path.cwd() + path = base_dir / path + path = path.resolve() + if not path.exists(): + raise FileNotFoundError(f"Image path does not exist: {path}") + return path + + +def _normalize_image_paths( + *, + image_path: Path | None = None, + image_paths: list[Path] | tuple[Path, ...] | None = None, +) -> list[Path]: + normalized = list(image_paths or []) + if image_path is not None: + normalized.insert(0, image_path) + if not normalized: + raise ValueError("At least one image path is required.") + return normalized + + +def _parse_json_response(raw_text: str) -> dict[str, Any]: + try: + parsed = json.loads(raw_text) + except json.JSONDecodeError: + start = raw_text.find("{") + end = raw_text.rfind("}") + if start == -1 or end == -1 or end <= start: + raise + parsed = json.loads(raw_text[start : end + 1]) + if not isinstance(parsed, dict): + raise ValueError("image_qa model response must be a JSON object.") + return parsed + + +def run_image_qa( + *, + image_path: Path | None = None, + image_paths: list[Path] | tuple[Path, ...] | None = None, + question: str, + model_client: Any, +) -> dict[str, Any]: + resolved_image_paths = _normalize_image_paths(image_path=image_path, image_paths=image_paths) + raw_text = model_client( + [ + { + "role": "user", + "content": [text_part(_build_prompt(question))] + + [_high_detail_image_part_from_path(path) for path in resolved_image_paths], + } + ], + max_output_tokens=32000, + ).strip() + parsed = _parse_json_response(raw_text) + result = { + "image_paths": [str(path) for path in resolved_image_paths], + "question": question, + **parsed, + } + if len(resolved_image_paths) == 1: + result["image_path"] = str(resolved_image_paths[0]) + return result + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Ask a visual question about a local image and print JSON.") + parser.add_argument( + "--image", + required=True, + action="append", + help="Path to an image file. Repeat --image to include multiple images.", + ) + parser.add_argument("--question", required=True, help="Question to answer from the image.") + parser.add_argument("--workspace-dir", default="", help="Optional base directory for relative image paths.") + parser.add_argument( + "--model-config", + default="", + help=( + "Path to a JSON/YAML config containing a top-level `model:` block. " + "If omitted, reads <workspace-dir>/config_snapshot/merged_config.yaml." + ), + ) + parser.add_argument("--timeout-seconds", type=int, default=60, help="HTTP request timeout.") + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + image_paths = [_resolve_image_path(image_path, workspace_dir=args.workspace_dir) for image_path in args.image] + model_client = load_tool_model( + model_config_arg=args.model_config, + workspace_dir=args.workspace_dir, + timeout_seconds=args.timeout_seconds, + ) + result = run_image_qa( + image_paths=image_paths, + question=args.question, + model_client=model_client, + ) + print(json.dumps(result, ensure_ascii=True, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/webwright/tools/persistent_local_browser.py b/src/webwright/tools/persistent_local_browser.py new file mode 100644 index 0000000..b861d1c --- /dev/null +++ b/src/webwright/tools/persistent_local_browser.py @@ -0,0 +1,314 @@ +"""CLI for managing a long-lived, reusable local Chromium browser session. + +Local-browser counterpart to ``browserbase_session.py``. Launches a +detached headless Chromium subprocess via the Playwright-bundled binary +with ``--remote-debugging-port=0`` and a per-session ``--user-data-dir``, +parses the printed ``DevTools listening on ws://...`` URL, and persists +``{id, pid, connectUrl, userDataDir}`` to a JSON file on disk so any +later Python/bash step can attach via +``playwright.chromium.connect_over_cdp(connectUrl)`` and end with +``await browser.disconnect()`` (NEVER ``browser.close()``) to keep the +browser alive across steps. + +Subcommands: + + * ``create`` -> spawn detached Chromium, write JSON, print id. + * ``info`` -> print whether the saved PID is still alive plus + the persisted JSON. + * ``release`` -> SIGTERM (then SIGKILL) the PID, optionally remove + the user-data-dir and the JSON file. + +Usage: + python -m webwright.tools.persistent_local_browser create --workspace-dir <ws> --out .lb_session.json + python -m webwright.tools.persistent_local_browser info --workspace-dir <ws> --session-file .lb_session.json + python -m webwright.tools.persistent_local_browser release --workspace-dir <ws> --session-file .lb_session.json --delete-file +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import signal +import subprocess +import sys +import time +import uuid +from pathlib import Path + +DEFAULT_SESSION_FILE = ".lb_session.json" +DEFAULT_USER_DATA_SUBDIR = ".lb_user_data" +_DEVTOOLS_RE = re.compile(r"DevTools listening on (ws://\S+)") + + +def _resolve_path(path_str: str, workspace_dir: str = "") -> Path: + path = Path(path_str) + if not path.is_absolute(): + base = Path(workspace_dir) if workspace_dir else Path.cwd() + path = base / path + return path + + +def _chromium_executable() -> str: + """Locate the Playwright-bundled Chromium executable.""" + try: + from playwright.sync_api import sync_playwright + except ImportError as exc: # pragma: no cover - import guard + raise SystemExit(f"error: playwright is not installed: {exc}") + with sync_playwright() as p: + path = p.chromium.executable_path + if not path or not Path(path).exists(): + raise SystemExit( + "error: Playwright chromium binary not found. Run `playwright install chromium`." + ) + return path + + +def _pid_alive(pid: int) -> bool: + if pid <= 0: + return False + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def _wait_for_devtools_url(proc: subprocess.Popen, timeout: float) -> str: + """Read Chromium's stderr until the DevTools ws:// URL appears.""" + assert proc.stderr is not None + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if proc.poll() is not None: + tail = "" + try: + tail = proc.stderr.read() or "" + except Exception: # noqa: BLE001 + pass + raise SystemExit( + f"error: chromium exited early (code={proc.returncode}); stderr tail:\n{tail}" + ) + line = proc.stderr.readline() + if not line: + time.sleep(0.05) + continue + match = _DEVTOOLS_RE.search(line) + if match: + return match.group(1).strip() + raise SystemExit( + f"error: timed out after {timeout:.1f}s waiting for 'DevTools listening on ws://...' line" + ) + + +def _cmd_create(args: argparse.Namespace) -> int: + workspace_dir = args.workspace_dir or str(Path.cwd()) + out_path = _resolve_path(args.out, workspace_dir) + user_data_dir = _resolve_path( + args.user_data_dir or DEFAULT_USER_DATA_SUBDIR, workspace_dir + ) + out_path.parent.mkdir(parents=True, exist_ok=True) + user_data_dir.mkdir(parents=True, exist_ok=True) + + chromium = _chromium_executable() + chromium_args = [ + chromium, + "--remote-debugging-port=0", + f"--user-data-dir={user_data_dir}", + "--no-first-run", + "--no-default-browser-check", + "--disable-features=TranslateUI,MediaRouter", + f"--window-size={args.window_width},{args.window_height}", + ] + if args.headless: + chromium_args.append("--headless=new") + if args.no_sandbox: + chromium_args.append("--no-sandbox") + chromium_args.extend(args.chromium_arg or []) + + # Detach into its own process group so it survives the parent shell exit. + popen_kwargs: dict = { + "stdout": subprocess.DEVNULL, + "stderr": subprocess.PIPE, + "stdin": subprocess.DEVNULL, + "text": True, + "bufsize": 1, + "close_fds": True, + } + if os.name == "posix": + popen_kwargs["start_new_session"] = True + + proc = subprocess.Popen(chromium_args, **popen_kwargs) # noqa: S603 + try: + connect_url = _wait_for_devtools_url(proc, args.startup_timeout) + except SystemExit: + try: + proc.terminate() + except Exception: # noqa: BLE001 + pass + raise + + session = { + "id": uuid.uuid4().hex, + "pid": proc.pid, + "connectUrl": connect_url, + "userDataDir": str(user_data_dir), + "executablePath": chromium, + "headless": bool(args.headless), + "createdAt": int(time.time()), + } + out_path.write_text(json.dumps(session, indent=2) + "\n", encoding="utf-8") + + print(f"LB_SESSION_ID={session['id']}") + print(f"LB_SESSION_PID={session['pid']}") + print(f"LB_SESSION_FILE={out_path}") + print(f"LB_CONNECT_URL={connect_url}") + return 0 + + +def _cmd_info(args: argparse.Namespace) -> int: + session_path = _resolve_path(args.session_file, args.workspace_dir) + if not session_path.exists(): + print(f"LB_INFO_MISSING file={session_path}") + return 1 + session = json.loads(session_path.read_text(encoding="utf-8")) + session["alive"] = _pid_alive(int(session.get("pid", 0))) + print(json.dumps(session, indent=2)) + return 0 + + +def _terminate_pid(pid: int, kill_timeout: float) -> str: + if pid <= 0 or not _pid_alive(pid): + return "not_running" + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + return "already_gone" + deadline = time.monotonic() + kill_timeout + while time.monotonic() < deadline: + if not _pid_alive(pid): + return "terminated" + time.sleep(0.1) + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + return "already_gone" + time.sleep(0.2) + return "killed" if not _pid_alive(pid) else "still_alive" + + +def _cmd_release(args: argparse.Namespace) -> int: + session_path = _resolve_path(args.session_file, args.workspace_dir) + if not session_path.exists(): + print(f"LB_RELEASE_SKIPPED missing={session_path}") + return 0 + session = json.loads(session_path.read_text(encoding="utf-8")) + pid = int(session.get("pid", 0)) + status = _terminate_pid(pid, args.kill_timeout) + print(f"LB_RELEASE_REQUESTED pid={pid} status={status}") + + if args.delete_user_data: + udd = session.get("userDataDir", "") + if udd and Path(udd).exists(): + try: + shutil.rmtree(udd) + print(f"LB_USER_DATA_DELETED {udd}") + except OSError as exc: + print(f"LB_USER_DATA_DELETE_FAILED {udd} {exc}") + + if args.delete_file: + try: + session_path.unlink() + print(f"LB_SESSION_FILE_DELETED {session_path}") + except OSError as exc: + print(f"LB_SESSION_FILE_DELETE_FAILED {session_path} {exc}") + return 0 + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="python -m webwright.tools.persistent_local_browser", + description="Manage a keep-alive local Chromium session shared across bash steps.", + ) + parser.add_argument( + "--workspace-dir", + default="", + help="Resolve --out / --session-file / --user-data-dir relative to this directory.", + ) + + sub = parser.add_subparsers(dest="command", required=True) + + create = sub.add_parser("create", help="Launch a detached Chromium and persist its connectUrl.") + create.add_argument("--out", default=DEFAULT_SESSION_FILE, help="Where to write the session JSON.") + create.add_argument( + "--user-data-dir", + default="", + help=f"Per-session Chromium user-data-dir (default: <workspace>/{DEFAULT_USER_DATA_SUBDIR}).", + ) + create.add_argument( + "--headless", + action=argparse.BooleanOptionalAction, + default=True, + help="Launch Chromium headless (default: True).", + ) + create.add_argument( + "--no-sandbox", + action=argparse.BooleanOptionalAction, + default=True, + help="Pass --no-sandbox (often required in containers/CI).", + ) + create.add_argument("--window-width", type=int, default=1280) + create.add_argument("--window-height", type=int, default=1800) + create.add_argument( + "--startup-timeout", + type=float, + default=30.0, + help="Seconds to wait for the DevTools URL to appear on stderr.", + ) + create.add_argument( + "--chromium-arg", + action="append", + default=[], + help="Extra Chromium command-line argument; repeat for multiple.", + ) + create.set_defaults(func=_cmd_create) + + info = sub.add_parser("info", help="Print the persisted session JSON and liveness.") + info.add_argument("--session-file", default=DEFAULT_SESSION_FILE) + info.set_defaults(func=_cmd_info) + + release = sub.add_parser("release", help="Terminate the persisted session.") + release.add_argument("--session-file", default=DEFAULT_SESSION_FILE) + release.add_argument( + "--delete-file", + action=argparse.BooleanOptionalAction, + default=True, + help="Also delete the session JSON file after release.", + ) + release.add_argument( + "--delete-user-data", + action=argparse.BooleanOptionalAction, + default=True, + help="Also remove the per-session Chromium user-data-dir.", + ) + release.add_argument( + "--kill-timeout", + type=float, + default=10.0, + help="Seconds to wait for SIGTERM before sending SIGKILL.", + ) + release.set_defaults(func=_cmd_release) + + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _build_parser().parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/webwright/tools/self_reflection.py b/src/webwright/tools/self_reflection.py new file mode 100644 index 0000000..a856694 --- /dev/null +++ b/src/webwright/tools/self_reflection.py @@ -0,0 +1,611 @@ +"""Self-reflection two-stage screenshot judge CLI. + +Previously named ``two_stage_judge``; renamed to ``self_reflection``. + +Stage 1: for each screenshot, send a (system, user + image) pair to the +configured model and parse a 1-5 ``Score`` with a short ``Reasoning``. + +Stage 2: drop every per-image ``Reasoning`` into the caller-provided final +user prompt template (via ``{image_reasonings}``), attach EVERY screenshot, +and make one final call that must end with ``Status: success`` or +``Status: failure``. + +The CLI reads all of its config from a single JSON file so the agent can +prepare it in one turn and invoke the tool in the next. + +Usage:: + + python -m webwright.tools.self_reflection --config self_reflect_config.json + +JSON schema (paths relative to ``--workspace-dir`` or the CWD):: + + { + "images": ["final_runs/run_001/screenshots/final_execution_1.png", ...], + "image_judge_system_prompt": "...", + "image_judge_user_prompt": "...", // sent verbatim with each image + "final_verdict_system_prompt": "...", + "final_verdict_user_prompt": "...{action_history_log}...{image_reasonings}..." + } + +Any of the four prompt fields may instead be supplied via +``<field>_file`` variants pointing to a text file on disk (recommended when +prompts contain many literal braces or newlines). + +The output JSON written to ``--output`` (or stdout) contains the per-image +records, the image path list, the final response, and +``predicted_label`` (``1`` for success, ``0`` for failure, ``null`` if the +``Status:`` line could not be parsed). Exit code: 0 if PASS, 1 otherwise. +""" + +from __future__ import annotations + +import argparse +import asyncio +import base64 +import json +import mimetypes +import re +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from webwright.models.base import text_part +from webwright.tools._model_config import load_tool_model + +DEFAULT_IMAGE_PARSE_MAX_RETRIES = 3 + +_PROMPT_FIELDS = ( + ("image_judge_system_prompt", True), + ("image_judge_user_prompt", True), + ("final_verdict_system_prompt", True), + ("final_verdict_user_prompt", True), +) + +_IMAGE_SUFFIXES = frozenset({".png", ".jpg", ".jpeg", ".webp"}) + + +# --------------------------------------------------------------------------- +# Image helpers +# --------------------------------------------------------------------------- + +def _resolve_image_path(image_path: str, workspace_dir: str = "") -> Path: + path = Path(image_path) + if not path.is_absolute(): + base_dir = Path(workspace_dir) if workspace_dir else Path.cwd() + path = base_dir / path + path = path.resolve() + if not path.exists(): + raise FileNotFoundError(f"Image path does not exist: {path}") + return path + + +def _final_execution_sort_key(name: str) -> tuple[int, str]: + match = re.match(r"final_execution_(\d+)_", name) + if match: + return (int(match.group(1)), name) + nums = re.findall(r"\d+", name) + return (int(nums[0]) if nums else 0, name) + + +def _run_id_sort_key(name: str) -> tuple[int, str]: + match = re.search(r"run_(\d+)", name) + if match: + return (int(match.group(1)), name) + return (0, name) + + +def _sorted_image_paths(image_dir: Path) -> list[Path]: + if not image_dir.is_dir(): + return [] + return sorted( + [path for path in image_dir.iterdir() if path.is_file() and path.suffix.lower() in _IMAGE_SUFFIXES], + key=lambda path: _final_execution_sort_key(path.name), + ) + + +def _discover_latest_run_screenshots( + final_runs_dir: Path, +) -> tuple[Path | None, list[Path]]: + """Find the highest-numbered ``final_runs/run_<id>/screenshots`` dir and its images. + + Returns ``(run_dir_or_None, sorted_image_paths)``. Empty list if no images found. + """ + if not final_runs_dir.exists() or not final_runs_dir.is_dir(): + return None, [] + candidates = sorted( + (d for d in final_runs_dir.iterdir() if d.is_dir() and re.fullmatch(r"run_\d+", d.name)), + key=lambda p: _run_id_sort_key(p.name), + ) + # Walk from highest-numbered run downward and pick the first one with any screenshots. + for run_dir in reversed(candidates): + screenshots_dir = run_dir / "screenshots" + images = _sorted_image_paths(screenshots_dir) + if images: + return run_dir, images + return None, [] + + +def _infer_run_dir_from_images(images: list[Path]) -> Path | None: + run_dirs = { + path.parent.parent.resolve() + for path in images + if path.parent.name == "screenshots" + } + if len(run_dirs) == 1: + return next(iter(run_dirs)) + return None + + +def _resolve_artifact_dir( + *, + images: list[Path], + discovered_run_dir: Path | None, + output_path: str, + workspace_dir: str, +) -> Path | None: + candidates: list[Path] = [] + + inferred_run_dir = _infer_run_dir_from_images(images) + if inferred_run_dir is not None: + candidates.append(inferred_run_dir) + + if discovered_run_dir is not None: + candidates.append(discovered_run_dir.resolve()) + + if output_path: + candidates.append(Path(output_path).resolve().parent) + + base_dir = Path(workspace_dir).resolve() if workspace_dir else Path.cwd().resolve() + candidates.append(base_dir) + + seen: set[Path] = set() + ordered_candidates: list[Path] = [] + for candidate in candidates: + if candidate in seen: + continue + seen.add(candidate) + ordered_candidates.append(candidate) + + for candidate in ordered_candidates: + if (candidate / "final_script_log.txt").is_file(): + return candidate + + return ordered_candidates[0] if ordered_candidates else None + + +def _load_action_history_log(artifact_dir: Path | None) -> str: + if artifact_dir is None: + return "" + log_path = artifact_dir / "final_script_log.txt" + if not log_path.is_file(): + return "" + return log_path.read_text(encoding="utf-8").rstrip() + + +def _render_final_verdict_user_prompt( + template: str, + *, + image_reasonings: str, + action_history_log: str, +) -> str: + rendered = template + if "{image_reasonings}" in template or "{action_history_log}" in template: + try: + rendered = template.format( + image_reasonings=image_reasonings, + action_history_log=action_history_log, + ) + except KeyError as exc: + raise ValueError( + "Unknown placeholder in final_verdict_user_prompt: " + f"{exc.args[0]!r}. Supported placeholders are " + "{image_reasonings} and {action_history_log}; double any literal " + "braces as {{ and }}." + ) from exc + return rendered + + +def _high_detail_image_part_from_path(image_path: Path) -> dict[str, Any]: + mime_type, _ = mimetypes.guess_type(str(image_path)) + encoded = base64.b64encode(image_path.read_bytes()).decode("ascii") + return { + "type": "input_image", + "image_url": f"data:{mime_type or 'image/png'};base64,{encoded}", + "detail": "high", + } + + +# --------------------------------------------------------------------------- +# Model call: plain message list -> text +# --------------------------------------------------------------------------- + +def _call_model( + *, + model_client: Any, + system_prompt: str, + user_content: list[dict[str, Any]], + max_new_tokens: int, +) -> str: + return model_client( + [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_content}, + ], + max_output_tokens=max_new_tokens, + ).strip() + + +def _model_endpoint(model_client: Any) -> str: + config = getattr(model_client, "config", None) + for key in ("openai_endpoint", "anthropic_endpoint", "openrouter_endpoint"): + value = getattr(config, key, "") + if value: + return str(value) + return "" + + +# --------------------------------------------------------------------------- +# Parsing helpers (ported from webjudge_online_mind2web_sandbox.py) +# --------------------------------------------------------------------------- + +def _parse_image_judge_response(response: str) -> tuple[str, int]: + score_match = re.search(r"(?is)\bscore\b[^1-5]*([1-5])\b", response) + reasoning_match = re.search( + r"(?is)(?:\*\*?\s*reasoning\s*\*\*?|reasoning)\s*[:\-]\s*" + r"(.*?)(?=\n\s*(?:\d+\.\s*)?(?:\*\*?\s*score\s*\*\*?|score)\s*[:\-]|\Z)", + response, + ) + + if score_match and reasoning_match: + reasoning = re.sub(r"\s+", " ", reasoning_match.group(1)).strip() + return reasoning, int(score_match.group(1)) + + try: + payload = json.loads(response) + except Exception: + payload = None + + if isinstance(payload, dict): + score = payload.get("Score", payload.get("score")) + reasoning = payload.get("Reasoning", payload.get("reasoning")) + if ( + isinstance(score, int) + and 1 <= score <= 5 + and isinstance(reasoning, str) + and reasoning.strip() + ): + return re.sub(r"\s+", " ", reasoning).strip(), score + + raise ValueError("Could not parse image judge response") + + +def _parse_final_verdict(response: str) -> int | None: + matches = list(re.finditer(r"(?i)status:\s*", response)) + if not matches: + return None + tail = response[matches[-1].end():].strip() + m = re.match(r"""^[\'\"โ€œโ€โ€˜โ€™\s]*(success|failure)\b""", tail, re.IGNORECASE) + if not m: + return None + return 1 if m.group(1).lower() == "success" else 0 + + +# --------------------------------------------------------------------------- +# Per-image scoring +# --------------------------------------------------------------------------- + +async def _judge_one_image( + *, + image_path: Path, + image_judge_system_prompt: str, + image_judge_user_prompt: str, + model_client: Any, + max_new_tokens: int, + max_parse_retries: int, +) -> dict[str, Any]: + user_content = [ + text_part(image_judge_user_prompt), + _high_detail_image_part_from_path(image_path), + ] + + last_response = "" + last_error: BaseException | None = None + for attempt in range(1, max_parse_retries + 1): + last_response = await asyncio.to_thread( + _call_model, + model_client=model_client, + system_prompt=image_judge_system_prompt, + user_content=user_content, + max_new_tokens=max_new_tokens, + ) + try: + reasoning, score = _parse_image_judge_response(last_response) + return { + "image_path": str(image_path), + "Response": last_response, + "Score": score, + "Reasoning": reasoning, + "Attempts": attempt, + "ParseFailed": False, + } + except Exception as exc: # noqa: BLE001 + last_error = exc + print( + f"[self_reflection] parse attempt {attempt}/{max_parse_retries} failed for " + f"{image_path}: {exc}", + file=sys.stderr, + ) + + return { + "image_path": str(image_path), + "Response": last_response, + "Score": 0, + "Reasoning": "", + "Attempts": max_parse_retries, + "ParseFailed": True, + "ParseError": str(last_error) if last_error is not None else "unknown", + } + + +# --------------------------------------------------------------------------- +# Orchestrator +# --------------------------------------------------------------------------- + +@dataclass +class SelfReflectionResult: + image_records: list[dict[str, Any]] + image_paths: list[str] + final_user_text: str + final_system_msg: str + final_response: str + predicted_label: int | None # 1 success, 0 failure, None unparsed + model: str = "" + endpoint: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "model": self.model, + "endpoint": self.endpoint, + "predicted_label": self.predicted_label, + "final_response": self.final_response, + "final_user_text": self.final_user_text, + "final_system_msg": self.final_system_msg, + "image_paths": self.image_paths, + "image_records": self.image_records, + } + + +async def run_self_reflection_async( + *, + images: list[Path], + image_judge_system_prompt: str, + image_judge_user_prompt: str, + final_verdict_system_prompt: str, + final_verdict_user_prompt: str, + action_history_log: str, + max_image_parse_retries: int, + final_max_new_tokens: int, + image_max_new_tokens: int, + model_client: Any, +) -> SelfReflectionResult: + model_name = str(getattr(model_client.config, "model_name", "")) + endpoint = _model_endpoint(model_client) + + if images: + per_image = await asyncio.gather( + *( + _judge_one_image( + image_path=path, + image_judge_system_prompt=image_judge_system_prompt, + image_judge_user_prompt=image_judge_user_prompt, + model_client=model_client, + max_new_tokens=image_max_new_tokens, + max_parse_retries=max_image_parse_retries, + ) + for path in images + ) + ) + else: + per_image = [] + + image_paths = [record["image_path"] for record in per_image] + reasonings = [record["Reasoning"] or "" for record in per_image] + + reasonings_block = "\n".join( + f"{i + 1}. {text}" for i, text in enumerate(reasonings) + ) + + final_user_text = _render_final_verdict_user_prompt( + final_verdict_user_prompt, + image_reasonings=reasonings_block, + action_history_log=action_history_log, + ) + + user_content: list[dict[str, Any]] = [text_part(final_user_text)] + for path_str in image_paths: + user_content.append(_high_detail_image_part_from_path(Path(path_str))) + + final_response = await asyncio.to_thread( + _call_model, + model_client=model_client, + system_prompt=final_verdict_system_prompt, + user_content=user_content, + max_new_tokens=final_max_new_tokens, + ) + predicted_label = _parse_final_verdict(final_response) + + return SelfReflectionResult( + image_records=list(per_image), + image_paths=image_paths, + final_user_text=final_user_text, + final_system_msg=final_verdict_system_prompt, + final_response=final_response, + predicted_label=predicted_label, + model=model_name, + endpoint=endpoint, + ) + + +def run_self_reflection(**kwargs: Any) -> SelfReflectionResult: + return asyncio.run(run_self_reflection_async(**kwargs)) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def _resolve_prompt(cfg: dict[str, Any], key: str, *, required: bool) -> str | None: + inline = cfg.get(key) + file_key = f"{key}_file" + file_path = cfg.get(file_key) + if inline is not None and file_path is not None: + raise ValueError(f"Provide only one of {key!r} or {file_key!r}, not both.") + if file_path is not None: + return Path(file_path).read_text(encoding="utf-8") + if inline is not None: + return inline + if required: + raise ValueError(f"Missing required prompt: {key} (or {file_key}).") + return None + + +def _load_config(config_arg: str) -> dict[str, Any]: + if config_arg == "-": + return json.loads(sys.stdin.read()) + return json.loads(Path(config_arg).read_text(encoding="utf-8")) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Two-stage screenshot judge. Reads a JSON config describing images and " + "prompts, calls the configured model, and prints a " + "JSON result with per-image records and the final verdict." + ) + ) + parser.add_argument("--config", required=True, help="Path to JSON config, or '-' for stdin.") + parser.add_argument("--workspace-dir", default="", help="Base directory for relative image paths.") + parser.add_argument("--output", default="", help="Write JSON result to this path instead of stdout.") + parser.add_argument( + "--auto-latest-run", + default="final_runs", + help=( + "When the config has no 'images' list, auto-discover screenshots from the " + "highest-numbered `<workspace-dir>/<this-value>/run_<id>/screenshots` folder. " + "Default: 'final_runs'. Pass '' (empty string) to disable auto-discovery." + ), + ) + parser.add_argument("--max-image-parse-retries", type=int, default=DEFAULT_IMAGE_PARSE_MAX_RETRIES) + parser.add_argument("--image-max-new-tokens", type=int, default=1024) + parser.add_argument("--final-max-new-tokens", type=int, default=8192) + parser.add_argument( + "--model-config", + default="", + help=( + "Path to a JSON/YAML config containing a top-level `model:` block. " + "If omitted, reads <workspace-dir>/config_snapshot/merged_config.yaml." + ), + ) + parser.add_argument("--timeout-seconds", type=int, default=120) + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + base_dir = Path(args.workspace_dir).resolve() if args.workspace_dir else Path.cwd().resolve() + + cfg = _load_config(args.config) + + prompts = { + key: _resolve_prompt(cfg, key, required=required) + for key, required in _PROMPT_FIELDS + } + + images_config = cfg.get("images") or cfg.get("images_path") or [] + resolved_images = [ + _resolve_image_path(p, workspace_dir=args.workspace_dir) for p in images_config + ] + discovered_run_dir = _infer_run_dir_from_images(resolved_images) + + # If config did not provide images, fall back to the latest run's screenshots. + if not resolved_images: + discovered: list[Path] = [] + discovered_source = "" + if args.auto_latest_run: + auto_root = Path(args.auto_latest_run) + if not auto_root.is_absolute(): + auto_root = base_dir / auto_root + auto_root = auto_root.resolve() + discovered_run_dir, discovered = _discover_latest_run_screenshots(auto_root) + if discovered_run_dir is not None: + discovered_source = str(discovered_run_dir / "screenshots") + if discovered: + resolved_images = discovered + print( + f"[self_reflection] auto-discovered {len(resolved_images)} screenshots from {discovered_source}", + file=sys.stderr, + ) + + artifact_dir = _resolve_artifact_dir( + images=resolved_images, + discovered_run_dir=discovered_run_dir, + output_path=args.output, + workspace_dir=args.workspace_dir, + ) + action_history_log = _load_action_history_log(artifact_dir) + + if not resolved_images: + print( + "[self_reflection] warning: no images provided; final stage will run without screenshot attachments.", + file=sys.stderr, + ) + + if not action_history_log: + print( + "[self_reflection] warning: no final_script_log.txt found; final prompt will omit action history content.", + file=sys.stderr, + ) + + model_client = load_tool_model( + model_config_arg=args.model_config, + workspace_dir=args.workspace_dir, + timeout_seconds=args.timeout_seconds, + ) + + result = run_self_reflection( + images=resolved_images, + image_judge_system_prompt=prompts["image_judge_system_prompt"], + image_judge_user_prompt=prompts["image_judge_user_prompt"], + final_verdict_system_prompt=prompts["final_verdict_system_prompt"], + final_verdict_user_prompt=prompts["final_verdict_user_prompt"], + action_history_log=action_history_log, + max_image_parse_retries=args.max_image_parse_retries, + final_max_new_tokens=args.final_max_new_tokens, + image_max_new_tokens=args.image_max_new_tokens, + model_client=model_client, + ) + + payload = result.to_dict() + serialized = json.dumps(payload, indent=2, ensure_ascii=False) + if args.output: + Path(args.output).write_text(serialized, encoding="utf-8") + print(f"Wrote result to {args.output}", file=sys.stderr) + else: + sys.stdout.write(serialized) + sys.stdout.write("\n") + + label = result.predicted_label + if label == 1: + print("JUDGE VERDICT: PASS", file=sys.stderr) + return 0 + if label == 0: + print("JUDGE VERDICT: FAIL", file=sys.stderr) + return 1 + print("JUDGE VERDICT: UNPARSED (treating as FAIL)", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/webwright/utils/__init__.py b/src/webwright/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/webwright/utils/logging.py b/src/webwright/utils/logging.py new file mode 100644 index 0000000..263361a --- /dev/null +++ b/src/webwright/utils/logging.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from datetime import datetime, timezone +import json +from pathlib import Path +from typing import Any + + +def append_runtime_log(path: Path | None, *, source: str, event: str, **data: Any) -> None: + if path is None: + return + path.parent.mkdir(parents=True, exist_ok=True) + payload = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "source": source, + "event": event, + **data, + } + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(payload, ensure_ascii=True)) + handle.write("\n") diff --git a/src/webwright/utils/runtime.py b/src/webwright/utils/runtime.py new file mode 100644 index 0000000..9941b3a --- /dev/null +++ b/src/webwright/utils/runtime.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +import asyncio +from typing import TypeVar + +T = TypeVar("T") + + +def run_async(coro) -> T: + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + raise RuntimeError("Webwright does not support running inside an active event loop.") diff --git a/src/webwright/utils/serialize.py b/src/webwright/utils/serialize.py new file mode 100644 index 0000000..ffa0d98 --- /dev/null +++ b/src/webwright/utils/serialize.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from typing import Any + +UNSET = object() + + +def recursive_merge(*dictionaries: dict | None) -> dict[str, Any]: + result: dict[str, Any] = {} + for dictionary in dictionaries: + if dictionary is None: + continue + for key, value in dictionary.items(): + if value is UNSET: + continue + if key in result and isinstance(result[key], dict) and isinstance(value, dict): + result[key] = recursive_merge(result[key], value) + elif isinstance(value, dict): + result[key] = recursive_merge(value) + else: + result[key] = value + return result diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..5c8135e --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,6 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) diff --git a/tests/unit/test_doctor.py b/tests/unit/test_doctor.py new file mode 100644 index 0000000..af2cdd1 --- /dev/null +++ b/tests/unit/test_doctor.py @@ -0,0 +1,94 @@ +from pathlib import Path + +from webwright.run.doctor import ( + check_chromium, + check_openai_key, + check_playwright, + check_plugin_manifests, + check_python, + check_screenshot, +) + + +def test_check_python(): + ok, message = check_python() + + assert isinstance(ok, bool) + assert isinstance(message, str) + + +def test_check_playwright(): + ok, message = check_playwright() + + assert isinstance(ok, bool) + assert isinstance(message, str) + + +def test_check_chromium(): + ok, message = check_chromium() + + assert isinstance(ok, bool) + assert isinstance(message, str) + + +def test_check_screenshot(): + ok, message = check_screenshot() + + assert isinstance(ok, bool) + assert isinstance(message, str) + + +def test_check_openai_key_exists(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + + ok, message = check_openai_key() + + assert ok is True + assert "found" in message + + +def test_check_openai_key_missing(monkeypatch): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + + ok, message = check_openai_key() + + assert ok is False + assert "missing" in message + + +def test_plugin_manifests_exist(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + + claude_dir = tmp_path / ".claude-plugin" + codex_dir = tmp_path / ".codex-plugin" + + claude_dir.mkdir() + codex_dir.mkdir() + + (claude_dir / "plugin.json").write_text("{}") + (codex_dir / "plugin.json").write_text("{}") + + ok, message = check_plugin_manifests() + + assert ok is True + assert "found" in message + + +def test_plugin_manifests_missing(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + + ok, message = check_plugin_manifests() + + assert ok is False + assert "missing" in message + + +def test_screenshot_file_cleanup(): + screenshot_path = Path("doctor_test.png") + + if screenshot_path.exists(): + screenshot_path.unlink() + + check_screenshot() + + assert not screenshot_path.exists() diff --git a/tests/unit/test_tool_model_routing.py b/tests/unit/test_tool_model_routing.py new file mode 100644 index 0000000..cf2c1ee --- /dev/null +++ b/tests/unit/test_tool_model_routing.py @@ -0,0 +1,69 @@ +import pytest +import yaml + +from webwright.config import get_config_from_spec +from webwright.tools._model_config import ( + DEFAULT_MERGED_CONFIG_RELPATH, + _extract_model_block, + resolve_model_config_path, +) +from webwright.utils.serialize import recursive_merge + + +def test_model_claude_sets_top_level_anthropic_model() -> None: + config = recursive_merge( + get_config_from_spec("base.yaml"), + get_config_from_spec("model_claude.yaml"), + ) + + assert config["model"]["model_class"] == "anthropic" + + +def test_model_claude_does_not_declare_per_tool_overrides() -> None: + config = get_config_from_spec("model_claude.yaml") + + assert "tools" not in config, "model_claude.yaml should rely on the top-level `model:` block" + + +def test_extract_model_block_reads_top_level_model(tmp_path) -> None: + config = {"model": {"model_class": "anthropic", "model_name": "claude-opus-4-7"}} + + assert _extract_model_block(config) == config["model"] + + +def test_extract_model_block_rejects_missing_block() -> None: + with pytest.raises(ValueError, match="missing a top-level"): + _extract_model_block({}) + + +def test_resolve_model_config_path_prefers_explicit_arg(tmp_path) -> None: + explicit = tmp_path / "explicit.yaml" + explicit.write_text("model: {model_class: anthropic, model_name: claude-opus-4-7}\n") + + snapshot_dir = tmp_path / "ws" / DEFAULT_MERGED_CONFIG_RELPATH.parent + snapshot_dir.mkdir(parents=True) + (snapshot_dir / DEFAULT_MERGED_CONFIG_RELPATH.name).write_text( + yaml.safe_dump({"model": {"model_class": "openai"}}) + ) + + resolved = resolve_model_config_path(str(explicit), workspace_dir=str(tmp_path / "ws")) + + assert resolved == explicit.resolve() + + +def test_resolve_model_config_path_falls_back_to_workspace_snapshot(tmp_path) -> None: + workspace = tmp_path / "ws" + snapshot_path = workspace / DEFAULT_MERGED_CONFIG_RELPATH + snapshot_path.parent.mkdir(parents=True) + snapshot_path.write_text( + yaml.safe_dump({"model": {"model_class": "anthropic", "model_name": "claude-opus-4-7"}}) + ) + + resolved = resolve_model_config_path("", workspace_dir=str(workspace)) + + assert resolved == snapshot_path.resolve() + + +def test_resolve_model_config_path_raises_when_nothing_found(tmp_path) -> None: + with pytest.raises(FileNotFoundError, match="No tool model config found"): + resolve_model_config_path("", workspace_dir=str(tmp_path))