chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,43 @@
|
||||
# Repo code review
|
||||
|
||||
## Goal
|
||||
|
||||
Review a small public git repository, run its tests, leave line-level review comments in the structured output, and write a patch-oriented review artifact.
|
||||
|
||||
## Why this is valuable
|
||||
|
||||
This demo shows a coding-agent workflow where the sandbox can inspect a real git worktree, run tests, reason over a diff, and produce review artifacts that a developer can act on. The manifest mounts `pypa/sampleproject` at a pinned ref with `GitRepo(...)`. The review contract is intentionally narrow: one finding should target the CI workflow, and one should target the missing type hints in `src/sample/simple.py`.
|
||||
|
||||
## Setup
|
||||
|
||||
Run the Unix-local example from the repository root:
|
||||
|
||||
```bash
|
||||
uv run python examples/sandbox/tutorials/repo_code_review/main.py
|
||||
uv run python examples/sandbox/tutorials/repo_code_review/evals.py
|
||||
```
|
||||
|
||||
This demo exits after the scripted review so the generated artifacts and eval contract stay deterministic.
|
||||
|
||||
To run the same review in Docker, build the shared tutorial image once and pass
|
||||
`--docker`:
|
||||
|
||||
```bash
|
||||
docker build -t sandbox-tutorials:latest -f examples/sandbox/tutorials/Dockerfile .
|
||||
uv run python examples/sandbox/tutorials/repo_code_review/main.py --docker
|
||||
uv run python examples/sandbox/tutorials/repo_code_review/evals.py
|
||||
```
|
||||
|
||||
## Expected artifacts
|
||||
|
||||
- `output/review.md`
|
||||
- `output/findings.jsonl`
|
||||
- Optional `output/fix.patch`
|
||||
|
||||
## Demo shape
|
||||
|
||||
- Inputs: `pypa/sampleproject` at a pinned git ref, mounted into the workspace as `repo/`.
|
||||
- Runtime primitives: sandbox-local bash, optional file edits, and a typed `RepoReviewResult` final output.
|
||||
- Workflow: one sandbox reviewer agent is enough here; there is no handoff because the task is a linear inspect -> test -> patch -> summarize loop.
|
||||
- Scratch space: the reviewer can use `scratchpad/` for notes or draft diffs, then return the final review object for the wrapper to persist.
|
||||
- Evals: `evals.py` checks that the two findings stay focused on `uv` in the test workflow and type hints in `src/sample/simple.py`, and that the patch only edits `simple.py`.
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Evaluate the repo code-review demo outputs."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
EXPECTED_FINDING_PATHS = {
|
||||
"repo/.github/workflows/test.yml",
|
||||
"repo/src/sample/simple.py",
|
||||
}
|
||||
|
||||
|
||||
def load_findings(findings_path: Path) -> list[dict[str, object]]:
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in findings_path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def validate_findings(findings: list[dict[str, object]]) -> None:
|
||||
if len(findings) != 2:
|
||||
raise ValueError(f"Expected 2 review findings, got {len(findings)}.")
|
||||
|
||||
finding_paths = {str(finding["file"]) for finding in findings}
|
||||
if finding_paths != EXPECTED_FINDING_PATHS:
|
||||
raise ValueError(
|
||||
f"Expected findings for {sorted(EXPECTED_FINDING_PATHS)}, got {sorted(finding_paths)}."
|
||||
)
|
||||
|
||||
workflow_comment = next(
|
||||
str(finding["comment"])
|
||||
for finding in findings
|
||||
if finding["file"] == "repo/.github/workflows/test.yml"
|
||||
)
|
||||
workflow_words = {word.strip("`.,:;()[]{}").lower() for word in workflow_comment.split()}
|
||||
if "nox" not in workflow_words:
|
||||
raise ValueError("Expected the workflow review comment to mention nox.")
|
||||
if not ({"uv", "pip", "install", "project", "test"} & workflow_words):
|
||||
raise ValueError(
|
||||
"Expected the workflow review comment to describe a concrete test-tooling concern."
|
||||
)
|
||||
|
||||
simple_comment = next(
|
||||
str(finding["comment"])
|
||||
for finding in findings
|
||||
if finding["file"] == "repo/src/sample/simple.py"
|
||||
)
|
||||
if "add_one" not in simple_comment or "-> int" not in simple_comment:
|
||||
raise ValueError("Expected the simple.py review comment to suggest type hints for add_one.")
|
||||
|
||||
|
||||
def validate_patch(patch_path: Path) -> None:
|
||||
patch_text = patch_path.read_text(encoding="utf-8")
|
||||
if "src/sample/simple.py" not in patch_text:
|
||||
raise ValueError("Expected the patch to modify src/sample/simple.py.")
|
||||
if ".github/workflows/test.yml" in patch_text or "noxfile.py" in patch_text:
|
||||
raise ValueError("Expected the patch to avoid CI and noxfile changes.")
|
||||
if "def add_one(number: int) -> int:" not in patch_text:
|
||||
raise ValueError("Expected the patch to add type hints to add_one.")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
type=Path,
|
||||
default=Path(__file__).resolve().parent / "output",
|
||||
help="Directory containing findings.jsonl and fix.patch.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
validate_findings(load_findings(args.output_dir / "findings.jsonl"))
|
||||
validate_patch(args.output_dir / "fix.patch")
|
||||
print("Repo review eval checks passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,173 @@
|
||||
"""
|
||||
Review a small GitHub repository and produce sandbox-generated findings artifacts.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from textwrap import dedent
|
||||
from typing import cast
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from agents import ModelSettings, Runner
|
||||
from agents.run import RunConfig
|
||||
from agents.sandbox import Manifest, SandboxAgent, SandboxRunConfig
|
||||
from agents.sandbox.capabilities import Filesystem, Shell
|
||||
from agents.sandbox.entries import File, GitRepo
|
||||
|
||||
if __package__ is None or __package__ == "":
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
|
||||
|
||||
from examples.sandbox.tutorials.misc import (
|
||||
DEFAULT_SANDBOX_IMAGE,
|
||||
console,
|
||||
create_sandbox_client_and_session,
|
||||
load_env_defaults,
|
||||
print_event,
|
||||
)
|
||||
|
||||
DEMO_DIR = Path(__file__).resolve().parent
|
||||
REPO_NAME = "pypa/sampleproject"
|
||||
REPO_REF = "621e4974ca25ce531773def586ba3ed8e736b3fc"
|
||||
DEFAULT_QUESTION = (
|
||||
"Review this small Python repository as a maintainer. Run the tests, inspect the "
|
||||
"project layout, and return exactly two concise line-level findings: one for "
|
||||
"`repo/.github/workflows/test.yml` about concrete nox/test installation reliability, "
|
||||
"and one for `repo/src/sample/simple.py` about adding explicit type hints to "
|
||||
"`add_one`. Return a patch artifact for the obvious `simple.py` type-hint fix."
|
||||
)
|
||||
AGENTS_MD = dedent(
|
||||
"""\
|
||||
# AGENTS.md
|
||||
|
||||
Review the mounted repository under `repo/` like a maintainer.
|
||||
|
||||
- Run `uv run python -m unittest discover -s tests` from `repo/` and report a short result summary.
|
||||
- Return exactly two findings, using these exact file paths:
|
||||
- `repo/.github/workflows/test.yml`: mention nox and a concrete test-tooling/install concern.
|
||||
- `repo/src/sample/simple.py`: mention `add_one` and suggest `-> int` type hints.
|
||||
- Do not return findings for `pyproject.toml`, `noxfile.py`, README files, or tests.
|
||||
- Do not edit the mounted repository. Return the suggested patch text in `fix_patch`.
|
||||
- Set `fix_patch` to a minimal git diff that only edits `repo/src/sample/simple.py` by changing
|
||||
`def add_one(number):` to `def add_one(number: int) -> int:`.
|
||||
- If you inspect files with shell commands, use paths under `repo/`; use `rg`.
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
class ReviewFinding(BaseModel):
|
||||
file: str = Field(
|
||||
description=(
|
||||
"Exact workspace-relative path under repo/. Preserve casing from the workspace file listing."
|
||||
)
|
||||
)
|
||||
line_number: int = Field(description="1-based line number for the review comment.")
|
||||
comment: str = Field(
|
||||
description=(
|
||||
"Concrete review comment for that line. Include a tiny git-diff-style "
|
||||
"suggestion in the comment when the fix is obvious."
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class RepoReviewResult(BaseModel):
|
||||
test_command: str = Field(description="Exact test command that was run.")
|
||||
test_result: str = Field(description="Short summary of the test outcome.")
|
||||
findings: list[ReviewFinding] = Field(description="Review findings ordered by severity.")
|
||||
review_markdown: str = Field(description="Human-readable review summary in Markdown.")
|
||||
fix_patch: str | None = Field(
|
||||
description="A minimal git diff patch if a fix was made, otherwise null."
|
||||
)
|
||||
|
||||
|
||||
def write_review_artifacts(output_dir: Path, review: RepoReviewResult) -> None:
|
||||
output_dir.mkdir(exist_ok=True)
|
||||
(output_dir / "review.md").write_text(review.review_markdown.strip() + "\n", encoding="utf-8")
|
||||
(output_dir / "findings.jsonl").write_text(
|
||||
"\n".join(
|
||||
json.dumps(finding.model_dump(mode="json"), sort_keys=True)
|
||||
for finding in review.findings
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
if review.fix_patch:
|
||||
(output_dir / "fix.patch").write_text(review.fix_patch.strip() + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
async def main(model: str, question: str, use_docker: bool, image: str) -> None:
|
||||
manifest = Manifest(
|
||||
entries={
|
||||
"AGENTS.md": File(content=AGENTS_MD.encode("utf-8")),
|
||||
"repo": GitRepo(repo=REPO_NAME, ref=REPO_REF),
|
||||
}
|
||||
)
|
||||
agent = SandboxAgent(
|
||||
name="Code Reviewer",
|
||||
model=model,
|
||||
instructions=AGENTS_MD,
|
||||
capabilities=[Shell(), Filesystem()],
|
||||
model_settings=ModelSettings(tool_choice="required"),
|
||||
output_type=RepoReviewResult,
|
||||
)
|
||||
|
||||
client, sandbox = await create_sandbox_client_and_session(
|
||||
manifest=manifest,
|
||||
use_docker=use_docker,
|
||||
image=image,
|
||||
)
|
||||
try:
|
||||
async with sandbox:
|
||||
result = Runner.run_streamed(
|
||||
agent,
|
||||
[{"role": "user", "content": question}],
|
||||
max_turns=25,
|
||||
run_config=RunConfig(
|
||||
sandbox=SandboxRunConfig(session=sandbox),
|
||||
tracing_disabled=True,
|
||||
workflow_name="Repo Review example",
|
||||
),
|
||||
)
|
||||
async for event in result.stream_events():
|
||||
print_event(event)
|
||||
if result.final_output is None:
|
||||
raise RuntimeError("Code Reviewer returned no structured review output.")
|
||||
print_event(str(result.final_output).strip())
|
||||
review = cast(RepoReviewResult, result.final_output)
|
||||
finally:
|
||||
await client.delete(sandbox)
|
||||
|
||||
write_review_artifacts(DEMO_DIR / "output", review)
|
||||
console.print(f"[green]Wrote review artifacts to {DEMO_DIR / 'output'}[/green]")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
load_env_defaults(DEMO_DIR / ".env")
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="gpt-5.4-mini",
|
||||
help="Model name to use.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--question",
|
||||
default=DEFAULT_QUESTION,
|
||||
help="Prompt to send to the agent.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--docker",
|
||||
action="store_true",
|
||||
help="Run this example in Docker instead of Unix-local.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--image",
|
||||
default=DEFAULT_SANDBOX_IMAGE,
|
||||
help="Docker image to use when --docker is set.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
asyncio.run(main(args.model, args.question, args.docker, args.image))
|
||||
Reference in New Issue
Block a user