chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:39:17 +08:00
commit 4ed4e9ff99
1368 changed files with 334957 additions and 0 deletions
@@ -0,0 +1,44 @@
# Dataroom metric extract
## Goal
Extract financial metrics from a synthetic 10-K packet, write the resulting table as CSV or JSONL, then validate the generated artifact with a deterministic eval script.
The packet uses synthetic company data, but the source docs are formatted as annual-report excerpts with 10-K `Part II, Item 7` MD&A sections and `Part II, Item 8` financial statement sections.
## Why this is valuable
This demo shows a single-pass structured extraction pattern: a sandbox agent reads messy filing documents and emits typed financial rows, then a separate host-side eval script checks the artifact. The wrapper does not repair or deduplicate model output after the fact; if the row set is wrong, `evals.py` fails and you iterate on the prompt or fixture data instead.
## Setup
Run the fixture generator and then the Unix-local example from the repository root. Set `OPENAI_API_KEY` in your shell environment before running the example.
```bash
uv run python examples/sandbox/tutorials/data/dataroom/setup.py
uv run python examples/sandbox/tutorials/dataroom_metric_extract/main.py --output-format csv
uv run python examples/sandbox/tutorials/dataroom_metric_extract/evals.py --artifact-path examples/sandbox/tutorials/dataroom_metric_extract/output/financial_metrics.csv
```
After the initial extraction, the demo keeps the sandbox session open for Rich-rendered follow-up prompts before writing the final artifact. Pass `--no-interactive` for a one-shot run.
To run extraction in Docker, build the shared tutorial image once and add `--docker`
to `main.py`:
```bash
docker build --tag sandbox-tutorials:latest examples/sandbox/tutorials
uv run python examples/sandbox/tutorials/dataroom_metric_extract/main.py --docker --output-format csv
uv run python examples/sandbox/tutorials/dataroom_metric_extract/evals.py --artifact-path examples/sandbox/tutorials/dataroom_metric_extract/output/financial_metrics.csv
```
## Expected artifacts
- `output/financial_metrics.csv`
- `output/financial_metrics.jsonl`
## Demo shape
- Inputs: the shared SEC fixture packet in `examples/sandbox/tutorials/data/dataroom/`.
- Runtime primitives: sandbox-local bash/file search plus typed agent outputs.
- Workflow: a fixed single-step pipeline where the sandbox extractor emits `FinancialMetricBatch`; no handoff is needed. `main.py` writes the selected artifact format, and `evals.py` validates that artifact in a separate step.
- Scratch space: the extractor may use `scratchpad/` for interim notes, but only the selected `output/financial_metrics.*` artifact is part of the final contract.
@@ -0,0 +1,315 @@
from __future__ import annotations
import argparse
import csv
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, TypeAlias
if __package__ is None or __package__ == "":
sys.path.insert(0, str(Path(__file__).resolve().parent))
if TYPE_CHECKING or __package__:
from .schemas import FinancialMetric, FinancialMetricBatch
else:
from schemas import FinancialMetric, FinancialMetricBatch
MetricKey: TypeAlias = tuple[str, str, str, str | None]
EXPECTED_SOURCE_METADATA: dict[str, str] = {
"data/10-k-mdna-overview.txt": (
"Part II, Item 7. Management's Discussion and Analysis of Financial Condition and "
"Results of Operations"
),
"data/10-k-mdna-liquidity.txt": (
"Part II, Item 7. Management's Discussion and Analysis of Financial Condition and "
"Results of Operations"
),
"data/10-k-note-segments.txt": ("Part II, Item 8. Financial Statements and Supplementary Data"),
"data/10-k-note-geography.txt": (
"Part II, Item 8. Financial Statements and Supplementary Data"
),
"data/10-k-note-balance-sheet.txt": (
"Part II, Item 8. Financial Statements and Supplementary Data"
),
"data/10-k-statements-of-operations.pdf": (
"Part II, Item 8. Financial Statements and Supplementary Data"
),
"data/10-k-balance-sheets.pdf": (
"Part II, Item 8. Financial Statements and Supplementary Data"
),
"data/10-k-statements-of-cash-flows.pdf": (
"Part II, Item 8. Financial Statements and Supplementary Data"
),
}
EXPECTED_ROWS: dict[MetricKey, tuple[float, str]] = {
("data/10-k-mdna-overview.txt", "Revenue", "FY2025", None): (1284.0, "USD millions"),
("data/10-k-mdna-overview.txt", "Revenue", "FY2024", None): (1008.0, "USD millions"),
("data/10-k-mdna-overview.txt", "Gross margin", "FY2025", None): (71.4, "percent"),
("data/10-k-mdna-overview.txt", "Gross margin", "FY2024", None): (68.2, "percent"),
("data/10-k-mdna-overview.txt", "Operating income", "FY2025", None): (186.0, "USD millions"),
("data/10-k-mdna-overview.txt", "Operating income", "FY2024", None): (118.0, "USD millions"),
(
"data/10-k-mdna-liquidity.txt",
"Net cash provided by operating activities",
"FY2025",
None,
): (248.0, "USD millions"),
(
"data/10-k-mdna-liquidity.txt",
"Net cash provided by operating activities",
"FY2024",
None,
): (192.0, "USD millions"),
("data/10-k-mdna-liquidity.txt", "Capital expenditures", "FY2025", None): (
86.0,
"USD millions",
),
("data/10-k-mdna-liquidity.txt", "Capital expenditures", "FY2024", None): (
73.0,
"USD millions",
),
("data/10-k-mdna-liquidity.txt", "Free cash flow", "FY2025", None): (
162.0,
"USD millions",
),
("data/10-k-mdna-liquidity.txt", "Free cash flow", "FY2024", None): (
119.0,
"USD millions",
),
("data/10-k-note-segments.txt", "Platform segment revenue", "FY2025", "Platform"): (
942.0,
"USD millions",
),
("data/10-k-note-segments.txt", "Platform segment revenue", "FY2024", "Platform"): (
711.0,
"USD millions",
),
("data/10-k-note-segments.txt", "Services segment revenue", "FY2025", "Services"): (
342.0,
"USD millions",
),
("data/10-k-note-segments.txt", "Services segment revenue", "FY2024", "Services"): (
297.0,
"USD millions",
),
("data/10-k-note-geography.txt", "Americas revenue", "FY2025", "Americas"): (
764.0,
"USD millions",
),
("data/10-k-note-geography.txt", "EMEA revenue", "FY2025", "EMEA"): (
343.0,
"USD millions",
),
("data/10-k-note-geography.txt", "APAC revenue", "FY2025", "APAC"): (
177.0,
"USD millions",
),
(
"data/10-k-note-balance-sheet.txt",
"Cash and cash equivalents",
"2025-12-31",
None,
): (422.0, "USD millions"),
(
"data/10-k-note-balance-sheet.txt",
"Cash and cash equivalents",
"2024-12-31",
None,
): (351.0, "USD millions"),
("data/10-k-note-balance-sheet.txt", "Deferred revenue", "2025-12-31", None): (
402.0,
"USD millions",
),
("data/10-k-note-balance-sheet.txt", "Deferred revenue", "2024-12-31", None): (
337.0,
"USD millions",
),
("data/10-k-statements-of-operations.pdf", "Net revenue", "FY2025", None): (
1284.0,
"USD millions",
),
("data/10-k-statements-of-operations.pdf", "Net revenue", "FY2024", None): (
1008.0,
"USD millions",
),
("data/10-k-statements-of-operations.pdf", "Gross profit", "FY2025", None): (
917.0,
"USD millions",
),
("data/10-k-statements-of-operations.pdf", "Gross profit", "FY2024", None): (
687.0,
"USD millions",
),
("data/10-k-statements-of-operations.pdf", "Operating income", "FY2025", None): (
186.0,
"USD millions",
),
("data/10-k-statements-of-operations.pdf", "Operating income", "FY2024", None): (
118.0,
"USD millions",
),
(
"data/10-k-balance-sheets.pdf",
"Cash and cash equivalents",
"2025-12-31",
None,
): (422.0, "USD millions"),
(
"data/10-k-balance-sheets.pdf",
"Cash and cash equivalents",
"2024-12-31",
None,
): (351.0, "USD millions"),
("data/10-k-balance-sheets.pdf", "Accounts receivable", "2025-12-31", None): (
211.0,
"USD millions",
),
("data/10-k-balance-sheets.pdf", "Accounts receivable", "2024-12-31", None): (
187.0,
"USD millions",
),
("data/10-k-balance-sheets.pdf", "Deferred revenue", "2025-12-31", None): (
402.0,
"USD millions",
),
("data/10-k-balance-sheets.pdf", "Deferred revenue", "2024-12-31", None): (
337.0,
"USD millions",
),
(
"data/10-k-statements-of-cash-flows.pdf",
"Net cash provided by operating activities",
"FY2025",
None,
): (248.0, "USD millions"),
(
"data/10-k-statements-of-cash-flows.pdf",
"Net cash provided by operating activities",
"FY2024",
None,
): (192.0, "USD millions"),
("data/10-k-statements-of-cash-flows.pdf", "Capital expenditures", "FY2025", None): (
86.0,
"USD millions",
),
("data/10-k-statements-of-cash-flows.pdf", "Capital expenditures", "FY2024", None): (
73.0,
"USD millions",
),
("data/10-k-statements-of-cash-flows.pdf", "Free cash flow", "FY2025", None): (
162.0,
"USD millions",
),
("data/10-k-statements-of-cash-flows.pdf", "Free cash flow", "FY2024", None): (
119.0,
"USD millions",
),
}
@dataclass(frozen=True)
class EvalSummary:
row_count: int
def load_metrics(artifact_path: Path) -> FinancialMetricBatch:
if artifact_path.suffix == ".jsonl":
metrics = [
FinancialMetric.model_validate_json(line)
for line in artifact_path.read_text(encoding="utf-8").splitlines()
if line.strip()
]
return FinancialMetricBatch(metrics=metrics)
if artifact_path.suffix == ".csv":
with artifact_path.open(encoding="utf-8", newline="") as input_file:
reader = csv.DictReader(input_file)
metrics = []
for row in reader:
row["segment"] = row["segment"] or None
row["value"] = float(row["value"])
metrics.append(FinancialMetric.model_validate(row))
return FinancialMetricBatch(metrics=metrics)
raise ValueError(f"Unsupported artifact type: {artifact_path}")
def validate_outputs(metrics: FinancialMetricBatch) -> EvalSummary:
rows = metrics.metrics
duplicate_keys: list[MetricKey] = []
seen_keys: set[MetricKey] = set()
rows_by_key: dict[MetricKey, FinancialMetric] = {
(
row.source_file.strip(),
row.metric_name.strip(),
row.fiscal_period,
row.segment.strip() if row.segment else None,
): row
for row in rows
}
for row in rows:
row_key = (
row.source_file.strip(),
row.metric_name.strip(),
row.fiscal_period,
row.segment.strip() if row.segment else None,
)
if row_key in seen_keys:
duplicate_keys.append(row_key)
seen_keys.add(row_key)
if duplicate_keys:
raise AssertionError(f"Duplicate metric rows found: {sorted(set(duplicate_keys))}.")
if len(rows) != len(EXPECTED_ROWS):
raise AssertionError(
f"Expected exactly {len(EXPECTED_ROWS)} metric rows, found {len(rows)}."
)
for source_file, expected_section in EXPECTED_SOURCE_METADATA.items():
source_rows = [row for row in rows if row.source_file.strip() == source_file]
if not source_rows:
raise AssertionError(f"Missing rows from {source_file}.")
bad_sections = {
row.filing_section for row in source_rows if row.filing_section != expected_section
}
if bad_sections:
raise AssertionError(
f"{source_file} filing_section mismatch. Expected {expected_section}, found {bad_sections}."
)
missing_rows = [
key
for key, (expected_value, expected_unit) in EXPECTED_ROWS.items()
if key not in rows_by_key
or rows_by_key[key].value != expected_value
or rows_by_key[key].unit != expected_unit
]
if missing_rows:
observed = sorted(rows_by_key)
raise AssertionError(
f"Missing or mismatched expected metric rows: {missing_rows}. Observed keys: {observed}."
)
unexpected_rows = sorted(set(rows_by_key) - set(EXPECTED_ROWS))
if unexpected_rows:
raise AssertionError(f"Unexpected metric rows found: {unexpected_rows}.")
return EvalSummary(row_count=len(rows))
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
"--artifact-path",
default=str(Path(__file__).resolve().parent / "output" / "financial_metrics.jsonl"),
help="Path to the generated JSONL or CSV artifact.",
)
args = parser.parse_args()
summary = validate_outputs(load_metrics(Path(args.artifact_path)))
print(f"Eval checks passed for {summary.row_count} metric row(s).")
@@ -0,0 +1,274 @@
"""
Extract structured financial metrics from a synthetic 10-K dataroom and write a
JSONL or CSV artifact.
"""
import argparse
import asyncio
import csv
import json
import sys
from collections.abc import Sequence
from pathlib import Path
from textwrap import dedent
from typing import TYPE_CHECKING, Literal, cast
from openai.types.shared.reasoning import Reasoning
from pydantic import BaseModel
from agents import ModelSettings, Runner, RunResultStreaming, TResponseInputItem
from agents.run import RunConfig
from agents.sandbox import Manifest, SandboxAgent, SandboxRunConfig
from agents.sandbox.capabilities import Shell
from agents.sandbox.entries import File, LocalDir
if __package__ is None or __package__ == "":
sys.path.insert(0, str(Path(__file__).resolve().parent))
sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
if TYPE_CHECKING or __package__:
from .schemas import FinancialMetric, FinancialMetricBatch
else:
from schemas import FinancialMetric, FinancialMetricBatch
from examples.sandbox.tutorials.misc import (
DEFAULT_SANDBOX_IMAGE,
console,
create_sandbox_client_and_session,
load_env_defaults,
print_event,
run_interactive_loop,
)
DEMO_DIR = Path(__file__).resolve().parent
DATAROOM_DATA_DIR = DEMO_DIR.parent / "data" / "dataroom"
DEFAULT_QUESTION = (
"Extract revenue, gross margin, operating income, cash flow, balance-sheet, segment, "
"and geography metrics from the 10-K packet into one row per metric-period-source. "
"For each table, include every explicit line item in the source, even when it is "
"similar to a line item in another source."
)
AGENTS_MD = dedent(
"""\
# AGENTS.md
Extract structured financial metrics from the synthetic 10-K packet under `data/`.
## Output (one row per metric-value occurrence)
Required fields: `source_file`, `filing_section`, `metric_name`, `fiscal_period`, `value`,
`unit` (`USD millions` or `percent`).
Optional field: `segment` (segment/geography if explicitly stated, else null).
## Rules
- Review all `.txt` and `.pdf` under `data/` (these PDFs contain searchable text).
- Use shell tools (`rg`, `sed`) for discovery/inspection; do not run Python from the sandbox shell.
- Do not read `data/setup.py`.
- Emit a separate row for each metric-period pair in each source file (do not dedupe across files).
- For tables, include every explicit table line item in that source. For example, the
statements-of-operations PDF has separate Net revenue, Gross profit, and Operating income rows.
- Only extract explicit source line items / table rows. Do not invent rollups or “cleaned up” metrics.
- Do not treat Gross profit and Gross margin as duplicates; they are distinct source metrics.
- Preserve labels as written (e.g., `Revenue` vs `Net revenue`).
## Completeness checklist
Before final output, verify the batch has exactly 41 rows from these source-level line items:
- `data/10-k-mdna-overview.txt`: Revenue, Gross margin, and Operating income for FY2025 and FY2024.
- `data/10-k-mdna-liquidity.txt`: Net cash provided by operating activities, Capital expenditures,
and Free cash flow for FY2025 and FY2024.
- `data/10-k-note-segments.txt`: Platform segment revenue and Services segment revenue for FY2025
and FY2024, with the matching segment names.
- `data/10-k-note-geography.txt`: Americas revenue, EMEA revenue, and APAC revenue for FY2025, with
the matching geography names as segments.
- `data/10-k-note-balance-sheet.txt`: Cash and cash equivalents and Deferred revenue for 2025-12-31
and 2024-12-31.
- `data/10-k-statements-of-operations.pdf`: Net revenue, Gross profit, and Operating income for
FY2025 and FY2024.
- `data/10-k-balance-sheets.pdf`: Cash and cash equivalents, Accounts receivable, and Deferred revenue
for 2025-12-31 and 2024-12-31.
- `data/10-k-statements-of-cash-flows.pdf`: Net cash provided by operating activities, Capital
expenditures, and Free cash flow for FY2025 and FY2024.
Return the structured rows directly in your final output.
"""
)
async def print_streamed_result(result: RunResultStreaming) -> BaseModel:
async for event in result.stream_events():
print_event(event)
if result.final_output is None:
raise RuntimeError("10-K Metric Extractor returned no structured metric output.")
print_event(str(result.final_output).strip())
return cast(BaseModel, result.final_output)
def write_jsonl(path: Path, metrics: Sequence[BaseModel]) -> None:
path.write_text(
"\n".join(metric.model_dump_json() for metric in metrics) + "\n",
encoding="utf-8",
)
def write_csv(path: Path, metrics: list[FinancialMetric]) -> None:
with path.open("w", encoding="utf-8", newline="") as output_file:
writer = csv.DictWriter(
output_file,
fieldnames=[
"source_file",
"filing_section",
"metric_name",
"fiscal_period",
"value",
"unit",
"segment",
],
)
writer.writeheader()
for metric in metrics:
writer.writerow(json.loads(metric.model_dump_json()))
def write_final_artifact(
output_dir: Path,
output_format: Literal["jsonl", "csv"],
metrics: list[FinancialMetric],
) -> Path:
output_path = output_dir / f"financial_metrics.{output_format}"
if output_format == "jsonl":
write_jsonl(output_path, metrics)
else:
write_csv(output_path, metrics)
return output_path
async def main(
model: str,
question: str,
output_format: Literal["jsonl", "csv"],
use_docker: bool,
image: str,
no_interactive: bool,
) -> None:
if not (DATAROOM_DATA_DIR / "10-k-mdna-overview.txt").exists():
raise SystemExit(
"Run `uv run python examples/sandbox/tutorials/data/dataroom/setup.py` "
"before starting this demo."
)
manifest = Manifest(
entries={
"AGENTS.md": File(content=AGENTS_MD.encode("utf-8")),
"data": LocalDir(src=DATAROOM_DATA_DIR),
}
)
agent = SandboxAgent(
name="10-K Metric Extractor",
model=model,
instructions=AGENTS_MD,
capabilities=[Shell()],
model_settings=ModelSettings(
reasoning=Reasoning(effort="high"),
tool_choice="required",
),
output_type=FinancialMetricBatch,
)
client, sandbox = await create_sandbox_client_and_session(
manifest=manifest,
use_docker=use_docker,
image=image,
)
try:
async with sandbox:
extracted_metrics: FinancialMetricBatch | None = None
async def run_turn(
conversation: list[TResponseInputItem],
) -> list[TResponseInputItem]:
nonlocal extracted_metrics
result = Runner.run_streamed(
agent,
conversation,
max_turns=25,
run_config=RunConfig(
sandbox=SandboxRunConfig(session=sandbox),
tracing_disabled=True,
workflow_name="Dataroom extraction example",
),
)
extracted_metrics = cast(FinancialMetricBatch, await print_streamed_result(result))
return result.to_input_list()
conversation: list[TResponseInputItem] = [{"role": "user", "content": question}]
conversation = await run_turn(conversation)
await run_interactive_loop(
conversation=conversation,
no_interactive=no_interactive,
run_turn=run_turn,
)
finally:
await client.delete(sandbox)
if extracted_metrics is None:
raise RuntimeError("10-K Metric Extractor returned no structured metric output.")
output_dir = DEMO_DIR / "output"
output_dir.mkdir(exist_ok=True)
artifact_path = write_final_artifact(output_dir, output_format, extracted_metrics.metrics)
console.print(
f"[green]Wrote {len(extracted_metrics.metrics)} metric row(s) to {artifact_path}[/green]"
)
if __name__ == "__main__":
load_env_defaults(DEMO_DIR / ".env")
parser = argparse.ArgumentParser()
parser.add_argument(
"--model",
default="gpt-5.4-mini",
help="Model name to use.",
)
parser.add_argument(
"--question",
default=DEFAULT_QUESTION,
help="Prompt to send to the agent.",
)
parser.add_argument(
"--output-format",
choices=("jsonl", "csv"),
default="csv",
help="Artifact format.",
)
parser.add_argument(
"--docker",
action="store_true",
help="Run this example in Docker instead of Unix-local.",
)
parser.add_argument(
"--image",
default=DEFAULT_SANDBOX_IMAGE,
help="Docker image to use when --docker is set.",
)
parser.add_argument(
"--no-interactive",
action="store_true",
help="Run the scripted turn and skip follow-up terminal input.",
)
args = parser.parse_args()
asyncio.run(
main(
args.model,
args.question,
args.output_format,
args.docker,
args.image,
args.no_interactive,
)
)
@@ -0,0 +1,33 @@
from typing import Literal
from pydantic import BaseModel, Field
class FinancialMetric(BaseModel):
source_file: str = Field(
description="Workspace-relative source path under data/, such as data/10-k-mdna-overview.txt."
)
filing_section: Literal[
"Part II, Item 7. Management's Discussion and Analysis of Financial Condition and Results of Operations",
"Part II, Item 8. Financial Statements and Supplementary Data",
] = Field(description="Normalized 10-K filing section for the source document.")
metric_name: str = Field(
description="Metric label exactly as written in the source document or table."
)
fiscal_period: Literal["FY2025", "FY2024", "2025-12-31", "2024-12-31"] = Field(
description="Annual period label for statement rows, or balance-sheet date for point-in-time rows."
)
value: float = Field(description="Numeric value from the source row.")
unit: Literal["USD millions", "percent"] = Field(
description="Unit for `value`; use USD millions for dollar amounts and percent for margins."
)
segment: str | None = Field(
default=None,
description="Reportable segment or geography when the row is segment-specific, otherwise null.",
)
class FinancialMetricBatch(BaseModel):
metrics: list[FinancialMetric] = Field(
description="One row per metric-period pair extracted from each source document."
)