wehub-resource-sync
125 lines
3.6 KiB
Bash
125 lines
3.6 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
# Benchmark command examples. This file is a catalog: copy or run individual
|
|
# blocks as needed. The early exit below prevents accidental full execution.
|
|
|
|
exit 0
|
|
|
|
# 1. Discovery
|
|
# List every registered task.
|
|
python -m bench_env.run --list
|
|
|
|
# 2. Single-task local checks
|
|
# Manual run against the Vite dev server.
|
|
python -m bench_env.run \
|
|
--task-id clock.CountAlarms \
|
|
--env-url http://localhost:3000 \
|
|
--agent human
|
|
|
|
# Model run against a local OpenAI-compatible endpoint.
|
|
python -m bench_env.run \
|
|
--task-id clock.CountAlarms \
|
|
--env-url http://localhost:3000 \
|
|
--model-base-url http://localhost:8001/v1 \
|
|
--model-name your-model-name \
|
|
--agent generic_v2 \
|
|
--headless
|
|
|
|
# 3. Small suite smoke tests
|
|
# Low-concurrency suite run against the Vite dev server.
|
|
python -m bench_env.run \
|
|
--suite ebay \
|
|
--parallel 2 \
|
|
--isolation pages \
|
|
--env-url http://localhost:3000 \
|
|
--model-base-url http://localhost:8001/v1 \
|
|
--model-name your-model-name \
|
|
--agent generic_v2 \
|
|
--headless \
|
|
--runs-dir runs/example_ebay
|
|
|
|
# Browser traffic proxy for Playwright-launched Chromium pages.
|
|
# Useful when app pages need browser-side network access through a local proxy,
|
|
# for example Google Maps assets or tiles.
|
|
python -m bench_env.run \
|
|
--suite ebay \
|
|
--parallel 2 \
|
|
--isolation pages \
|
|
--env-url http://localhost:3000 \
|
|
--model-base-url http://localhost:8001/v1 \
|
|
--model-name your-model-name \
|
|
--agent generic_v2 \
|
|
--headless \
|
|
--proxy http://127.0.0.1:7890 \
|
|
--runs-dir runs/example_ebay_proxy
|
|
|
|
# 4. Medium-scale model runs
|
|
# Multi-browser run against the nginx gateway.
|
|
python -m bench_env.run \
|
|
--suite redbook \
|
|
--parallel 16 --headless --browsers 2 \
|
|
--isolation pages \
|
|
--env-url https://localhost:4180 \
|
|
--model-base-url http://localhost:8006/v1 \
|
|
--model-name qwen3-vl-4b-10s \
|
|
--agent generic_v2 \
|
|
--runs-dir runs/new_redbook_10s
|
|
|
|
# Remote-model run. Provide these through your shell environment.
|
|
python -m bench_env.run \
|
|
--suite x \
|
|
--split test \
|
|
--parallel 8 --headless --browsers 2 \
|
|
--isolation pages \
|
|
--env-url https://localhost:4180 \
|
|
--model-base-url "$YUNWU_API_URL" \
|
|
--model-api-key "$YUNWU_API_KEY" \
|
|
--model-name gemini-3.1-pro-preview \
|
|
--agent generic_v2 \
|
|
--runs-dir runs/new_x_gemini
|
|
|
|
# 5. Human validation and reruns
|
|
# Manual suite run against the nginx gateway.
|
|
|
|
# Manual redbook suite check.
|
|
python -m bench_env.run \
|
|
--suite redbook \
|
|
--env-url https://localhost:4180 \
|
|
--agent human \
|
|
--runs-dir runs/new_redbook_human
|
|
|
|
# Rerun failed episodes from a previous run directory.
|
|
python -m bench_env.run --rerun runs/new_redbook_human/20260512_215537 \
|
|
--rerun-scope failed \
|
|
--suite redbook \
|
|
--env-url https://localhost:4180 \
|
|
--agent human
|
|
|
|
# 6. Large-scale benchmark runs
|
|
# High-concurrency split run with loop detection and vLLM monitoring.
|
|
python -m bench_env.run \
|
|
--split test \
|
|
--parallel 128 --headless --browsers 16 \
|
|
--isolation pages \
|
|
--env-url https://localhost:4180 \
|
|
--model-base-url http://localhost:8006/v1 \
|
|
--model-name qwen3-vl-4b-10s \
|
|
--proxy http://127.0.0.1:7890 \
|
|
--agent generic_v2 \
|
|
--loop-detect 10 \
|
|
--runs-dir runs/new_qwen4b_10s \
|
|
--monitor
|
|
|
|
# Multi-process variant
|
|
python -m bench_env.run \
|
|
--split test \
|
|
--parallel 64 --headless --browsers 16 --processes 16 \
|
|
--isolation pages \
|
|
--env-url https://localhost:4180 \
|
|
--model-base-url http://localhost:8006/v1 \
|
|
--model-name qwen3-vl-4b-10s \
|
|
--proxy http://127.0.0.1:7890 \
|
|
--agent generic_v2 \
|
|
--loop-detect 10 \
|
|
--runs-dir runs/new_qwen4b_10s_16procs_64envs_16browsers_pages \
|
|
--monitor |