wehub-resource-sync
6ede33ccdb
Build and Push Docker Images / create_manifest (web, surfsense-web, , cpu) (push) Has been cancelled
Build and Push Docker Images / finalize_release (push) Has been cancelled
Obsidian Plugin Lint / lint (push) Has been cancelled
Build and Push Docker Images / build (./surfsense_web, cpu, ./surfsense_web/Dockerfile, web, surfsense-web, ubuntu-24.04-arm, linux/arm64, arm64, , runner, false, cpu) (push) Has been cancelled
Build and Push Docker Images / build (./surfsense_web, cpu, ./surfsense_web/Dockerfile, web, surfsense-web, ubuntu-latest, linux/amd64, amd64, , runner, false, cpu) (push) Has been cancelled
Build and Push Docker Images / compute_version (push) Has been cancelled
Build and Push Docker Images / build (./surfsense_backend, cpu, ./surfsense_backend/Dockerfile, backend, surfsense-backend, ubuntu-24.04-arm, linux/arm64, arm64, , production, false, cpu) (push) Has been cancelled
Build and Push Docker Images / build (./surfsense_backend, cpu, ./surfsense_backend/Dockerfile, backend, surfsense-backend, ubuntu-latest, linux/amd64, amd64, , production, false, cpu) (push) Has been cancelled
Build and Push Docker Images / build (./surfsense_backend, cu126, ./surfsense_backend/Dockerfile, backend, surfsense-backend, ubuntu-24.04-arm, linux/arm64, arm64, -cuda126, production, true, cuda126) (push) Has been cancelled
Build and Push Docker Images / build (./surfsense_backend, cu126, ./surfsense_backend/Dockerfile, backend, surfsense-backend, ubuntu-latest, linux/amd64, amd64, -cuda126, production, true, cuda126) (push) Has been cancelled
Build and Push Docker Images / build (./surfsense_backend, cu128, ./surfsense_backend/Dockerfile, backend, surfsense-backend, ubuntu-24.04-arm, linux/arm64, arm64, -cuda, production, true, cuda) (push) Has been cancelled
Build and Push Docker Images / build (./surfsense_backend, cu128, ./surfsense_backend/Dockerfile, backend, surfsense-backend, ubuntu-latest, linux/amd64, amd64, -cuda, production, true, cuda) (push) Has been cancelled
Build and Push Docker Images / verify_digests (push) Has been cancelled
Build and Push Docker Images / create_manifest (backend, surfsense-backend, , cpu) (push) Has been cancelled
Build and Push Docker Images / create_manifest (backend, surfsense-backend, -cuda, cuda) (push) Has been cancelled
Build and Push Docker Images / create_manifest (backend, surfsense-backend, -cuda126, cuda126) (push) Has been cancelled
69 lines
2.3 KiB
Python
69 lines
2.3 KiB
Python
"""Tests for the shared scenario formatter used in head-to-head reports."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from surfsense_evals.core.scenarios import format_scenario_md
|
|
|
|
|
|
def test_head_to_head_renders_both_arms_same_slug():
|
|
extra = {
|
|
"scenario": "head-to-head",
|
|
"provider_model": "anthropic/claude-sonnet-4.5",
|
|
}
|
|
line = format_scenario_md(extra)
|
|
assert "head-to-head" in line
|
|
assert "anthropic/claude-sonnet-4.5" in line
|
|
|
|
|
|
def test_head_to_head_includes_vision_slug_when_recorded():
|
|
extra = {
|
|
"scenario": "head-to-head",
|
|
"provider_model": "anthropic/claude-sonnet-4.5",
|
|
"vision_provider_model": "anthropic/claude-sonnet-4.5",
|
|
}
|
|
line = format_scenario_md(extra)
|
|
assert "ingest VLM" in line
|
|
assert "claude-sonnet-4.5" in line
|
|
|
|
|
|
def test_symmetric_cheap_calls_out_native_arm_disadvantage():
|
|
extra = {
|
|
"scenario": "symmetric-cheap",
|
|
"provider_model": "openai/gpt-5.4-mini",
|
|
"vision_provider_model": "anthropic/claude-sonnet-4.5",
|
|
}
|
|
line = format_scenario_md(extra)
|
|
assert "**symmetric-cheap**" in line
|
|
assert "gpt-5.4-mini" in line
|
|
# The "structurally loses" disclaimer must be there so reviewers
|
|
# don't read this as a fair comparison.
|
|
assert "structurally loses" in line.lower() or "structurally_loses" in line.lower()
|
|
|
|
|
|
def test_cost_arbitrage_distinguishes_native_and_surfsense_slugs():
|
|
extra = {
|
|
"scenario": "cost-arbitrage",
|
|
"provider_model": "openai/gpt-5.4-mini",
|
|
"native_arm_model": "anthropic/claude-sonnet-4.5",
|
|
"vision_provider_model": "anthropic/claude-sonnet-4.5",
|
|
}
|
|
line = format_scenario_md(extra)
|
|
assert "**cost-arbitrage**" in line
|
|
# Both slugs surface; reader can see the asymmetry at a glance.
|
|
assert "anthropic/claude-sonnet-4.5" in line
|
|
assert "openai/gpt-5.4-mini" in line
|
|
assert "fraction of the per-query cost" in line
|
|
|
|
|
|
def test_legacy_artifact_without_scenario_renders_as_head_to_head():
|
|
"""Old run_artifact.json files don't have ``scenario`` — must still render."""
|
|
|
|
extra = {"provider_model": "anthropic/claude-sonnet-4.5"}
|
|
line = format_scenario_md(extra)
|
|
assert "head-to-head" in line
|
|
|
|
|
|
def test_none_extra_does_not_crash():
|
|
line = format_scenario_md(None)
|
|
assert "head-to-head" in line
|