chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1 @@
|
||||
"""Repository-local helper scripts used by tests and smoke harnesses."""
|
||||
@@ -0,0 +1,883 @@
|
||||
"""One-shot generator: emit the dynamic-shot meta-short-drama SKILL.md.
|
||||
|
||||
Hand-writing 10 shot slots × 6 step types each is error-prone. This
|
||||
script composes the per-shot YAML blocks from a template and prints
|
||||
the full SKILL.md to stdout. Pipe to the bundled SKILL.md path:
|
||||
|
||||
python scripts/_gen_meta_short_drama.py > \
|
||||
src/opensquilla/skills/bundled/meta-short-drama/SKILL.md
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
MAX_SHOTS = 10 # 1..MAX_SHOTS slots emitted in the DAG
|
||||
SLUG_TMPL = "{{ inputs.workspace_dir }}/meta_short_drama/{{ inputs.user_message | slugify | truncate(40) }}"
|
||||
|
||||
HEAD = '''---
|
||||
name: meta-short-drama
|
||||
description: "Use this meta-skill instead of answering directly when the current user asks to generate an AI short-drama or 短剧 from a topic. The workflow infers render style, character identity, and shot count (1-10, default 5) from the request (filling in conservative defaults when missing), drafts a strict shot-by-shot shooting script, pauses for one free-form review (the user can approve, adjust render style / character / shot count / shot details, or cancel in plain language), optionally re-drafts the script with the user's adjustments, generates one universal full-cast identity-reference image plus per-shot composition images, then per-shot video clips (each video anchored to BOTH the universal reference image and its own composition image so the character identity AND scene layout stay consistent), bookends them with a title card and an ending card, burns subtitles in the user's language, and saves the script alongside the final MP4. Do not use it for slide decks, document-decision analysis, single-image generation, isolated script writing, or pasted historical short-drama examples."
|
||||
kind: meta
|
||||
meta_priority: 75
|
||||
always: false
|
||||
final_text_mode: "step:deliver"
|
||||
triggers:
|
||||
- "生成短剧"
|
||||
- "生成一个短剧"
|
||||
- "生成一段短剧"
|
||||
- "做一个AI短剧"
|
||||
- "帮我做一个短剧"
|
||||
- "三分镜短剧"
|
||||
- "短视频分镜成片"
|
||||
- "分镜成片"
|
||||
- "generate a short drama"
|
||||
- "generate short drama"
|
||||
- "make a short drama from"
|
||||
- "topic to short drama mp4"
|
||||
- "shot list to final mp4"
|
||||
provenance:
|
||||
origin: opensquilla-original
|
||||
license: Apache-2.0
|
||||
metadata:
|
||||
opensquilla:
|
||||
risk: high
|
||||
capabilities: [network-read, filesystem-write, process-control]
|
||||
composition_skills:
|
||||
- ai-video-script
|
||||
- nano-banana-pro
|
||||
- seedance-2-prompt
|
||||
- video-still-animator
|
||||
- video-merger
|
||||
- srt-from-script
|
||||
- subtitle-burner
|
||||
- title-card-image
|
||||
- text-file-read
|
||||
composition:
|
||||
steps:
|
||||
# =========================================================================
|
||||
# 1. Best-effort intake — extract RENDER_STYLE / IDENTITY_ANCHOR / N_SHOTS
|
||||
# from the user message, or fill in conservative defaults. Never asks
|
||||
# the user here; the user gets one combined chance to adjust after
|
||||
# seeing the actual script in step 3.
|
||||
# =========================================================================
|
||||
- id: intake_extract
|
||||
kind: llm_chat
|
||||
with:
|
||||
system: "Extract or invent a short-drama intake contract. Match the user's language for RENDER_STYLE / IDENTITY_ANCHOR. Be conservative — pick safe defaults rather than asking the user."
|
||||
task: |
|
||||
Read the request and emit exactly this 7-line block, in this
|
||||
order, with no extra commentary:
|
||||
|
||||
TOPIC: <one short line — the actual story/product topic>
|
||||
RENDER_STYLE: <render aesthetic, one line in user's language>
|
||||
AUTO_FILLED_RENDER_STYLE: <yes|no>
|
||||
IDENTITY_ANCHOR: <one line in user's language describing main character(s)>
|
||||
AUTO_FILLED_IDENTITY_ANCHOR: <yes|no>
|
||||
N_SHOTS: <integer 1..10, default 5>
|
||||
AUTO_FILLED_N_SHOTS: <yes|no>
|
||||
|
||||
Rules:
|
||||
- Detect dominant language of the request. Use that language for
|
||||
RENDER_STYLE and IDENTITY_ANCHOR. Downstream models accept
|
||||
Chinese natively (seedance is Chinese-first).
|
||||
- If user named a render style verbatim → copy it, AUTO_FILLED_RENDER_STYLE: no.
|
||||
- Else INFER a render style from the TOPIC's genre, era, and
|
||||
tone — DO NOT default to anime. Pick whichever of these
|
||||
best fits the story you just read; fall through to a fresh
|
||||
descriptor if none match exactly. Use the user's language.
|
||||
* 现代职场 / 都市爽剧 / 商战 / 反转 / corporate drama →
|
||||
电影级写实, 真实摄影, 戏剧化强光对比, 高对比度色调
|
||||
/ Cinematic realism, dramatic high-contrast lighting
|
||||
* 古风 / 武侠 / 仙侠 / 宫廷 / wuxia / xianxia →
|
||||
水墨风, 中国传统工笔画, 柔和留白构图
|
||||
/ Ink-wash painting, traditional Chinese gongbi style
|
||||
* 校园 / 青春 / 恋爱 / 治愈 / slice-of-life / romance →
|
||||
日系胶片质感, 柔和自然光, 浅景深, 温暖调色
|
||||
/ Japanese film aesthetic, soft natural light, warm grade
|
||||
* 科幻 / 赛博朋克 / 未来 / sci-fi / cyberpunk →
|
||||
赛博朋克霓虹, 体积光雾气, 高对比反射, 未来感
|
||||
/ Cyberpunk neon, volumetric haze, future-noir
|
||||
* 恐怖 / 悬疑 / 惊悚 / horror / thriller / noir →
|
||||
低调照明, 高反差暗调, 电影黑色风格
|
||||
/ Low-key lighting, high-contrast noir, cinematic shadow
|
||||
* 童话 / 绘本 / 儿童 / fairytale / picture-book / kids →
|
||||
水彩绘本插画, 柔和纸面纹理, 暖色调
|
||||
/ Watercolour storybook, soft paper texture, warm palette
|
||||
* 商品 / 广告 / 带货 / product / commercial →
|
||||
影棚布光, 浅景深产品特写, 干净背景
|
||||
/ Studio lighting, hero-product close-up, clean background
|
||||
* 美食 / 烹饪 / food / cooking →
|
||||
顶光美食摄影, 自然质感, 浅景深
|
||||
/ Top-down food photography, natural texture, shallow DOF
|
||||
* 科普 / 教学 / 信息图 / explainer / educational →
|
||||
扁平信息图风格, 简洁配色, 平面构图
|
||||
/ Flat infographic style, clean palette, geometric layout
|
||||
* 卡通 / 动画 / 二次元 / 萌系 — only when the user really
|
||||
wants anime → 2D 动漫插画, 扁平上色, 柔和赛璐璐阴影
|
||||
/ 2D anime illustration, flat cel-shading
|
||||
* none of the above → write ONE descriptive line that
|
||||
matches the topic's mood (NOT anime by default). Examples:
|
||||
documentary realism / oil-painting cinematic / vintage
|
||||
super-8 grain / minimalist black-and-white photography.
|
||||
AUTO_FILLED_RENDER_STYLE: yes
|
||||
- If user described main character(s) with at least
|
||||
ethnicity + age + hair + outfit → summarise ≤40 words,
|
||||
AUTO_FILLED_IDENTITY_ANCHOR: no.
|
||||
- Else invent ONE or TWO original characters fitting the TOPIC.
|
||||
- If user named shot count (3 个分镜 / "5 shots" / etc.) → use it
|
||||
clamped 1..10, AUTO_FILLED_N_SHOTS: no.
|
||||
- Else default N_SHOTS: 5, AUTO_FILLED_N_SHOTS: yes.
|
||||
- Never ask the user a question. The user reviews in step 3.
|
||||
|
||||
User request:
|
||||
{{ inputs.user_message | xml_escape | truncate(1500) }}
|
||||
|
||||
# =========================================================================
|
||||
# 2. Draft the script with whatever values we have. Free (LLM only).
|
||||
# =========================================================================
|
||||
- id: script_draft
|
||||
kind: agent
|
||||
skill: ai-video-script
|
||||
depends_on: [intake_extract]
|
||||
with:
|
||||
task: |
|
||||
Generate a strict-format short-drama shooting script following
|
||||
ai-video-script's SKILL.md OUTPUT FORMAT section. Use the
|
||||
N_SHOTS value from the intake contract below (clamp 1..10).
|
||||
Default DURATION_S total: 50 (~10s per shot for the default 5
|
||||
shots). ASPECT_RATIO: 9:16.
|
||||
|
||||
Output style: plain text only. No emoji, no decorative symbols.
|
||||
|
||||
Language: match the user's request language for every field.
|
||||
Both downstream models accept CJK natively — do NOT translate
|
||||
Chinese stories into English.
|
||||
|
||||
IDENTITY_ANCHOR and RENDER_STYLE below are caller-supplied —
|
||||
paste them byte-for-byte into every shot's IMAGE_PROMPT and
|
||||
VIDEO_PROMPT. Do not paraphrase or invent alternates.
|
||||
|
||||
Intake contract:
|
||||
{{ outputs.intake_extract | truncate(1500) }}
|
||||
|
||||
User original request:
|
||||
{{ inputs.user_message | xml_escape | truncate(1200) }}
|
||||
|
||||
Emit OVERVIEW.IDENTITY_ANCHOR, OVERVIEW.RENDER_STYLE, and
|
||||
OVERVIEW.N_SHOTS lines so downstream steps can re-extract them.
|
||||
|
||||
# =========================================================================
|
||||
# 2b. Persist the draft to disk BEFORE the review pause so the user
|
||||
# can hand-edit the file directly while reviewing. The next step
|
||||
# reads it back so manual edits propagate even when the user's
|
||||
# reply doesn't mention them.
|
||||
# =========================================================================
|
||||
- id: script_save_draft
|
||||
kind: tool_call
|
||||
tool: write_file
|
||||
tool_allowlist: [write_file]
|
||||
depends_on: [script_draft]
|
||||
tool_args:
|
||||
path: "<<SLUG>>/script.txt"
|
||||
content: "{{ outputs.script_draft }}"
|
||||
|
||||
# =========================================================================
|
||||
# 3. ONE combined review gate — free-form. The user can approve,
|
||||
# rewrite anything, or cancel.
|
||||
# =========================================================================
|
||||
- id: review_gate
|
||||
kind: user_input
|
||||
depends_on: [script_save_draft, script_draft, intake_extract]
|
||||
clarify:
|
||||
mode: form
|
||||
intro: |
|
||||
脚本就绪。下面是脚本预览 + 我对风格/角色/分镜数做的假设
|
||||
(标 AUTO_FILLED: yes 的项是我替你填的,你可以改)。
|
||||
|
||||
脚本草稿已存到本次运行目录的 script.txt —— 想直接改文件也行,
|
||||
下一步会重新读盘,你的手动编辑会一起带进去。
|
||||
|
||||
你怎么回都行 —— 不用按固定格式:
|
||||
- 满意就直接说 "ok" / "继续" / "proceed"
|
||||
- 想换风格 → 写一句新的 RENDER_STYLE
|
||||
- 想换角色 → 写新的 IDENTITY_ANCHOR
|
||||
- 想改分镜数 → 直接说 "5 个分镜" / "改成 7 镜头"
|
||||
- 想改某镜内容 → 直接说 "镜头2节奏快点" / "shot 3 换成屋顶场景"
|
||||
- 不想做了 → 说 "取消" / "cancel" / "停"
|
||||
|
||||
预估成本(选继续才会发生):
|
||||
- N 张镜头图 + 1 张全角色参考图 (nano-banana-pro) ≈ N × $0.05 + $0.05-$0.10
|
||||
- N 段视频 (seedance-2.0) ≈ $0.15/s × 总时长
|
||||
(脚本里每镜 DURATION_S 决定时长)
|
||||
- 封面 + 结尾卡 (本地 Pillow + ffmpeg,免费)
|
||||
- ffmpeg 拼接 + 烧字幕
|
||||
合计随 N_SHOTS 与总时长缩放。
|
||||
|
||||
=== 我做的假设 ===
|
||||
{{ outputs.intake_extract | truncate(800) }}
|
||||
|
||||
=== 脚本草稿 ===
|
||||
{{ outputs.script_draft | truncate(3500) }}
|
||||
nl_extract: true
|
||||
fields:
|
||||
- name: review
|
||||
type: string
|
||||
required: true
|
||||
prompt: |
|
||||
用户对脚本草稿的整段回复 — 直接把用户说的所有文字原样
|
||||
放进这个字段,不要总结、不要重写、不要解释。这是一个
|
||||
catch-all 字段:任何同意/拒绝/修改意见/吐槽/闲聊都属于这里。
|
||||
The user's verbatim reply about the script draft. Copy the
|
||||
user's entire reply text into this single field — do not
|
||||
summarise, paraphrase, translate, or split it. This is a
|
||||
catch-all: approvals, rejections, edits, off-topic remarks
|
||||
all belong here. If the user's reply is empty or pure
|
||||
whitespace, emit "(empty)" so the field always has a value.
|
||||
max_chars: 4000
|
||||
cancel_keywords: ["cancel", "取消", "算了", "停止", "stop", "abort"]
|
||||
timeout_hours: 24
|
||||
|
||||
# =========================================================================
|
||||
# 4. Parse the free-form review.
|
||||
# =========================================================================
|
||||
- id: review_normalize
|
||||
kind: llm_chat
|
||||
depends_on: [review_gate]
|
||||
with:
|
||||
system: "Emit a strict 6-line block. No commentary outside it."
|
||||
task: |
|
||||
Parse the user's free-form review of the script draft and emit
|
||||
exactly this block:
|
||||
|
||||
DECISION: <proceed|cancel>
|
||||
HAS_OVERRIDES: <yes|no>
|
||||
NEW_RENDER_STYLE: <new one-line value, or "unchanged">
|
||||
NEW_IDENTITY_ANCHOR: <new one-line value, or "unchanged">
|
||||
NEW_N_SHOTS: <integer 1..10, or "unchanged">
|
||||
NEW_NOTES: <any other adjustments to story / shots / voiceover, or "unchanged">
|
||||
|
||||
Rules:
|
||||
- DECISION: cancel only on explicit cancel/取消/算了/停 words.
|
||||
- DECISION: proceed otherwise (approvals AND adjustments).
|
||||
- HAS_OVERRIDES: yes if ANY of NEW_RENDER_STYLE /
|
||||
NEW_IDENTITY_ANCHOR / NEW_N_SHOTS / NEW_NOTES differs from
|
||||
"unchanged".
|
||||
- NEW_RENDER_STYLE / NEW_IDENTITY_ANCHOR / NEW_NOTES: use the
|
||||
same language as the user's reply.
|
||||
- NEW_N_SHOTS: extract integer (e.g. "改成 5 镜头" → 5).
|
||||
Clamp 1..10. Else "unchanged".
|
||||
|
||||
Free-form user review:
|
||||
{{ inputs.get('collected', {}).get('review_gate', {}) | tojson | truncate(2200) }}
|
||||
|
||||
Original assumptions (for delta detection):
|
||||
{{ outputs.intake_extract | truncate(800) }}
|
||||
|
||||
# =========================================================================
|
||||
# 4b. Re-read the script from disk so any hand-edits the user made to
|
||||
# script.txt during the review pause are honoured by the redraft
|
||||
# step. When the user didn't touch the file this is just an echo
|
||||
# of the original draft.
|
||||
# =========================================================================
|
||||
- id: script_reread
|
||||
kind: skill_exec
|
||||
skill: text-file-read
|
||||
depends_on: [review_gate, script_save_draft]
|
||||
with:
|
||||
input: "<<SLUG>>/script.txt"
|
||||
|
||||
# =========================================================================
|
||||
# 5. Re-draft script when the user supplied adjustments. Free.
|
||||
# =========================================================================
|
||||
- id: script_revised
|
||||
kind: agent
|
||||
skill: ai-video-script
|
||||
depends_on: [review_normalize, script_reread]
|
||||
when: "'DECISION: proceed' in outputs.review_normalize and 'HAS_OVERRIDES: yes' in outputs.review_normalize"
|
||||
with:
|
||||
task: |
|
||||
Re-draft the script applying the user's overrides. Keep the
|
||||
same OUTPUT FORMAT as ai-video-script's SKILL.md. If NEW_N_SHOTS
|
||||
is an integer, use exactly that many shot blocks (1..10).
|
||||
Otherwise keep the original N_SHOTS.
|
||||
|
||||
Output style: plain text only. No emoji.
|
||||
Language: keep the user's original request language.
|
||||
|
||||
Apply overrides in priority: NEW_NOTES → NEW_N_SHOTS →
|
||||
NEW_RENDER_STYLE → NEW_IDENTITY_ANCHOR. "unchanged" fields
|
||||
inherit from the previous script verbatim.
|
||||
|
||||
Previous script (re-read from disk — if the user hand-edited
|
||||
script.txt during review, those edits are already baked in
|
||||
here, so preserve them):
|
||||
{{ outputs.script_reread | truncate(8000) }}
|
||||
|
||||
Parsed overrides:
|
||||
{{ outputs.review_normalize | truncate(1500) }}
|
||||
|
||||
User original request:
|
||||
{{ inputs.user_message | xml_escape | truncate(800) }}
|
||||
|
||||
# =========================================================================
|
||||
# 6. Pick the final script everyone downstream reads.
|
||||
# =========================================================================
|
||||
- id: final_script
|
||||
kind: llm_chat
|
||||
depends_on: [review_normalize, script_reread, script_revised]
|
||||
with:
|
||||
system: "Echo one of two inputs verbatim. No commentary. No new content."
|
||||
task: |
|
||||
If a revised script block is present below, echo it verbatim.
|
||||
Otherwise echo the re-read script verbatim (this preserves any
|
||||
hand-edits the user made to script.txt during review).
|
||||
|
||||
REVISED (may be empty):
|
||||
{{ outputs.get('script_revised', '') | truncate(8000) }}
|
||||
|
||||
RE-READ FROM DISK:
|
||||
{{ outputs.script_reread | truncate(8000) }}
|
||||
|
||||
# =========================================================================
|
||||
# 7. Save the final script to disk (overwrites the draft so the file
|
||||
# on disk always reflects the post-review canonical script —
|
||||
# important when the LLM produced a revision the user didn't write
|
||||
# by hand).
|
||||
# =========================================================================
|
||||
- id: script_save
|
||||
kind: tool_call
|
||||
tool: write_file
|
||||
tool_allowlist: [write_file]
|
||||
depends_on: [final_script]
|
||||
tool_args:
|
||||
path: "<<SLUG>>/script.txt"
|
||||
content: "{{ outputs.final_script }}"
|
||||
|
||||
# =========================================================================
|
||||
# 8. Title / subtitle / ending text extracts (cheap llm_chat).
|
||||
# =========================================================================
|
||||
- id: title_extract
|
||||
kind: llm_chat
|
||||
depends_on: [final_script]
|
||||
with:
|
||||
system: "Return one line of text. No quotes, no prefix, no commentary."
|
||||
task: |
|
||||
From the script, output exactly the value after "TITLE:"
|
||||
inside the "=== OVERVIEW ===" block. Single line.
|
||||
|
||||
Script:
|
||||
{{ outputs.final_script | truncate(8000) }}
|
||||
|
||||
- id: subtitle_extract
|
||||
kind: llm_chat
|
||||
depends_on: [final_script]
|
||||
with:
|
||||
system: "Return one line of text. No quotes, no prefix, no commentary."
|
||||
task: |
|
||||
Compose a short subtitle for the cover card describing this
|
||||
drama in 5-12 characters (or 2-4 English words). Match the
|
||||
script's language. Examples:
|
||||
Chinese script → "AI 短剧 · 30 秒"
|
||||
English script → "AI Short Drama · 30s"
|
||||
|
||||
Script (read OVERVIEW.TITLE / DURATION_S / AUDIENCE):
|
||||
{{ outputs.final_script | truncate(2000) }}
|
||||
|
||||
- id: ending_text_extract
|
||||
kind: llm_chat
|
||||
depends_on: [final_script]
|
||||
with:
|
||||
system: "Return one line of text. No quotes, no prefix, no commentary."
|
||||
task: |
|
||||
Output the appropriate ending-card text. Single line, no commentary.
|
||||
Chinese script → 完
|
||||
English script → THE END
|
||||
Other languages → THE END
|
||||
|
||||
Script (sample to detect language):
|
||||
{{ outputs.final_script | truncate(1500) }}
|
||||
|
||||
# =========================================================================
|
||||
# 8b. Universal identity-reference image. One full-cast neutral lineup
|
||||
# PNG that every shot's video step uses as the IDENTITY anchor
|
||||
# (input_reference). Each shot ALSO passes its own composition
|
||||
# PNG (N_shot.png) as a second reference. Two-anchor model:
|
||||
# slot 1 (reference.png) → who the characters look like
|
||||
# slot 2 (N_shot.png) → how the scene is laid out
|
||||
# =========================================================================
|
||||
- id: reference_prompt_extract
|
||||
kind: llm_chat
|
||||
depends_on: [final_script]
|
||||
with:
|
||||
system: "Return one line of text. No quotes, no prefix, no commentary."
|
||||
task: |
|
||||
Build a single-line image prompt for a full-cast identity
|
||||
reference card. The picture must show EVERY named character
|
||||
that appears in ANY shot of the script (NOT just the
|
||||
OVERVIEW.IDENTITY_ANCHOR anchors — supporting cast, cameo
|
||||
characters, anyone the script mentions by name in any SHOT
|
||||
block also belongs here), standing together in a neutral
|
||||
lineup against a neutral backdrop. The downstream video model
|
||||
uses this image as the universal identity anchor for every
|
||||
shot.
|
||||
|
||||
Procedure (do these silently in your head; only emit the final
|
||||
single-line prompt):
|
||||
|
||||
1. Read the entire script. Enumerate every distinct named
|
||||
character that appears in ANY SHOT_N block's IMAGE_PROMPT
|
||||
or VIDEO_PROMPT. Include characters who appear in only one
|
||||
shot. Deduplicate by name. Let N be the count.
|
||||
2. For each character, write the most complete canonical
|
||||
attribute string the script gives them (name, age,
|
||||
ethnicity, hair, outfit, distinguishing accessory). Pull
|
||||
missing fields from OVERVIEW.IDENTITY_ANCHOR if needed.
|
||||
3. Compose the final prompt as a single line in this exact
|
||||
order:
|
||||
|
||||
<char 1 description>; <char 2 description>; ...; <char N description>, ALL <N> characters standing side by side in a horizontal full-body group lineup, every character clearly visible from head to toe, evenly spaced across frame, wide-angle group photo, neutral studio lighting, neutral light grey backdrop, no props, no background scene, group portrait composition, <OVERVIEW.RENDER_STYLE verbatim>, --ar 9:16
|
||||
|
||||
- Use ; (semicolon) BETWEEN characters, exactly as in the
|
||||
examples above.
|
||||
- State the integer N explicitly inside "ALL <N> characters".
|
||||
- If N = 1, still say "ALL 1 character" and drop the
|
||||
"side by side / horizontal lineup" phrasing — write
|
||||
"single-character full-body portrait" instead.
|
||||
|
||||
Output a single line. No quotes. No commentary outside the
|
||||
prompt itself.
|
||||
|
||||
Script (READ THE FULL SCRIPT, including every SHOT_N block,
|
||||
not just OVERVIEW):
|
||||
{{ outputs.final_script | truncate(8000) }}
|
||||
|
||||
- id: reference_image
|
||||
kind: skill_exec
|
||||
skill: nano-banana-pro
|
||||
depends_on: [reference_prompt_extract, review_normalize]
|
||||
when: "'DECISION: proceed' in outputs.review_normalize"
|
||||
with:
|
||||
prompt: "{{ outputs.reference_prompt_extract | truncate(800) }}"
|
||||
filename: "<<SLUG>>/reference.png"
|
||||
aspect_ratio: "9:16"
|
||||
image_size: "1K"
|
||||
# Use 3-pro as primary here: this image runs ONCE per drama and
|
||||
# has to render every cast member visibly, which 3-pro handles
|
||||
# better than 3.1-flash on dense multi-subject prompts. Per-shot
|
||||
# images keep 3.1-flash for cost.
|
||||
model: "google/gemini-3-pro-image-preview"
|
||||
max_retries: 1
|
||||
fallback_model: "google/gemini-3.1-flash-image-preview"
|
||||
placeholder_on_fail: "yes"
|
||||
|
||||
# =========================================================================
|
||||
# 9. Cover card image + 2s video (gated on proceed).
|
||||
# =========================================================================
|
||||
- id: cover_image
|
||||
kind: skill_exec
|
||||
skill: title-card-image
|
||||
depends_on: [title_extract, subtitle_extract, review_normalize]
|
||||
when: "'DECISION: proceed' in outputs.review_normalize"
|
||||
with:
|
||||
text: "{{ outputs.title_extract | truncate(40) }}"
|
||||
subtitle: "{{ outputs.subtitle_extract | truncate(40) }}"
|
||||
output: "<<SLUG>>/0_cover.png"
|
||||
background: "#101018"
|
||||
text_color: "#ffffff"
|
||||
font_size: 80
|
||||
subtitle_size: 32
|
||||
width: 720
|
||||
height: 1280
|
||||
|
||||
- id: cover_video
|
||||
kind: skill_exec
|
||||
skill: video-still-animator
|
||||
depends_on: [cover_image, review_normalize]
|
||||
when: "'DECISION: proceed' in outputs.review_normalize"
|
||||
with:
|
||||
input_image: "<<SLUG>>/0_cover.png"
|
||||
output_path: "<<SLUG>>/0_cover.mp4"
|
||||
duration: 2
|
||||
width: 720
|
||||
height: 1280
|
||||
fps: 24
|
||||
zoom_rate: 0.0008
|
||||
'''
|
||||
|
||||
# Per-shot extract block template (img_prompt, vid_prompt, duration).
|
||||
EXTRACT_TMPL = '''
|
||||
# ---- SHOT_{N} extracts (run even if shot doesn't exist; returns sentinel) ----
|
||||
- id: shot{N}_img_prompt
|
||||
kind: llm_chat
|
||||
depends_on: [final_script]
|
||||
with:
|
||||
system: "Return one line of text. No quotes, no prefix, no commentary."
|
||||
task: |
|
||||
If the script contains a "=== SHOT_{N} ===" block:
|
||||
output exactly the value after "IMAGE_PROMPT:" inside that block.
|
||||
Single line, no quotes, no label.
|
||||
If it does NOT (because N_SHOTS < {N}):
|
||||
output exactly the literal sentinel: __SHOT_ABSENT__
|
||||
|
||||
Script:
|
||||
{{{{ outputs.final_script | truncate(8000) }}}}
|
||||
|
||||
- id: shot{N}_vid_prompt
|
||||
kind: llm_chat
|
||||
depends_on: [final_script]
|
||||
with:
|
||||
system: "Return one line of text. No quotes, no prefix, no commentary."
|
||||
task: |
|
||||
If the script contains a "=== SHOT_{N} ===" block:
|
||||
output exactly the value after "VIDEO_PROMPT:" inside that block.
|
||||
Single line.
|
||||
If it does NOT: output exactly: __SHOT_ABSENT__
|
||||
|
||||
Script:
|
||||
{{{{ outputs.final_script | truncate(8000) }}}}
|
||||
|
||||
- id: shot{N}_duration
|
||||
kind: llm_chat
|
||||
depends_on: [final_script]
|
||||
with:
|
||||
system: "Return exactly one integer or the literal __SHOT_ABSENT__. No commentary."
|
||||
task: |
|
||||
If the script contains a "=== SHOT_{N} ===" block:
|
||||
output exactly the integer after "DURATION_S:" inside that
|
||||
block, clamped to [3, 15]. Digits only, no units.
|
||||
If it does NOT: output exactly: __SHOT_ABSENT__
|
||||
|
||||
Script:
|
||||
{{{{ outputs.final_script | truncate(8000) }}}}
|
||||
'''
|
||||
|
||||
# Per-shot image + video + fallback template.
|
||||
EXEC_TMPL = '''
|
||||
# ---- SHOT_{N} image / video / fallback ----
|
||||
- id: shot{N}_image
|
||||
kind: skill_exec
|
||||
skill: nano-banana-pro
|
||||
depends_on: [shot{N}_img_prompt, review_normalize]
|
||||
when: "'DECISION: proceed' in outputs.review_normalize and '__SHOT_ABSENT__' not in outputs.shot{N}_img_prompt"
|
||||
with:
|
||||
prompt: "{{{{ outputs.shot{N}_img_prompt | truncate(800) }}}}"
|
||||
filename: "<<SLUG>>/{N}_shot.png"
|
||||
aspect_ratio: "9:16"
|
||||
image_size: "1K"
|
||||
max_retries: 1
|
||||
fallback_model: "google/gemini-3-pro-image-preview"
|
||||
placeholder_on_fail: "yes"
|
||||
|
||||
- id: shot{N}_video
|
||||
kind: skill_exec
|
||||
skill: seedance-2-prompt
|
||||
depends_on: [shot{N}_vid_prompt, shot{N}_duration, reference_image, shot{N}_image, review_normalize]
|
||||
when: "'DECISION: proceed' in outputs.review_normalize and '__SHOT_ABSENT__' not in outputs.shot{N}_vid_prompt"
|
||||
on_failure: shot{N}_video_fallback
|
||||
with:
|
||||
# Prepend Assets Mapping so seedance knows the role of each
|
||||
# input_reference image. Mirrors the upstream JiMeng prompt
|
||||
# convention (see references/recipes.md "Mode: All-Reference"):
|
||||
# @image1 / reference[1] = identity anchor (full-cast lineup)
|
||||
# @image2 / reference[2] = scene composition (this shot)
|
||||
# Keeping the preamble in English even when the shot directive
|
||||
# is Chinese — seedance parses English instruction prefixes
|
||||
# reliably regardless of the user-content language.
|
||||
prompt: "Mode: All-Reference. Assets Mapping: reference[1] is the full-cast identity anchor (USE strictly for character likeness, faces, hair, skin tone, outfits, and accessories — keep these byte-identical to the reference across cuts). reference[2] is THIS shot's scene composition reference (USE for camera angle, framing, character blocking, prop placement, and background layout). Shot directive: {{{{ outputs.shot{N}_vid_prompt | truncate(700) }}}}"
|
||||
filename: "<<SLUG>>/{N}_shot.mp4"
|
||||
input_image: ""
|
||||
input_reference: "<<SLUG>>/reference.png"
|
||||
input_reference_2: "<<SLUG>>/{N}_shot.png"
|
||||
aspect_ratio: "9:16"
|
||||
# `| int(5)` parses the duration extract as an integer, falling
|
||||
# back to 5 if the LLM emitted anything non-numeric (sentinel
|
||||
# __SHOT_ABSENT__, units like "10s", chain-of-thought text). A
|
||||
# raw truncate would slice "__SHOT_ABSENT__" to "__S" and crash
|
||||
# the downstream CLI's duration validator.
|
||||
duration: "{{{{ outputs.shot{N}_duration | int(5) }}}}"
|
||||
model: "bytedance/seedance-2.0"
|
||||
max_retries: 2
|
||||
|
||||
- id: shot{N}_video_fallback
|
||||
kind: skill_exec
|
||||
skill: video-still-animator
|
||||
with:
|
||||
input_image: "<<SLUG>>/{N}_shot.png"
|
||||
output_path: "<<SLUG>>/{N}_shot.mp4"
|
||||
duration: "{{{{ outputs.shot{N}_duration | int(5) }}}}"
|
||||
width: 720
|
||||
height: 1280
|
||||
fps: 24
|
||||
'''
|
||||
|
||||
# Tail blocks (ending, merge, subtitles, deliver).
|
||||
TAIL = '''
|
||||
# =========================================================================
|
||||
# Ending card image + 1.5s video.
|
||||
# =========================================================================
|
||||
- id: ending_image
|
||||
kind: skill_exec
|
||||
skill: title-card-image
|
||||
depends_on: [ending_text_extract, review_normalize]
|
||||
when: "'DECISION: proceed' in outputs.review_normalize"
|
||||
with:
|
||||
text: "{{ outputs.ending_text_extract | truncate(20) }}"
|
||||
subtitle: ""
|
||||
output: "<<SLUG>>/99_ending.png"
|
||||
background: "#0a0a10"
|
||||
text_color: "#e0e0e8"
|
||||
font_size: 96
|
||||
width: 720
|
||||
height: 1280
|
||||
|
||||
- id: ending_video
|
||||
kind: skill_exec
|
||||
skill: video-still-animator
|
||||
depends_on: [ending_image, review_normalize]
|
||||
when: "'DECISION: proceed' in outputs.review_normalize"
|
||||
with:
|
||||
input_image: "<<SLUG>>/99_ending.png"
|
||||
output_path: "<<SLUG>>/99_ending.mp4"
|
||||
duration: 2
|
||||
width: 720
|
||||
height: 1280
|
||||
fps: 24
|
||||
zoom_rate: 0.0005
|
||||
|
||||
# =========================================================================
|
||||
# Stitch cover + shots(1..10 that exist) + ending. video-merger sorts
|
||||
# numeric prefix; 0_cover < 1..10_shot < 99_ending.
|
||||
# =========================================================================
|
||||
- id: merge
|
||||
kind: skill_exec
|
||||
skill: video-merger
|
||||
depends_on:
|
||||
- cover_video
|
||||
- shot1_video
|
||||
- shot2_video
|
||||
- shot3_video
|
||||
- shot4_video
|
||||
- shot5_video
|
||||
- shot6_video
|
||||
- shot7_video
|
||||
- shot8_video
|
||||
- shot9_video
|
||||
- shot10_video
|
||||
- ending_video
|
||||
- review_normalize
|
||||
when: "'DECISION: proceed' in outputs.review_normalize"
|
||||
with:
|
||||
input_dir: "<<SLUG>>"
|
||||
output_path: "<<SLUG>>/final.mp4"
|
||||
mode: "full"
|
||||
transition: 0.5
|
||||
fps: 24
|
||||
crf: 22
|
||||
preset: "medium"
|
||||
|
||||
- id: subtitles_srt
|
||||
kind: skill_exec
|
||||
skill: srt-from-script
|
||||
depends_on: [final_script, review_normalize]
|
||||
when: "'DECISION: proceed' in outputs.review_normalize"
|
||||
with:
|
||||
script: "{{ outputs.final_script }}"
|
||||
output_path: "<<SLUG>>/subs.srt"
|
||||
gap_ms: 200
|
||||
leading_offset_ms: 2000
|
||||
|
||||
- id: subtitled_final
|
||||
kind: skill_exec
|
||||
skill: subtitle-burner
|
||||
depends_on: [merge, subtitles_srt, review_normalize]
|
||||
when: "'DECISION: proceed' in outputs.review_normalize"
|
||||
with:
|
||||
input: "<<SLUG>>/final.mp4"
|
||||
subtitles: "<<SLUG>>/subs.srt"
|
||||
output: "<<SLUG>>/final_subtitled.mp4"
|
||||
font_size: 42
|
||||
margin_v: 80
|
||||
|
||||
- id: deliver
|
||||
kind: llm_chat
|
||||
depends_on: [final_script, review_normalize, script_save]
|
||||
with:
|
||||
system: "Write a concise delivery message in the user's language. No emoji. Branch on DECISION."
|
||||
task: |
|
||||
Compose a 4-10 line summary tailored to the user's decision.
|
||||
|
||||
User original request:
|
||||
{{ inputs.user_message | xml_escape | truncate(400) }}
|
||||
|
||||
Decision marker:
|
||||
{{ outputs.review_normalize | truncate(400) }}
|
||||
|
||||
Final script:
|
||||
{{ outputs.final_script | truncate(2500) }}
|
||||
|
||||
Script saved at:
|
||||
{{ outputs.script_save | truncate(200) }}
|
||||
|
||||
Merge output:
|
||||
{{ outputs.get('merge', '') | truncate(800) }}
|
||||
|
||||
Subtitled-final output:
|
||||
{{ outputs.get('subtitled_final', '') | truncate(800) }}
|
||||
|
||||
Branching rules:
|
||||
- If "DECISION: proceed":
|
||||
* Title (from final_script OVERVIEW.TITLE), shot count, total duration.
|
||||
* Headline path = subtitled_final (the burned-in subtitle MP4).
|
||||
* Also list: un-subtitled merge path, SRT path, script.txt path,
|
||||
folder containing intermediates.
|
||||
* Mention HAS_OVERRIDES if yes.
|
||||
- If "DECISION: cancel":
|
||||
* Acknowledge, note the script was still saved at script_save's
|
||||
path so it's not lost.
|
||||
* Offer to re-trigger.
|
||||
Respond in the same language as the user's original request.
|
||||
---
|
||||
|
||||
# meta-short-drama
|
||||
|
||||
End-to-end short-drama generator with one free-form user-review gate
|
||||
before any paid step. **1-10 shots** (default 5), title card + ending
|
||||
card, in-language burned subtitles, and the generated script is saved
|
||||
to disk regardless of outcome.
|
||||
|
||||
## What it does
|
||||
|
||||
1. **`intake_extract`** scans the user message for RENDER_STYLE,
|
||||
IDENTITY_ANCHOR, and N_SHOTS (1-10). Fills in defaults when missing.
|
||||
2. **`script_draft`** calls `ai-video-script` with the inferred values
|
||||
pasted verbatim into every shot prompt.
|
||||
3. **`review_gate`** — single free-form pause. The user can approve,
|
||||
rewrite render style / character / shot count / shot details, or
|
||||
cancel in plain language.
|
||||
4. **`review_normalize`** parses the free-form reply.
|
||||
5. **`script_revised`** (conditional) redrafts when overrides present.
|
||||
6. **`final_script`** echoes the canonical script.
|
||||
7. **`script_save`** writes `script.txt` to the run folder
|
||||
(always — even on cancel, so the user keeps the draft).
|
||||
8. **`title_extract` / `subtitle_extract` / `ending_text_extract`**
|
||||
pull cover/ending text in the script's language.
|
||||
9. **`cover_image` + `cover_video`** — Pillow title card + 2s Ken-Burns
|
||||
clip (`0_cover.mp4` — sorts first in merge).
|
||||
10. **Per-shot extracts × 10** — for shots 1..10 the LLM emits either
|
||||
the real prompts/duration OR the literal sentinel `__SHOT_ABSENT__`.
|
||||
Image/video steps gate on the sentinel so unused slots stay dormant.
|
||||
11. **Image generation per active shot** — `nano-banana-pro`, retry +
|
||||
fallback model + placeholder PNG (image step never aborts DAG).
|
||||
12. **`reference_prompt_extract` + `reference_image`** — one extra
|
||||
`nano-banana-pro` call produces `reference.png`, a full-cast neutral
|
||||
lineup of every named character on a neutral backdrop. Used as the
|
||||
universal IDENTITY anchor for every shot's seedance call so the
|
||||
character does not drift across cuts (nano-banana would otherwise
|
||||
re-roll subtly different faces per shot).
|
||||
13. **Video generation per active shot** — `seedance-2.0`, retry twice;
|
||||
on persistent refusal the Ken-Burns substitute fires using the
|
||||
shot's PNG. Each shot passes TWO reference images to seedance,
|
||||
AND the per-shot prompt is wrapped with an explicit "Assets
|
||||
Mapping" preamble in the upstream JiMeng convention so seedance
|
||||
knows the role of each reference:
|
||||
reference[1] = `reference.png` (full-cast identity anchor — used
|
||||
strictly for character likeness / faces / hair /
|
||||
outfits / accessories across all shots)
|
||||
reference[2] = `N_shot.png` (this shot's scene composition
|
||||
reference — used for camera angle, framing,
|
||||
blocking, prop placement, background layout)
|
||||
The Assets Mapping preamble is in English even when the per-shot
|
||||
directive is Chinese — seedance parses English instruction prefixes
|
||||
reliably regardless of the user-content language. Empty / missing
|
||||
references are still filtered before the API call (so direct CLI
|
||||
callers using a single anchor remain backwards-compatible).
|
||||
13. **`ending_image` + `ending_video`** — Pillow "完" / "THE END" card
|
||||
+ 1.5s Ken-Burns clip (`99_ending.mp4` — sorts last).
|
||||
14. **`merge`** — `video-merger` stitches `0_cover` + active shots
|
||||
+ `99_ending` via numeric-prefix sort. ffmpeg cross-fade transitions.
|
||||
15. **`subtitles_srt`** — SRT cues from VOICEOVER per shot, shifted by
|
||||
the 2-second cover duration so cue timing matches the merged
|
||||
timeline.
|
||||
16. **`subtitled_final`** — `subtitle-burner` burns the SRT into
|
||||
`final_subtitled.mp4`.
|
||||
17. **`deliver`** — always runs, branches on DECISION. Lists the saved
|
||||
script path so the user keeps a copy regardless.
|
||||
|
||||
## Outputs
|
||||
|
||||
```
|
||||
<workspace>/meta_short_drama/<slug>/
|
||||
script.txt # full final script (always)
|
||||
reference.png # full-cast identity reference (used by every shot_video)
|
||||
0_cover.png 0_cover.mp4
|
||||
1_shot.png 1_shot.mp4 ┐
|
||||
2_shot.png 2_shot.mp4 ├ only for active shots (1..N_SHOTS)
|
||||
... ┘
|
||||
99_ending.png 99_ending.mp4
|
||||
subs.srt
|
||||
final.mp4 # merged, no subtitles
|
||||
final_subtitled.mp4 # subtitled — the deliverable
|
||||
```
|
||||
|
||||
## Dependencies
|
||||
|
||||
| Skill | Purpose | Models / Tools |
|
||||
|---|---|---|
|
||||
| `ai-video-script` | Structured shot list (1-10 shots) | LLM |
|
||||
| `nano-banana-pro` | Per-shot first-frame PNG | OpenRouter Gemini 3.1 / 3 pro |
|
||||
| `seedance-2-prompt` | Per-shot MP4 | OpenRouter Seedance 2.0 (or Volcengine ARK) |
|
||||
| `video-still-animator` | Ken-Burns fallback / cover & ending clips | ffmpeg ≥ 5.0 |
|
||||
| `video-merger` | Stitch cover + shots + ending | ffmpeg ≥ 5.0 |
|
||||
| `srt-from-script` | VOICEOVER → SRT with cover offset | Python stdlib |
|
||||
| `subtitle-burner` | Burn SRT into MP4 | ffmpeg + libass |
|
||||
| `title-card-image` | Pillow cover + ending PNG cards | Pillow |
|
||||
| (builtin) `write_file` | Save script.txt (no skill needed) | OpenSquilla builtin |
|
||||
| `text-file-read` | Re-read script.txt after review pause | Python stdlib |
|
||||
|
||||
Environment:
|
||||
- `OPENROUTER_API_KEY` must be set.
|
||||
- `ffmpeg` and `ffprobe` on PATH.
|
||||
- Pillow installed (already in opensquilla deps).
|
||||
|
||||
## Risk
|
||||
|
||||
`high` — writes files, spends real OpenRouter credits, runs ffmpeg
|
||||
subprocesses. The review_gate ensures user consent before any paid step.
|
||||
|
||||
## Limits (v2)
|
||||
|
||||
- 1-10 shots; default 5. The DAG always declares 10 slots but
|
||||
`__SHOT_ABSENT__` gating keeps unused slots dormant.
|
||||
- Per-shot duration follows the script's DURATION_S (clamped 3-15s by
|
||||
seedance API). Total drama length scales linearly.
|
||||
- 9:16 portrait.
|
||||
- Per-shot seedance failures fall back to Ken-Burns. Image step
|
||||
has its own placeholder fallback. Prompt-extract llm_chats still
|
||||
abort the run if they return malformed output.
|
||||
- Concurrent runs with identical user_message collide on the same
|
||||
slug-derived subdir.
|
||||
|
||||
## When NOT to use
|
||||
|
||||
- Single image / single clip / script-only / stitch-only — use the
|
||||
underlying skills directly.
|
||||
'''
|
||||
|
||||
|
||||
def render() -> str:
|
||||
parts: list[str] = [HEAD]
|
||||
|
||||
# All 10 shot extract blocks together.
|
||||
for n in range(1, MAX_SHOTS + 1):
|
||||
parts.append(EXTRACT_TMPL.format(N=n))
|
||||
|
||||
# All 10 shot exec blocks together.
|
||||
for n in range(1, MAX_SHOTS + 1):
|
||||
parts.append(EXEC_TMPL.format(N=n))
|
||||
|
||||
parts.append(TAIL)
|
||||
rendered = "".join(parts)
|
||||
return rendered.replace("<<SLUG>>", SLUG_TMPL)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
sys.stdout.write(render())
|
||||
@@ -0,0 +1,378 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from opensquilla.cli.tui.backend.streaming import StreamingPlane
|
||||
from opensquilla.cli.tui.backend.transcript import (
|
||||
MessageItem,
|
||||
RouterDecisionItem,
|
||||
ToolItem,
|
||||
ToolPreviewPolicy,
|
||||
TranscriptStore,
|
||||
ViewportRequest,
|
||||
build_args_preview,
|
||||
build_output_preview,
|
||||
project_viewport,
|
||||
)
|
||||
from opensquilla.cli.tui.renderers.selection import get_renderer_backend
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
FIXTURE_PATH = PROJECT_ROOT / "tests" / "unit" / "cli" / "tui" / "replay_fixtures.py"
|
||||
DENSE_HISTORY_VIEWPORT = ViewportRequest(scroll_offset=200, viewport_height=24, overscan=3)
|
||||
DENSE_HISTORY_PREVIEW_POLICY = ToolPreviewPolicy()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReplaySummary:
|
||||
renderer: str
|
||||
fixture: str
|
||||
event_count: int
|
||||
text_chars: int
|
||||
tool_count: int
|
||||
router_decision_count: int
|
||||
wall_ms: float
|
||||
flush_count: int
|
||||
max_buffer_chars: int
|
||||
coalescing_ratio: float
|
||||
transcript_items: int
|
||||
visible_items: int
|
||||
expanded_tools: int
|
||||
projection_wall_ms: float
|
||||
available: bool
|
||||
skip_reason: str | None
|
||||
rendered_text_matches: bool
|
||||
plugin_error_count: int
|
||||
errors: list[str]
|
||||
|
||||
|
||||
class _ReplayStreamOutput:
|
||||
def __init__(self, output_handle: _ReplayOutputHandle) -> None:
|
||||
self._output_handle = output_handle
|
||||
|
||||
async def __aenter__(self):
|
||||
return self._output_handle.write
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class _ReplayOutputHandle:
|
||||
def __init__(self) -> None:
|
||||
self.flush_count = 0
|
||||
self.max_payload_chars = 0
|
||||
|
||||
def write(self, payload: str) -> None:
|
||||
self.flush_count += 1
|
||||
self.max_payload_chars = max(self.max_payload_chars, len(payload))
|
||||
|
||||
async def write_through(self, payload: str) -> None:
|
||||
self.write(payload)
|
||||
|
||||
def stream_output(self) -> _ReplayStreamOutput:
|
||||
return _ReplayStreamOutput(self)
|
||||
|
||||
|
||||
def _load_fixture_module() -> Any:
|
||||
spec = importlib.util.spec_from_file_location("tui_replay_fixtures", FIXTURE_PATH)
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError(f"Unable to load replay fixtures from {FIXTURE_PATH}")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _build_events(fixture: str) -> list[Any]:
|
||||
fixtures = _load_fixture_module()
|
||||
if fixture == "long-stream":
|
||||
return list(fixtures.build_long_stream_events())
|
||||
if fixture == "dense-history":
|
||||
return list(fixtures.build_dense_history_events())
|
||||
raise ValueError(f"Unsupported fixture: {fixture}")
|
||||
|
||||
|
||||
def _expected_stream_text(events: list[Any]) -> str:
|
||||
return "".join(
|
||||
str(event.payload.get("text", ""))
|
||||
for event in events
|
||||
if event.kind == "text_delta"
|
||||
)
|
||||
|
||||
|
||||
def _text_chars_for(event: Any) -> int:
|
||||
payload = event.payload
|
||||
if event.kind == "text_delta":
|
||||
return len(str(payload.get("text", "")))
|
||||
if event.kind == "history_message":
|
||||
return len(str(payload.get("content", "")))
|
||||
if event.kind == "tool_card":
|
||||
return len(str(payload.get("summary", "")))
|
||||
return 0
|
||||
|
||||
|
||||
def _optional_str(value: object) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return str(value)
|
||||
|
||||
|
||||
def _optional_float(value: object) -> float | None:
|
||||
if isinstance(value, int | float):
|
||||
return float(value)
|
||||
return None
|
||||
|
||||
|
||||
def _int_or_default(value: object, default: int) -> int:
|
||||
if not isinstance(value, int | float | str | bytes | bytearray):
|
||||
return default
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _append_transcript_event(store: TranscriptStore, event: Any) -> None:
|
||||
payload = event.payload
|
||||
if event.kind == "router_decision":
|
||||
store.append(
|
||||
RouterDecisionItem(
|
||||
tier=str(payload.get("tier", "")),
|
||||
model=str(payload.get("model", "")),
|
||||
baseline_model=_optional_str(payload.get("baseline_model")),
|
||||
confidence=_optional_float(payload.get("confidence")),
|
||||
rollout_phase=_optional_str(payload.get("rollout_phase")),
|
||||
timestamp_ms=event.timestamp_ms,
|
||||
)
|
||||
)
|
||||
elif event.kind == "history_message":
|
||||
store.append(
|
||||
MessageItem(
|
||||
role=str(payload.get("role", "")),
|
||||
text=str(payload.get("content", "")),
|
||||
run_id=None,
|
||||
timestamp_ms=event.timestamp_ms,
|
||||
)
|
||||
)
|
||||
elif event.kind == "tool_card":
|
||||
args_preview = build_args_preview(
|
||||
{
|
||||
"line_count": payload.get("line_count"),
|
||||
"rendered_bytes": payload.get("rendered_bytes"),
|
||||
},
|
||||
DENSE_HISTORY_PREVIEW_POLICY,
|
||||
)
|
||||
output_preview = build_output_preview(
|
||||
str(payload.get("summary", "")),
|
||||
DENSE_HISTORY_PREVIEW_POLICY,
|
||||
)
|
||||
store.append(
|
||||
ToolItem(
|
||||
tool_id=str(payload.get("tool_use_id", "")),
|
||||
name=str(payload.get("name", "tool")),
|
||||
status="done",
|
||||
args_preview=args_preview.text,
|
||||
output_preview=output_preview.text,
|
||||
expanded=bool(payload.get("expanded_candidate", False)),
|
||||
timestamp_ms=event.timestamp_ms,
|
||||
detail_line_count=_int_or_default(payload.get("line_count"), 1),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _flush_streaming_plane(
|
||||
renderer: Any,
|
||||
streaming_plane: StreamingPlane,
|
||||
) -> None:
|
||||
flush = streaming_plane.finish()
|
||||
if flush is not None:
|
||||
await renderer.aappend_text(flush.text)
|
||||
|
||||
|
||||
async def _render_event(
|
||||
renderer: Any,
|
||||
streaming_plane: StreamingPlane,
|
||||
event: Any,
|
||||
) -> None:
|
||||
payload = event.payload
|
||||
if event.kind == "text_delta":
|
||||
flush = streaming_plane.append(str(payload.get("text", "")))
|
||||
if flush is not None:
|
||||
await renderer.aappend_text(flush.text)
|
||||
elif event.kind == "tool_start":
|
||||
await _flush_streaming_plane(renderer, streaming_plane)
|
||||
args = payload.get("args")
|
||||
await renderer.atool_start(
|
||||
str(payload.get("name", "tool")),
|
||||
args if isinstance(args, dict) else None,
|
||||
str(payload.get("tool_use_id", "")),
|
||||
)
|
||||
elif event.kind == "tool_finished":
|
||||
await _flush_streaming_plane(renderer, streaming_plane)
|
||||
elapsed = payload.get("elapsed")
|
||||
await renderer.atool_finished(
|
||||
str(payload.get("tool_use_id", "")),
|
||||
success=bool(payload.get("success", True)),
|
||||
elapsed=elapsed if isinstance(elapsed, float) else None,
|
||||
error=str(payload["error"]) if "error" in payload else None,
|
||||
)
|
||||
elif event.kind == "router_decision":
|
||||
await _flush_streaming_plane(renderer, streaming_plane)
|
||||
await renderer.astatus(
|
||||
"route: "
|
||||
f"{payload.get('tier')} -> {payload.get('model')} "
|
||||
f"(baseline {payload.get('baseline_model')})",
|
||||
style="cyan",
|
||||
)
|
||||
elif event.kind == "history_message":
|
||||
await _flush_streaming_plane(renderer, streaming_plane)
|
||||
await renderer.astatus(
|
||||
f"{payload.get('role')}: {str(payload.get('content', ''))[:120]}",
|
||||
style="dim",
|
||||
)
|
||||
elif event.kind == "tool_card":
|
||||
await _flush_streaming_plane(renderer, streaming_plane)
|
||||
await renderer.astatus(
|
||||
f"tool: {payload.get('name')} {payload.get('summary')}",
|
||||
style="magenta",
|
||||
)
|
||||
elif event.kind == "done":
|
||||
await _flush_streaming_plane(renderer, streaming_plane)
|
||||
await renderer.afinalize()
|
||||
|
||||
|
||||
async def run_replay(renderer: str, fixture: str, *, repeat: int = 1) -> ReplaySummary:
|
||||
if repeat < 1:
|
||||
raise ValueError("--repeat must be >= 1")
|
||||
backend = get_renderer_backend(renderer)
|
||||
|
||||
errors: list[str] = []
|
||||
event_count = 0
|
||||
text_chars = 0
|
||||
tool_count = 0
|
||||
router_decision_count = 0
|
||||
text_delta_count = 0
|
||||
streaming_flush_count = 0
|
||||
flush_count = 0
|
||||
max_buffer_chars = 0
|
||||
transcript_items = 0
|
||||
visible_items = 0
|
||||
expanded_tools = 0
|
||||
projection_wall_ms = 0.0
|
||||
rendered_text_matches = True
|
||||
started_at = time.perf_counter()
|
||||
|
||||
for _ in range(repeat):
|
||||
events = _build_events(fixture)
|
||||
output_handle = _ReplayOutputHandle() if fixture == "long-stream" else None
|
||||
replay_renderer = (
|
||||
backend.create_renderer(title="tui-replay", output_handle=output_handle)
|
||||
if fixture == "long-stream"
|
||||
else None
|
||||
)
|
||||
streaming_plane = StreamingPlane()
|
||||
transcript_store = TranscriptStore() if fixture == "dense-history" else None
|
||||
for event in events:
|
||||
event_count += 1
|
||||
text_chars += _text_chars_for(event)
|
||||
if event.kind == "text_delta":
|
||||
text_delta_count += 1
|
||||
if event.kind in {"tool_start", "tool_card"}:
|
||||
tool_count += 1
|
||||
if event.kind == "router_decision":
|
||||
router_decision_count += 1
|
||||
try:
|
||||
if transcript_store is not None:
|
||||
_append_transcript_event(transcript_store, event)
|
||||
elif replay_renderer is not None:
|
||||
await _render_event(replay_renderer, streaming_plane, event)
|
||||
except Exception as exc: # pragma: no cover - summarized for CLI evidence.
|
||||
errors.append(f"{event.kind}: {exc}")
|
||||
max_buffer_chars = max(
|
||||
max_buffer_chars,
|
||||
streaming_plane.max_buffer_chars,
|
||||
)
|
||||
if replay_renderer is not None:
|
||||
await replay_renderer.aclose()
|
||||
rendered_text_matches = rendered_text_matches and (
|
||||
getattr(replay_renderer, "buffer", "") == _expected_stream_text(events)
|
||||
)
|
||||
if transcript_store is not None:
|
||||
snapshot = transcript_store.snapshot()
|
||||
projection_started_at = time.perf_counter()
|
||||
projection = project_viewport(snapshot, DENSE_HISTORY_VIEWPORT)
|
||||
projection_wall_ms += (
|
||||
time.perf_counter() - projection_started_at
|
||||
) * 1_000
|
||||
transcript_items += len(snapshot)
|
||||
visible_items += len(projection.items)
|
||||
expanded_tools += sum(
|
||||
1 for item in snapshot if isinstance(item, ToolItem) and item.expanded
|
||||
)
|
||||
streaming_flush_count += streaming_plane.flush_count
|
||||
if output_handle is not None:
|
||||
flush_count += output_handle.flush_count
|
||||
elif replay_renderer is not None:
|
||||
flush_count += int(getattr(replay_renderer, "flush_count", 0))
|
||||
|
||||
wall_ms = (time.perf_counter() - started_at) * 1_000
|
||||
coalescing_ratio = (
|
||||
round(streaming_flush_count / text_delta_count, 6)
|
||||
if text_delta_count > 0
|
||||
else 0.0
|
||||
)
|
||||
return ReplaySummary(
|
||||
renderer=renderer,
|
||||
fixture=fixture,
|
||||
event_count=event_count,
|
||||
text_chars=text_chars,
|
||||
tool_count=tool_count,
|
||||
router_decision_count=router_decision_count,
|
||||
wall_ms=round(wall_ms, 3),
|
||||
flush_count=flush_count,
|
||||
max_buffer_chars=max_buffer_chars,
|
||||
coalescing_ratio=coalescing_ratio,
|
||||
transcript_items=transcript_items,
|
||||
visible_items=visible_items,
|
||||
expanded_tools=expanded_tools,
|
||||
projection_wall_ms=round(projection_wall_ms, 3),
|
||||
available=True,
|
||||
skip_reason=None,
|
||||
rendered_text_matches=rendered_text_matches,
|
||||
plugin_error_count=0,
|
||||
errors=errors,
|
||||
)
|
||||
|
||||
|
||||
def write_summary(summary: ReplaySummary, summary_json: Path) -> None:
|
||||
summary_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
summary_json.write_text(json.dumps(asdict(summary), indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Replay synthetic TUI events.")
|
||||
parser.add_argument("--renderer", choices=("opentui",), required=True)
|
||||
parser.add_argument("--fixture", choices=("long-stream", "dense-history"), required=True)
|
||||
parser.add_argument("--summary-json", type=Path, required=True)
|
||||
parser.add_argument("--repeat", type=int, default=1)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = _parse_args(argv)
|
||||
summary = asyncio.run(
|
||||
run_replay(args.renderer, args.fixture, repeat=args.repeat),
|
||||
)
|
||||
write_summary(summary, args.summary_json)
|
||||
return 1 if summary.errors else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,888 @@
|
||||
"""Lifestyle meta-skill benchmark against the OpenClaw t3 baseline.
|
||||
|
||||
This catalog is intentionally narrower than ``compare_meta_skill_openclaw``:
|
||||
it covers retained practical work/life meta-skills and frames each case so the
|
||||
OpenSquilla meta-skill orchestration path can be judged against OpenClaw's
|
||||
t3 Opus 4.8 baseline.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import asdict
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
if __package__ in {None, ""}:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from scripts.compare_meta_skill_openclaw import (
|
||||
ComparisonCase,
|
||||
EndpointResult,
|
||||
JudgeResult,
|
||||
LLMJudge,
|
||||
OpenClawRunner,
|
||||
OpenSquillaRunner,
|
||||
RubricCriterion,
|
||||
_openclaw_session_file_events,
|
||||
apply_judge_result,
|
||||
criterion,
|
||||
extract_text_from_events,
|
||||
read_judge_api_key,
|
||||
read_openclaw_token,
|
||||
read_opensquilla_token,
|
||||
score_response,
|
||||
)
|
||||
|
||||
REPORT_DIR = Path(
|
||||
os.environ.get("OPENSQUILLA_LIFESTYLE_COMPARE_REPORT_DIR", ".reports/meta-skill-comparison")
|
||||
)
|
||||
OPENCLAW_T3_MODEL = os.environ.get("OPENCLAW_T3_MODEL", "t3-opus-4.7")
|
||||
OPENCLAW_BASELINE_LABEL = "OpenClaw + t3 + capability-equivalent normal skills baseline"
|
||||
MATCHED_OPENCLAW_NORMAL_SKILLS = (
|
||||
"OpenSquilla multi-search-engine -> OpenClaw multi-search-engine",
|
||||
"OpenSquilla docx -> OpenClaw word-docx",
|
||||
"OpenSquilla xlsx -> OpenClaw excel-xlsx",
|
||||
"OpenSquilla pdf-toolkit -> OpenClaw pdf-toolkit",
|
||||
"OpenSquilla deep-research -> OpenClaw deep-research-pro",
|
||||
"OpenSquilla weather -> OpenClaw weather",
|
||||
"OpenSquilla summarize -> OpenClaw summarize",
|
||||
"OpenSquilla memory -> OpenClaw longterm-memory/notes if installed",
|
||||
"OpenSquilla pptx -> OpenClaw pptx/presentation skill if installed",
|
||||
)
|
||||
BENCHMARK_LABEL = f"OpenSquilla + Squilla Router vs {OPENCLAW_BASELINE_LABEL}"
|
||||
LIFESTYLE_JUDGE_SUBSCORE_RANGES: dict[str, tuple[int, int]] = {
|
||||
"final_artifact_quality": (0, 40),
|
||||
"task_completion": (0, 20),
|
||||
"evidence_traceability": (0, 15),
|
||||
"actionability": (0, 10),
|
||||
"risk_boundary_safety": (0, 10),
|
||||
"meta_skill_fit": (0, 5),
|
||||
}
|
||||
|
||||
|
||||
KID_PROJECT_RUBRIC: tuple[RubricCriterion, ...] = (
|
||||
criterion(
|
||||
"age_fit",
|
||||
"Adapts the plan to child age and guardian involvement.",
|
||||
r"8 岁",
|
||||
r"age",
|
||||
r"年龄",
|
||||
r"家长",
|
||||
r"guardian",
|
||||
),
|
||||
criterion(
|
||||
"step_plan",
|
||||
"Creates a clear day-by-day or session-by-session plan.",
|
||||
r"Day",
|
||||
r"第",
|
||||
r"步骤",
|
||||
r"step",
|
||||
r"timeline",
|
||||
r"时间表",
|
||||
),
|
||||
criterion(
|
||||
"materials_budget",
|
||||
"Lists materials, budget, and household substitutes.",
|
||||
r"materials",
|
||||
r"材料",
|
||||
r"预算",
|
||||
r"substitute",
|
||||
r"替代",
|
||||
),
|
||||
criterion(
|
||||
"safety",
|
||||
"Flags safety hazards and supervision requirements.",
|
||||
r"safety",
|
||||
r"安全",
|
||||
r"supervision",
|
||||
r"监督",
|
||||
r"adult",
|
||||
r"大人",
|
||||
),
|
||||
criterion(
|
||||
"learning_objectives",
|
||||
"Explains what the child should learn and present.",
|
||||
r"learn",
|
||||
r"学习",
|
||||
r"原理",
|
||||
r"presentation",
|
||||
r"展示",
|
||||
),
|
||||
criterion(
|
||||
"weather_or_constraints",
|
||||
"Handles outdoor/weather/deadline constraints and assumptions.",
|
||||
r"weather",
|
||||
r"天气",
|
||||
r"deadline",
|
||||
r"截止",
|
||||
r"assumption",
|
||||
r"假设",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
LIFESTYLE_COMPARISON_CASES: list[ComparisonCase] = [
|
||||
ComparisonCase(
|
||||
case_id="kid_project_balcony_plants",
|
||||
skill_name="meta-kid-project-planner",
|
||||
scenario="lifestyle_primary",
|
||||
prompt=(
|
||||
"孩子 8 岁，科学课两周后要交一个小项目。她想做“阳台种豆芽/小植物观察”，"
|
||||
"家里有透明杯、纸巾、绿豆、尺子和彩笔，预算最好 50 元以内。我们住杭州，"
|
||||
"阳台有半天太阳，平时我只能晚上陪 20 分钟。请帮我做一个孩子能看懂、家长也能执行的计划："
|
||||
"每天做什么、材料清单和替代品、安全注意、怎么记录数据和画图、最后展示怎么讲，"
|
||||
"如果天气或光照不稳定要怎么调整，哪些地方你只能先假设。"
|
||||
),
|
||||
expected_advantage=(
|
||||
"OpenSquilla + Squilla Router should activate kid-project-planner, combine "
|
||||
"age fit, materials, weather-aware constraints, safety review, and parent "
|
||||
"learning objectives, then beat OpenClaw + t3 Opus 4.8 on an executable "
|
||||
"child-and-guardian project plan."
|
||||
),
|
||||
optimization_if_not_better=(
|
||||
"If OpenSquilla does not beat OpenClaw, strengthen kid-project-planner to "
|
||||
"always produce kid-facing steps, guardian notes, material substitutes, "
|
||||
"safety checks, data-recording templates, and assumption labels."
|
||||
),
|
||||
rubric=KID_PROJECT_RUBRIC,
|
||||
failure_modes=(
|
||||
"Gives a generic plant project answer without adapting to an 8-year-old.",
|
||||
(
|
||||
"Misses the 50 RMB budget, nightly 20-minute supervision, "
|
||||
"or light/weather constraints."
|
||||
),
|
||||
"Omits safety, data recording, or presentation guidance.",
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
ENGLISH_LIFESTYLE_PROMPTS: dict[str, str] = {
|
||||
"kid_project_balcony_plants": (
|
||||
"My child is 8 and needs to submit a small science project in two weeks. She wants to do "
|
||||
"a balcony sprout or small-plant observation project. At home we have clear cups, paper "
|
||||
"towels, mung beans, a ruler, and colored pens, and I want to keep the budget under "
|
||||
"RMB 50. We live in Hangzhou, the balcony gets half a day of sun, and I can only "
|
||||
"help for 20 minutes "
|
||||
"in the evening. Please make a plan that a child can understand and a parent can actually "
|
||||
"supervise: what to do each day, materials and substitutes, safety notes, how to "
|
||||
"record data "
|
||||
"and draw charts, how to present the final result, how to adjust if weather or light is "
|
||||
"unstable, and what you have to assume."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _placeholder_result(endpoint: str, case: ComparisonCase) -> EndpointResult:
|
||||
return EndpointResult(
|
||||
endpoint=endpoint,
|
||||
case_id=case.case_id,
|
||||
ok=False,
|
||||
elapsed_s=0.0,
|
||||
response_text="",
|
||||
score=asdict(score_response("", case)),
|
||||
error="not run",
|
||||
model=OPENCLAW_T3_MODEL if endpoint == "openclaw" else None,
|
||||
)
|
||||
|
||||
|
||||
def build_lifestyle_rows(language: str = "zh") -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for case in _cases_for_language(language):
|
||||
rows.append(
|
||||
{
|
||||
"case": _case_to_dict(case),
|
||||
"benchmark": BENCHMARK_LABEL,
|
||||
"opensquilla": asdict(_placeholder_result("opensquilla", case)),
|
||||
"openclaw": asdict(_placeholder_result("openclaw", case)),
|
||||
"baseline_model": OPENCLAW_T3_MODEL,
|
||||
"baseline_winner": "tie",
|
||||
"winner": "tie",
|
||||
"score_basis": "not_run",
|
||||
"opensquilla_better": False,
|
||||
"recommended_optimization": case.optimization_if_not_better,
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def render_lifestyle_markdown(rows: list[dict[str, Any]]) -> str:
|
||||
total = len(rows)
|
||||
sq_wins = sum(1 for row in rows if row["winner"] == "opensquilla")
|
||||
claw_wins = sum(1 for row in rows if row["winner"] == "openclaw")
|
||||
ties = sum(1 for row in rows if row["winner"] == "tie")
|
||||
lines = [
|
||||
"# OpenSquilla Meta-Skills vs OpenClaw t3 Matched-Skills Lifestyle Benchmark",
|
||||
"",
|
||||
f"Benchmark: {BENCHMARK_LABEL}",
|
||||
f"{OPENCLAW_BASELINE_LABEL} model: `{OPENCLAW_T3_MODEL}`",
|
||||
"Matched OpenClaw normal skills: "
|
||||
+ ", ".join(f"`{skill}`" for skill in MATCHED_OPENCLAW_NORMAL_SKILLS),
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
(
|
||||
f"OpenSquilla + Squilla Router wins: {sq_wins}/{total}; "
|
||||
f"{OPENCLAW_BASELINE_LABEL} wins: {claw_wins}/{total}; "
|
||||
f"ties/not-run: {ties}."
|
||||
),
|
||||
"",
|
||||
(
|
||||
"| Case | Meta-skill | OpenSquilla model | OpenClaw model | Deterministic "
|
||||
"| Judge 0-100 | Final artifact | Basis | Winner | Issue |"
|
||||
),
|
||||
"| --- | --- | --- | --- | ---: | ---: | ---: | --- | --- | --- |",
|
||||
]
|
||||
for row in rows:
|
||||
lines.append(
|
||||
(
|
||||
"| {case} | `{skill}` | `{sq_model}` | `{claw_model}` | {det} "
|
||||
"| {judge} | {artifact} | {basis} | {winner} | {issue} |"
|
||||
).format(
|
||||
case=row["case"]["case_id"],
|
||||
skill=row["case"]["skill_name"],
|
||||
sq_model=row["opensquilla"].get("model") or "",
|
||||
claw_model=row["openclaw"].get("model") or "",
|
||||
det=f"{row['opensquilla']['score']['total']}-{row['openclaw']['score']['total']}",
|
||||
judge=_judge_scores_cell(row),
|
||||
artifact=_judge_final_artifact_cell(row),
|
||||
basis=row.get("score_basis", ""),
|
||||
winner=row["winner"],
|
||||
issue=_judge_issue_cell(row).replace("|", "/"),
|
||||
)
|
||||
)
|
||||
lines.extend(["", "## Cases", ""])
|
||||
for row in rows:
|
||||
case = row["case"]
|
||||
lines.append(f"### {case['case_id']}")
|
||||
lines.append("")
|
||||
lines.append(f"- Meta-skill: `{case['skill_name']}`")
|
||||
lines.append(f"- Scenario: {case['scenario']}")
|
||||
lines.append(f"- Expected advantage: {case['expected_advantage']}")
|
||||
lines.append(f"- Baseline: {OPENCLAW_BASELINE_LABEL} (`{OPENCLAW_T3_MODEL}`)")
|
||||
lines.append("- Rubric: " + ", ".join(item["name"] for item in case["rubric"]))
|
||||
lines.append("- Failure modes: " + "; ".join(case["failure_modes"]))
|
||||
lines.append("")
|
||||
lines.append("```text")
|
||||
lines.append(case["prompt"])
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def render_lifestyle_prompts_markdown(rows: list[dict[str, Any]]) -> str:
|
||||
lines = [
|
||||
"# Lifestyle Test Prompts",
|
||||
"",
|
||||
]
|
||||
for row in rows:
|
||||
case = row["case"]
|
||||
lines.append(f"## {case['case_id']}")
|
||||
lines.append("")
|
||||
lines.append("### 中文")
|
||||
lines.append("")
|
||||
lines.append("```text")
|
||||
original = next(
|
||||
item
|
||||
for item in LIFESTYLE_COMPARISON_CASES
|
||||
if item.case_id == case["case_id"].removesuffix("_en")
|
||||
)
|
||||
lines.append(original.prompt)
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
lines.append("### English")
|
||||
lines.append("")
|
||||
lines.append("```text")
|
||||
lines.append(ENGLISH_LIFESTYLE_PROMPTS[original.case_id])
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _judge_scores_cell(row: dict[str, Any]) -> str:
|
||||
judge = row.get("judge") if isinstance(row.get("judge"), dict) else {}
|
||||
scores = judge.get("scores") if isinstance(judge.get("scores"), dict) else {}
|
||||
if not scores:
|
||||
return ""
|
||||
return f"{scores.get('opensquilla', '')}-{scores.get('openclaw', '')}"
|
||||
|
||||
|
||||
def _judge_final_artifact_cell(row: dict[str, Any]) -> str:
|
||||
judge = row.get("judge") if isinstance(row.get("judge"), dict) else {}
|
||||
raw = judge.get("raw") if isinstance(judge.get("raw"), dict) else {}
|
||||
subscores = raw.get("subscores") if isinstance(raw.get("subscores"), dict) else {}
|
||||
opensquilla = (
|
||||
subscores.get("opensquilla") if isinstance(subscores.get("opensquilla"), dict) else {}
|
||||
)
|
||||
openclaw = subscores.get("openclaw") if isinstance(subscores.get("openclaw"), dict) else {}
|
||||
if not opensquilla and not openclaw:
|
||||
return ""
|
||||
return (
|
||||
f"{opensquilla.get('final_artifact_quality', '')}-"
|
||||
f"{openclaw.get('final_artifact_quality', '')}"
|
||||
)
|
||||
|
||||
|
||||
def _judge_issue_cell(row: dict[str, Any]) -> str:
|
||||
if row.get("invalid_reasons"):
|
||||
return "; ".join(str(item) for item in row["invalid_reasons"])
|
||||
if row.get("judge_error"):
|
||||
return str(row["judge_error"])
|
||||
judge = row.get("judge") if isinstance(row.get("judge"), dict) else {}
|
||||
if row.get("score_basis") == "llm_judge":
|
||||
raw = judge.get("raw") if isinstance(judge.get("raw"), dict) else {}
|
||||
if not judge.get("scores") or not raw.get("subscores") or not judge.get("rationale"):
|
||||
return "incomplete_judge_payload"
|
||||
return ""
|
||||
|
||||
|
||||
def write_lifestyle_reports(
|
||||
rows: list[dict[str, Any]], stamp: str | None = None
|
||||
) -> tuple[Path, Path]:
|
||||
REPORT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
if stamp is None:
|
||||
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||
jsonl_path = REPORT_DIR / f"openclaw_t3_vs_opensquilla_lifestyle_meta_{stamp}.jsonl"
|
||||
md_path = REPORT_DIR / f"openclaw_t3_vs_opensquilla_lifestyle_meta_{stamp}.md"
|
||||
prompts_path = REPORT_DIR / f"openclaw_t3_vs_opensquilla_lifestyle_meta_prompts_{stamp}.md"
|
||||
with jsonl_path.open("w", encoding="utf-8") as fh:
|
||||
for row in rows:
|
||||
fh.write(json.dumps(row, ensure_ascii=False) + "\n")
|
||||
md_path.write_text(render_lifestyle_markdown(rows), encoding="utf-8")
|
||||
prompts_path.write_text(render_lifestyle_prompts_markdown(rows), encoding="utf-8")
|
||||
print(f"wrote {jsonl_path}")
|
||||
print(f"wrote {md_path}")
|
||||
print(f"wrote {prompts_path}")
|
||||
return jsonl_path, md_path
|
||||
|
||||
|
||||
async def run_live(args: argparse.Namespace) -> list[dict[str, Any]]:
|
||||
selected = _select_cases(args.case, language=args.prompt_language)
|
||||
if not args.openclaw_config and not args.openclaw_baseline_jsonl:
|
||||
raise SystemExit("Pass --openclaw-config or set OPENCLAW_CONFIG.")
|
||||
opensquilla = OpenSquillaRunner(
|
||||
args.opensquilla_url,
|
||||
args.opensquilla_token,
|
||||
elevated=args.opensquilla_elevated,
|
||||
agent_id=args.opensquilla_agent_id,
|
||||
isolated_agent_per_case=args.opensquilla_isolated_agents,
|
||||
run_id=args.opensquilla_run_id,
|
||||
)
|
||||
openclaw = None
|
||||
openclaw_baseline = {}
|
||||
openclaw_state_dir = Path(args.openclaw_config).parent if args.openclaw_config else None
|
||||
if args.openclaw_baseline_jsonl:
|
||||
openclaw_baseline = load_openclaw_baseline(
|
||||
Path(args.openclaw_baseline_jsonl),
|
||||
selected,
|
||||
state_dir=openclaw_state_dir,
|
||||
)
|
||||
else:
|
||||
openclaw = OpenClawRunner(
|
||||
args.openclaw_url,
|
||||
read_openclaw_token(Path(args.openclaw_config)),
|
||||
args.openclaw_idle_timeout,
|
||||
state_dir=openclaw_state_dir,
|
||||
)
|
||||
judge = None
|
||||
if args.judge_llm:
|
||||
if not args.judge_model:
|
||||
raise SystemExit("Pass --judge-model or set OPENSQUILLA_JUDGE_MODEL.")
|
||||
judge = LLMJudge(
|
||||
model=args.judge_model,
|
||||
api_key=args.judge_api_key,
|
||||
base_url=args.judge_base_url,
|
||||
timeout_s=args.judge_timeout,
|
||||
)
|
||||
|
||||
rows: list[dict[str, Any]] = []
|
||||
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||
for case in selected:
|
||||
print(f"running {case.case_id} ...", flush=True)
|
||||
if openclaw_baseline:
|
||||
sq_result = await opensquilla.run(case, args.timeout)
|
||||
claw_result = openclaw_baseline[case.case_id]
|
||||
else:
|
||||
assert openclaw is not None
|
||||
sq_result, claw_result = await asyncio.gather(
|
||||
opensquilla.run(case, args.timeout),
|
||||
openclaw.run(case, args.timeout),
|
||||
)
|
||||
if not claw_result.model:
|
||||
claw_result.model = OPENCLAW_T3_MODEL
|
||||
row = _compare_results(case, sq_result, claw_result)
|
||||
if judge is not None and row.get("score_basis") != "invalid_endpoint":
|
||||
try:
|
||||
judge_result = await _judge_lifestyle_with_retries(
|
||||
judge,
|
||||
case,
|
||||
sq_result,
|
||||
claw_result,
|
||||
)
|
||||
row = _apply_lifestyle_judge_result(row, judge_result, case)
|
||||
except Exception as exc:
|
||||
row["judge_error"] = f"{type(exc).__name__}: {exc}"
|
||||
rows.append(row)
|
||||
judge_suffix = ""
|
||||
if row.get("judge"):
|
||||
judge_suffix = (
|
||||
f" judge={_judge_scores_cell(row) or 'n/a'}"
|
||||
f" final_artifact={_judge_final_artifact_cell(row) or 'n/a'}"
|
||||
)
|
||||
elif row.get("judge_error"):
|
||||
judge_suffix = f" judge_error={row['judge_error']}"
|
||||
print(
|
||||
f"{case.case_id}: opensquilla={sq_result.score['total']} "
|
||||
f"openclaw_t3={claw_result.score['total']}{judge_suffix} "
|
||||
f"opensquilla_model={sq_result.model or ''} "
|
||||
f"openclaw_model={claw_result.model or OPENCLAW_T3_MODEL} "
|
||||
f"winner={row['winner']}",
|
||||
flush=True,
|
||||
)
|
||||
write_lifestyle_reports(rows, stamp=stamp)
|
||||
write_lifestyle_reports(rows, stamp=stamp)
|
||||
return rows
|
||||
|
||||
|
||||
async def judge_existing(args: argparse.Namespace) -> list[dict[str, Any]]:
|
||||
if not args.judge_jsonl:
|
||||
raise SystemExit("Pass --judge-jsonl.")
|
||||
if not args.judge_model:
|
||||
raise SystemExit("Pass --judge-model or set OPENSQUILLA_JUDGE_MODEL.")
|
||||
judge = LLMJudge(
|
||||
model=args.judge_model,
|
||||
api_key=args.judge_api_key,
|
||||
base_url=args.judge_base_url,
|
||||
timeout_s=args.judge_timeout,
|
||||
)
|
||||
rows = [
|
||||
json.loads(line)
|
||||
for line in Path(args.judge_jsonl).read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
judged_rows: list[dict[str, Any]] = []
|
||||
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||
for row in rows:
|
||||
case = _case_from_dict(row["case"])
|
||||
opensquilla = _endpoint_from_dict(row["opensquilla"])
|
||||
openclaw = _endpoint_from_dict(row["openclaw"])
|
||||
row.setdefault("baseline_winner", row.get("winner", "tie"))
|
||||
row.setdefault("score_basis", "deterministic")
|
||||
try:
|
||||
judge_result = await _judge_lifestyle_with_retries(
|
||||
judge,
|
||||
case,
|
||||
opensquilla,
|
||||
openclaw,
|
||||
)
|
||||
judged = _apply_lifestyle_judge_result(row, judge_result, case)
|
||||
except Exception as exc:
|
||||
judged = dict(row)
|
||||
judged["judge_error"] = f"{type(exc).__name__}: {exc}"
|
||||
judged_rows.append(judged)
|
||||
print(
|
||||
f"judged {case.case_id}: winner={judged.get('winner')} "
|
||||
f"judge={_judge_scores_cell(judged) or 'n/a'}",
|
||||
flush=True,
|
||||
)
|
||||
write_lifestyle_reports(judged_rows, stamp=stamp)
|
||||
write_lifestyle_reports(judged_rows, stamp=stamp)
|
||||
return judged_rows
|
||||
|
||||
|
||||
def load_openclaw_baseline(
|
||||
path: Path,
|
||||
cases: list[ComparisonCase],
|
||||
*,
|
||||
state_dir: Path | None = None,
|
||||
) -> dict[str, EndpointResult]:
|
||||
rows = [
|
||||
json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()
|
||||
]
|
||||
by_case = {str(row["case"]["case_id"]): row for row in rows}
|
||||
baseline: dict[str, EndpointResult] = {}
|
||||
for case in cases:
|
||||
row = by_case.get(case.case_id)
|
||||
if row is None:
|
||||
raise SystemExit(f"OpenClaw baseline missing case {case.case_id!r} in {path}")
|
||||
baseline_prompt = str(row.get("case", {}).get("prompt", ""))
|
||||
if baseline_prompt != case.prompt:
|
||||
raise SystemExit(
|
||||
f"OpenClaw baseline prompt mismatch for {case.case_id!r}; "
|
||||
"use the exact prompt that produced the locked baseline"
|
||||
)
|
||||
result = _endpoint_from_dict(row["openclaw"])
|
||||
refreshed = _refreshed_openclaw_text_from_state(
|
||||
result.session_key,
|
||||
case.prompt,
|
||||
state_dir,
|
||||
)
|
||||
if refreshed and len(refreshed) > len(result.response_text.strip()):
|
||||
result.response_text = refreshed
|
||||
result.ok = True
|
||||
result.error = None
|
||||
result.score = asdict(score_response(refreshed, case))
|
||||
if not result.model:
|
||||
result.model = OPENCLAW_T3_MODEL
|
||||
baseline[case.case_id] = result
|
||||
return baseline
|
||||
|
||||
|
||||
def _endpoint_from_dict(data: dict[str, Any]) -> EndpointResult:
|
||||
return EndpointResult(
|
||||
endpoint=str(data.get("endpoint", "openclaw")),
|
||||
case_id=str(data["case_id"]),
|
||||
ok=bool(data.get("ok")),
|
||||
elapsed_s=float(data.get("elapsed_s", 0.0)),
|
||||
response_text=str(data.get("response_text", "")),
|
||||
score=data.get("score") if isinstance(data.get("score"), dict) else {},
|
||||
error=str(data["error"]) if data.get("error") else None,
|
||||
session_key=str(data["session_key"]) if data.get("session_key") else None,
|
||||
model=str(data["model"]) if data.get("model") else None,
|
||||
provider=str(data["provider"]) if data.get("provider") else None,
|
||||
event_count=int(data.get("event_count", 0)),
|
||||
)
|
||||
|
||||
|
||||
def _refreshed_openclaw_text_from_state(
|
||||
session_key: str | None,
|
||||
prompt: str,
|
||||
state_dir: Path | None,
|
||||
) -> str:
|
||||
path = _openclaw_session_file_for_key(state_dir, session_key)
|
||||
if path is None:
|
||||
return ""
|
||||
return extract_text_from_events(
|
||||
_openclaw_session_file_events(path, session_key or "", after_prompt=prompt)
|
||||
)
|
||||
|
||||
|
||||
def _openclaw_session_file_for_key(
|
||||
state_dir: Path | None,
|
||||
session_key: str | None,
|
||||
) -> Path | None:
|
||||
if state_dir is None or not session_key:
|
||||
return None
|
||||
sessions_dir = state_dir / "agents" / "main" / "sessions"
|
||||
if not sessions_dir.exists():
|
||||
return None
|
||||
for trajectory_path in sessions_dir.glob("*.trajectory.jsonl"):
|
||||
try:
|
||||
text = trajectory_path.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
continue
|
||||
if session_key not in text:
|
||||
continue
|
||||
session_file = trajectory_path.with_name(
|
||||
trajectory_path.name.replace(".trajectory.jsonl", ".jsonl")
|
||||
)
|
||||
if session_file.exists():
|
||||
return session_file
|
||||
return None
|
||||
|
||||
|
||||
def _compare_results(
|
||||
case: ComparisonCase,
|
||||
opensquilla: EndpointResult,
|
||||
openclaw: EndpointResult,
|
||||
) -> dict[str, Any]:
|
||||
invalid_reasons = _invalid_endpoint_reasons(opensquilla, openclaw)
|
||||
if invalid_reasons:
|
||||
return {
|
||||
"case": _case_to_dict(case),
|
||||
"benchmark": BENCHMARK_LABEL,
|
||||
"opensquilla": asdict(opensquilla),
|
||||
"openclaw": asdict(openclaw),
|
||||
"baseline_model": openclaw.model or OPENCLAW_T3_MODEL,
|
||||
"baseline_winner": "invalid",
|
||||
"winner": "invalid",
|
||||
"score_basis": "invalid_endpoint",
|
||||
"opensquilla_better": False,
|
||||
"invalid_reasons": invalid_reasons,
|
||||
"recommended_optimization": None,
|
||||
}
|
||||
sq_total = int(opensquilla.score["total"])
|
||||
claw_total = int(openclaw.score["total"])
|
||||
if sq_total > claw_total:
|
||||
winner = "opensquilla"
|
||||
elif claw_total > sq_total:
|
||||
winner = "openclaw"
|
||||
else:
|
||||
winner = "tie"
|
||||
return {
|
||||
"case": _case_to_dict(case),
|
||||
"benchmark": BENCHMARK_LABEL,
|
||||
"opensquilla": asdict(opensquilla),
|
||||
"openclaw": asdict(openclaw),
|
||||
"baseline_model": openclaw.model or OPENCLAW_T3_MODEL,
|
||||
"baseline_winner": winner,
|
||||
"winner": winner,
|
||||
"score_basis": "deterministic",
|
||||
"opensquilla_better": winner == "opensquilla",
|
||||
"recommended_optimization": None
|
||||
if winner == "opensquilla"
|
||||
else case.optimization_if_not_better,
|
||||
}
|
||||
|
||||
|
||||
def _invalid_endpoint_reasons(*results: EndpointResult) -> list[str]:
|
||||
reasons: list[str] = []
|
||||
for result in results:
|
||||
if not result.ok:
|
||||
reasons.append(f"{result.endpoint}: not ok")
|
||||
if not result.response_text.strip():
|
||||
reasons.append(f"{result.endpoint}: empty response")
|
||||
if _looks_like_unrelated_bootstrap(result.response_text):
|
||||
reasons.append(f"{result.endpoint}: unrelated bootstrap response")
|
||||
if result.error:
|
||||
reasons.append(f"{result.endpoint}: {result.error}")
|
||||
return reasons
|
||||
|
||||
|
||||
def _looks_like_unrelated_bootstrap(text: str) -> bool:
|
||||
lowered = text.lower()
|
||||
bootstrap_phrases = (
|
||||
"bootstrap removed",
|
||||
"ready for the task",
|
||||
"what would you like me to do",
|
||||
"who am i",
|
||||
"what should they call you",
|
||||
)
|
||||
return len(text.strip()) < 500 and any(phrase in lowered for phrase in bootstrap_phrases)
|
||||
|
||||
|
||||
def _apply_lifestyle_judge_result(
|
||||
row: dict[str, Any],
|
||||
judge_result: JudgeResult,
|
||||
case: ComparisonCase,
|
||||
) -> dict[str, Any]:
|
||||
normalized = _normalized_lifestyle_judge_result(judge_result)
|
||||
if normalized is None:
|
||||
raise RuntimeError("judge response missing required scores, subscores, or rationale")
|
||||
updated = apply_judge_result(row, normalized, case)
|
||||
updated["benchmark"] = BENCHMARK_LABEL
|
||||
updated["baseline_model"] = row.get("baseline_model") or OPENCLAW_T3_MODEL
|
||||
return updated
|
||||
|
||||
|
||||
async def _judge_lifestyle_with_retries(
|
||||
judge: LLMJudge,
|
||||
case: ComparisonCase,
|
||||
opensquilla: EndpointResult,
|
||||
openclaw: EndpointResult,
|
||||
*,
|
||||
attempts: int = 3,
|
||||
) -> JudgeResult:
|
||||
errors: list[str] = []
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
result = await judge.judge(case, opensquilla, openclaw)
|
||||
except Exception as exc:
|
||||
errors.append(f"attempt {attempt}: {type(exc).__name__}: {exc}")
|
||||
continue
|
||||
normalized = _normalized_lifestyle_judge_result(result)
|
||||
if normalized is not None:
|
||||
return normalized
|
||||
errors.append(f"attempt {attempt}: incomplete weighted judge payload")
|
||||
raise RuntimeError("; ".join(errors))
|
||||
|
||||
|
||||
def _lifestyle_judge_result_is_complete(judge_result: JudgeResult) -> bool:
|
||||
return _normalized_lifestyle_judge_result(judge_result) is not None
|
||||
|
||||
|
||||
def _normalized_lifestyle_judge_result(judge_result: JudgeResult) -> JudgeResult | None:
|
||||
if not judge_result.rationale.strip():
|
||||
return None
|
||||
raw = judge_result.raw if isinstance(judge_result.raw, dict) else {}
|
||||
totals = _lifestyle_weighted_totals(raw)
|
||||
if totals is None:
|
||||
return None
|
||||
winner = "tie"
|
||||
if totals["opensquilla"] > totals["openclaw"]:
|
||||
winner = "opensquilla"
|
||||
elif totals["openclaw"] > totals["opensquilla"]:
|
||||
winner = "openclaw"
|
||||
normalized_raw = dict(raw)
|
||||
normalized_raw["scores"] = totals
|
||||
normalized_raw["winner"] = winner
|
||||
normalized_raw["score_source"] = "weighted_subscores"
|
||||
return JudgeResult(
|
||||
winner=winner,
|
||||
scores=totals,
|
||||
confidence=judge_result.confidence,
|
||||
rationale=judge_result.rationale,
|
||||
risks=judge_result.risks,
|
||||
raw=normalized_raw,
|
||||
model=judge_result.model,
|
||||
)
|
||||
|
||||
|
||||
def _lifestyle_weighted_totals(raw: dict[str, Any]) -> dict[str, int] | None:
|
||||
subscores = raw.get("subscores") if isinstance(raw.get("subscores"), dict) else {}
|
||||
totals: dict[str, int] = {}
|
||||
for label in ("opensquilla", "openclaw"):
|
||||
candidate = subscores.get(label)
|
||||
if not isinstance(candidate, dict):
|
||||
return None
|
||||
total = 0
|
||||
for name, (low, high) in LIFESTYLE_JUDGE_SUBSCORE_RANGES.items():
|
||||
if name not in candidate:
|
||||
return None
|
||||
try:
|
||||
value = int(candidate[name])
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if value < low or value > high:
|
||||
return None
|
||||
total += value
|
||||
totals[label] = total
|
||||
return totals
|
||||
|
||||
|
||||
def _case_to_dict(case: ComparisonCase) -> dict[str, Any]:
|
||||
data = asdict(case)
|
||||
data["rubric"] = [asdict(item) for item in case.rubric]
|
||||
return data
|
||||
|
||||
|
||||
def _case_from_dict(data: dict[str, Any]) -> ComparisonCase:
|
||||
rubric = tuple(
|
||||
RubricCriterion(
|
||||
name=str(item["name"]),
|
||||
description=str(item["description"]),
|
||||
patterns=tuple(str(pattern) for pattern in item["patterns"]),
|
||||
weight=int(item.get("weight", 1)),
|
||||
)
|
||||
for item in data.get("rubric", ())
|
||||
)
|
||||
return ComparisonCase(
|
||||
case_id=str(data["case_id"]),
|
||||
skill_name=str(data["skill_name"]),
|
||||
prompt=str(data["prompt"]),
|
||||
expected_advantage=str(data["expected_advantage"]),
|
||||
optimization_if_not_better=str(data["optimization_if_not_better"]),
|
||||
scenario=str(data["scenario"]),
|
||||
rubric=rubric,
|
||||
failure_modes=tuple(str(item) for item in data.get("failure_modes", ())),
|
||||
)
|
||||
|
||||
|
||||
def _cases_for_language(language: str) -> list[ComparisonCase]:
|
||||
if language == "zh":
|
||||
return LIFESTYLE_COMPARISON_CASES
|
||||
if language != "en":
|
||||
raise SystemExit(f"Unknown prompt language {language!r}. Valid: zh, en")
|
||||
localized: list[ComparisonCase] = []
|
||||
for case in LIFESTYLE_COMPARISON_CASES:
|
||||
localized.append(
|
||||
ComparisonCase(
|
||||
case_id=f"{case.case_id}_en",
|
||||
skill_name=case.skill_name,
|
||||
prompt=ENGLISH_LIFESTYLE_PROMPTS[case.case_id],
|
||||
expected_advantage=case.expected_advantage,
|
||||
optimization_if_not_better=case.optimization_if_not_better,
|
||||
scenario=f"{case.scenario}_en",
|
||||
rubric=case.rubric,
|
||||
failure_modes=case.failure_modes,
|
||||
)
|
||||
)
|
||||
return localized
|
||||
|
||||
|
||||
def _select_cases(case_arg: str, language: str = "zh") -> list[ComparisonCase]:
|
||||
cases = _cases_for_language(language)
|
||||
if case_arg == "all":
|
||||
return cases
|
||||
selected = [
|
||||
case
|
||||
for case in cases
|
||||
if case.case_id == case_arg or case.case_id.removesuffix("_en") == case_arg
|
||||
]
|
||||
if not selected:
|
||||
valid = ", ".join(case.case_id for case in cases)
|
||||
raise SystemExit(f"Unknown case {case_arg!r}. Valid: {valid}")
|
||||
return selected
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--run-live", action="store_true", help="Run both gateways.")
|
||||
parser.add_argument(
|
||||
"--judge-jsonl",
|
||||
help="Judge an existing lifestyle comparison JSONL without rerunning gateways.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--write-dry-run",
|
||||
action="store_true",
|
||||
help="Write prompt/catalog reports without live gateway calls.",
|
||||
)
|
||||
parser.add_argument("--case", default="all", help="Case id or 'all'.")
|
||||
parser.add_argument("--prompt-language", choices=["zh", "en"], default="zh")
|
||||
parser.add_argument("--timeout", type=float, default=240.0)
|
||||
parser.add_argument("--opensquilla-url", default="ws://127.0.0.1:18791/ws")
|
||||
parser.add_argument("--opensquilla-token", default=read_opensquilla_token())
|
||||
parser.add_argument(
|
||||
"--opensquilla-agent-id",
|
||||
default="main",
|
||||
help="Base OpenSquilla agent id for live runs.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--opensquilla-isolated-agents",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Create a distinct OpenSquilla agent id per case to avoid "
|
||||
"agent-level context pollution."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--opensquilla-run-id",
|
||||
help="Stable run id used in isolated OpenSquilla agent ids.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--opensquilla-elevated",
|
||||
default="bypass",
|
||||
choices=["off", "on", "bypass", "full"],
|
||||
help="Gateway elevated mode for OpenSquilla tool calls.",
|
||||
)
|
||||
parser.add_argument("--openclaw-url", default="ws://127.0.0.1:18789/ws")
|
||||
parser.add_argument("--openclaw-config", default=os.environ.get("OPENCLAW_CONFIG"))
|
||||
parser.add_argument(
|
||||
"--openclaw-baseline-jsonl",
|
||||
help="Reuse OpenClaw results from an existing report; live run only calls OpenSquilla.",
|
||||
)
|
||||
parser.add_argument("--openclaw-idle-timeout", type=float, default=90.0)
|
||||
parser.add_argument("--judge-llm", action="store_true")
|
||||
parser.add_argument("--judge-model", default=os.environ.get("OPENSQUILLA_JUDGE_MODEL"))
|
||||
parser.add_argument("--judge-api-key", default=read_judge_api_key())
|
||||
parser.add_argument(
|
||||
"--judge-base-url",
|
||||
default=os.environ.get("OPENSQUILLA_JUDGE_BASE_URL", "https://openrouter.ai/api/v1"),
|
||||
)
|
||||
parser.add_argument("--judge-timeout", type=float, default=120.0)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
if args.judge_jsonl:
|
||||
asyncio.run(judge_existing(args))
|
||||
return
|
||||
if args.run_live:
|
||||
asyncio.run(run_live(args))
|
||||
return
|
||||
rows = build_lifestyle_rows(args.prompt_language)
|
||||
if args.write_dry_run:
|
||||
write_lifestyle_reports(rows)
|
||||
return
|
||||
print(render_lifestyle_prompts_markdown(rows))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,361 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check Qwen/DashScope provider payload parity invariants from raw traces."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Iterator
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
DASHSCOPE_CACHE_MARKER_LIMIT = 4
|
||||
|
||||
|
||||
def _iter_json_values(path: Path) -> Iterator[Any]:
|
||||
if path.is_dir():
|
||||
for child in sorted(path.rglob("*")):
|
||||
if child.is_file() and child.suffix.lower() in {".json", ".jsonl"}:
|
||||
yield from _iter_json_values(child)
|
||||
return
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return
|
||||
if path.suffix.lower() == ".jsonl":
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
value = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
yield {"__source_path": str(path), "__value": value}
|
||||
return
|
||||
try:
|
||||
value = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
return
|
||||
yield {"__source_path": str(path), "__value": value}
|
||||
|
||||
|
||||
def _walk_dicts(value: Any) -> Iterator[dict[str, Any]]:
|
||||
if isinstance(value, dict):
|
||||
yield value
|
||||
for child in value.values():
|
||||
yield from _walk_dicts(child)
|
||||
elif isinstance(value, list):
|
||||
for child in value:
|
||||
yield from _walk_dicts(child)
|
||||
|
||||
|
||||
def _payloads_from_value(source_path: str, value: Any) -> Iterator[dict[str, Any]]:
|
||||
seen_payload_ids: set[int] = set()
|
||||
for obj in _walk_dicts(value):
|
||||
payload = obj.get("payload")
|
||||
if isinstance(payload, dict) and payload.get("model") and isinstance(
|
||||
payload.get("messages"),
|
||||
list,
|
||||
):
|
||||
seen_payload_ids.add(id(payload))
|
||||
yield {
|
||||
"source_path": source_path,
|
||||
"instance_id": _instance_id_from_path(source_path),
|
||||
"payload": payload,
|
||||
}
|
||||
elif (
|
||||
id(obj) not in seen_payload_ids
|
||||
and obj.get("model")
|
||||
and isinstance(obj.get("messages"), list)
|
||||
):
|
||||
yield {
|
||||
"source_path": source_path,
|
||||
"instance_id": _instance_id_from_path(source_path),
|
||||
"payload": obj,
|
||||
}
|
||||
|
||||
|
||||
def _instance_id_from_path(source_path: str) -> str:
|
||||
path = Path(source_path)
|
||||
if path.name in {"llm_calls.jsonl", "provider_trace.jsonl", "request_proof.jsonl"}:
|
||||
return path.parent.name
|
||||
if path.parent.name:
|
||||
return path.parent.name
|
||||
return ""
|
||||
|
||||
|
||||
def _extra_body(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
extra = payload.get("extra_body")
|
||||
return extra if isinstance(extra, dict) else {}
|
||||
|
||||
|
||||
def _thinking_enabled(payload: dict[str, Any]) -> bool:
|
||||
extra = _extra_body(payload)
|
||||
return bool(
|
||||
extra.get("enable_thinking")
|
||||
or payload.get("enable_thinking")
|
||||
or payload.get("thinking")
|
||||
or payload.get("reasoning")
|
||||
)
|
||||
|
||||
|
||||
def _message_reasoning_replayed(payload: dict[str, Any]) -> bool:
|
||||
for message in payload.get("messages") or []:
|
||||
if isinstance(message, dict) and message.get("role") == "assistant":
|
||||
reasoning = message.get("reasoning_content")
|
||||
if isinstance(reasoning, str) and reasoning.strip():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _tool_call_pairing_ok(payload: dict[str, Any]) -> tuple[bool, str]:
|
||||
pending: list[str] = []
|
||||
for message in payload.get("messages") or []:
|
||||
if not isinstance(message, dict):
|
||||
continue
|
||||
if message.get("role") == "assistant":
|
||||
for call in message.get("tool_calls") or []:
|
||||
if isinstance(call, dict) and call.get("id"):
|
||||
pending.append(str(call["id"]))
|
||||
elif message.get("role") == "tool":
|
||||
tool_call_id = message.get("tool_call_id")
|
||||
if tool_call_id in pending:
|
||||
pending.remove(tool_call_id)
|
||||
if pending:
|
||||
return False, f"unpaired assistant tool_call ids: {','.join(pending[:5])}"
|
||||
return True, "assistant tool calls and tool results are paired"
|
||||
|
||||
|
||||
_BOOLEAN_SCHEMA_KEYWORD_ALLOWLIST = {
|
||||
"additionalProperties",
|
||||
"deprecated",
|
||||
"nullable",
|
||||
"strict",
|
||||
"uniqueItems",
|
||||
}
|
||||
|
||||
|
||||
def _boolean_schema_paths(value: Any, prefix: str = "$", *, key: str | None = None) -> list[str]:
|
||||
if isinstance(value, bool):
|
||||
if key in _BOOLEAN_SCHEMA_KEYWORD_ALLOWLIST:
|
||||
return []
|
||||
return [prefix]
|
||||
if isinstance(value, dict):
|
||||
paths: list[str] = []
|
||||
for key, child in value.items():
|
||||
paths.extend(_boolean_schema_paths(child, f"{prefix}.{key}", key=key))
|
||||
return paths
|
||||
if isinstance(value, list):
|
||||
paths: list[str] = []
|
||||
for index, child in enumerate(value):
|
||||
paths.extend(_boolean_schema_paths(child, f"{prefix}[{index}]"))
|
||||
return paths
|
||||
return []
|
||||
|
||||
|
||||
def _cache_marker_count(payload: dict[str, Any]) -> int:
|
||||
count = 0
|
||||
for obj in _walk_dicts(payload.get("messages") or []):
|
||||
if "cache_control" in obj:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def _row(
|
||||
*,
|
||||
source_path: str,
|
||||
instance_id: str,
|
||||
model: str,
|
||||
check: str,
|
||||
status: str,
|
||||
detail: str,
|
||||
) -> dict[str, str]:
|
||||
return {
|
||||
"source_path": source_path,
|
||||
"instance_id": instance_id,
|
||||
"model": model,
|
||||
"check": check,
|
||||
"status": status,
|
||||
"detail": detail,
|
||||
}
|
||||
|
||||
|
||||
def _check_payload(
|
||||
source_path: str,
|
||||
instance_id: str,
|
||||
payload: dict[str, Any],
|
||||
) -> list[dict[str, str]]:
|
||||
model = str(payload.get("model") or "")
|
||||
model_lower = model.lower()
|
||||
qwen_flash = "qwen3.6-flash" in model_lower
|
||||
thinking = _thinking_enabled(payload)
|
||||
extra = _extra_body(payload)
|
||||
rows: list[dict[str, str]] = []
|
||||
|
||||
def add(check: str, status: str, detail: str) -> None:
|
||||
rows.append(
|
||||
_row(
|
||||
source_path=source_path,
|
||||
instance_id=instance_id,
|
||||
model=model,
|
||||
check=check,
|
||||
status=status,
|
||||
detail=detail,
|
||||
)
|
||||
)
|
||||
|
||||
if thinking:
|
||||
add(
|
||||
"dashscope_enable_thinking",
|
||||
"pass"
|
||||
if extra.get("enable_thinking") is True or payload.get("enable_thinking") is True
|
||||
else "fail",
|
||||
"enable_thinking is true"
|
||||
if extra.get("enable_thinking") is True or payload.get("enable_thinking") is True
|
||||
else "thinking appears enabled but enable_thinking is not true",
|
||||
)
|
||||
add(
|
||||
"dashscope_max_completion_tokens",
|
||||
"pass" if "max_completion_tokens" in payload else "fail",
|
||||
"max_completion_tokens present"
|
||||
if "max_completion_tokens" in payload
|
||||
else "DashScope reasoning payload should use max_completion_tokens",
|
||||
)
|
||||
forced_tool_choice = payload.get("tool_choice")
|
||||
forced_tool_choice_allowed = forced_tool_choice is None or forced_tool_choice == "auto"
|
||||
add(
|
||||
"dashscope_thinking_no_forced_tool_choice",
|
||||
"pass" if forced_tool_choice_allowed else "fail",
|
||||
"no forced tool_choice during thinking"
|
||||
if forced_tool_choice_allowed
|
||||
else "forced tool_choice present during thinking",
|
||||
)
|
||||
else:
|
||||
add("dashscope_enable_thinking", "skip", "thinking not detected")
|
||||
add("dashscope_max_completion_tokens", "skip", "thinking not detected")
|
||||
add("dashscope_thinking_no_forced_tool_choice", "skip", "thinking not detected")
|
||||
|
||||
if qwen_flash:
|
||||
add(
|
||||
"qwen_flash_no_reasoning_replay",
|
||||
"fail" if _message_reasoning_replayed(payload) else "pass",
|
||||
"historical assistant reasoning_content replayed"
|
||||
if _message_reasoning_replayed(payload)
|
||||
else "no historical reasoning_content replay",
|
||||
)
|
||||
preserve_thinking = bool(extra.get("preserve_thinking") or payload.get("preserve_thinking"))
|
||||
add(
|
||||
"qwen_flash_no_preserve_thinking",
|
||||
"fail" if preserve_thinking else "pass",
|
||||
"preserve_thinking present for qwen3.6-flash"
|
||||
if preserve_thinking
|
||||
else "preserve_thinking absent for qwen3.6-flash",
|
||||
)
|
||||
else:
|
||||
add("qwen_flash_no_reasoning_replay", "skip", "not qwen3.6-flash")
|
||||
add("qwen_flash_no_preserve_thinking", "skip", "not qwen3.6-flash")
|
||||
|
||||
stream_options = payload.get("stream_options")
|
||||
if payload.get("stream") is False:
|
||||
add("stream_include_usage", "skip", "non-stream request")
|
||||
else:
|
||||
include_usage = (
|
||||
isinstance(stream_options, dict) and stream_options.get("include_usage") is True
|
||||
)
|
||||
add(
|
||||
"stream_include_usage",
|
||||
"pass" if include_usage else "fail",
|
||||
"stream_options.include_usage is true"
|
||||
if include_usage
|
||||
else "stream_options.include_usage is missing or false",
|
||||
)
|
||||
|
||||
marker_count = _cache_marker_count(payload)
|
||||
if marker_count == 0:
|
||||
add("cache_marker_limit", "warn", "no cache markers found")
|
||||
else:
|
||||
add(
|
||||
"cache_marker_limit",
|
||||
"pass" if marker_count <= DASHSCOPE_CACHE_MARKER_LIMIT else "fail",
|
||||
f"cache markers={marker_count}, limit={DASHSCOPE_CACHE_MARKER_LIMIT}",
|
||||
)
|
||||
|
||||
paired, detail = _tool_call_pairing_ok(payload)
|
||||
add("tool_call_pairing", "pass" if paired else "fail", detail)
|
||||
|
||||
boolean_paths = _boolean_schema_paths(payload.get("tools") or [])
|
||||
add(
|
||||
"tool_schema_no_boolean_values",
|
||||
"pass" if not boolean_paths else "fail",
|
||||
"no boolean schema values"
|
||||
if not boolean_paths
|
||||
else "boolean schema values at " + ",".join(boolean_paths[:5]),
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def analyze_paths(paths: Iterable[Path]) -> tuple[dict[str, Any], list[dict[str, str]]]:
|
||||
rows: list[dict[str, str]] = []
|
||||
checked_payloads = 0
|
||||
for path in paths:
|
||||
for wrapped in _iter_json_values(path):
|
||||
source_path = str(wrapped.get("__source_path") or path)
|
||||
for item in _payloads_from_value(source_path, wrapped.get("__value")):
|
||||
checked_payloads += 1
|
||||
rows.extend(
|
||||
_check_payload(
|
||||
item["source_path"],
|
||||
item["instance_id"],
|
||||
item["payload"],
|
||||
)
|
||||
)
|
||||
failures = Counter(row["check"] for row in rows if row["status"] == "fail")
|
||||
warnings = Counter(row["check"] for row in rows if row["status"] == "warn")
|
||||
summary = {
|
||||
"checked_payloads": checked_payloads,
|
||||
"rows": len(rows),
|
||||
"failed_checks": sum(failures.values()),
|
||||
"warning_checks": sum(warnings.values()),
|
||||
"failed_checks_by_name": dict(sorted(failures.items())),
|
||||
"warnings_by_name": dict(sorted(warnings.items())),
|
||||
}
|
||||
return summary, rows
|
||||
|
||||
|
||||
def write_outputs(
|
||||
summary: dict[str, Any],
|
||||
rows: list[dict[str, str]],
|
||||
*,
|
||||
json_path: Path,
|
||||
csv_path: Path,
|
||||
) -> None:
|
||||
json_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
csv_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
json_path.write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
fields = ["source_path", "instance_id", "model", "check", "status", "detail"]
|
||||
with csv_path.open("w", encoding="utf-8", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=fields, extrasaction="ignore")
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("paths", nargs="+", type=Path)
|
||||
parser.add_argument("--json-output", type=Path, default=Path("qwen_payload_parity.json"))
|
||||
parser.add_argument("--csv-output", type=Path, default=Path("qwen_payload_parity.csv"))
|
||||
args = parser.parse_args()
|
||||
summary, rows = analyze_paths(args.paths)
|
||||
write_outputs(summary, rows, json_path=args.json_output, csv_path=args.csv_output)
|
||||
print(json.dumps(summary, ensure_ascii=False, sort_keys=True))
|
||||
return 1 if summary["failed_checks"] else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,198 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from statistics import mean
|
||||
from typing import Any
|
||||
|
||||
DASHSCOPE_DUPLICATE_MARKER = "duplicate tool interaction omitted"
|
||||
PROVIDER_COMPACTION_MARKER = "Historical tool call omitted for provider context budget"
|
||||
BARE_THINK_CLOSE_MARKER = "</think>"
|
||||
|
||||
|
||||
def _iter_jsonl(path: Path) -> list[dict[str, Any]]:
|
||||
if not path.exists():
|
||||
return []
|
||||
rows: list[dict[str, Any]] = []
|
||||
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(row, dict):
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def _row_text(row: dict[str, Any]) -> str:
|
||||
return json.dumps(row, ensure_ascii=False, sort_keys=True)
|
||||
|
||||
|
||||
def _numeric_timestamp(row: dict[str, Any]) -> float | None:
|
||||
for key in ("ts", "timestamp", "time", "created_at"):
|
||||
value = row.get(key)
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return float(value)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _instance_dirs(root: Path) -> list[Path]:
|
||||
if not root.exists():
|
||||
return []
|
||||
return sorted(path for path in root.iterdir() if path.is_dir())
|
||||
|
||||
|
||||
def _latency_summary(values: list[float]) -> dict[str, float | int | None]:
|
||||
if not values:
|
||||
return {"count": 0, "mean": None, "min": None, "max": None}
|
||||
ordered = sorted(values)
|
||||
return {
|
||||
"count": len(ordered),
|
||||
"mean": round(mean(ordered), 3),
|
||||
"min": round(ordered[0], 3),
|
||||
"max": round(ordered[-1], 3),
|
||||
}
|
||||
|
||||
|
||||
def _prediction_summary(predictions_path: Path | None) -> dict[str, Any]:
|
||||
rows = _iter_jsonl(predictions_path) if predictions_path is not None else []
|
||||
submitted = 0
|
||||
empty = 0
|
||||
instance_ids: set[str] = set()
|
||||
for row in rows:
|
||||
submitted += 1
|
||||
instance_id = row.get("instance_id")
|
||||
if isinstance(instance_id, str):
|
||||
instance_ids.add(instance_id)
|
||||
model_patch = row.get("model_patch")
|
||||
if not isinstance(model_patch, str) or not model_patch.strip():
|
||||
empty += 1
|
||||
return {
|
||||
"submitted": submitted,
|
||||
"unique_instance_ids": len(instance_ids),
|
||||
"empty_model_patch": empty,
|
||||
}
|
||||
|
||||
|
||||
def analyze_artifact_root(
|
||||
artifact_root: str | Path,
|
||||
*,
|
||||
predictions_path: str | Path | None = None,
|
||||
) -> dict[str, Any]:
|
||||
root = Path(artifact_root)
|
||||
prediction_path = Path(predictions_path) if predictions_path is not None else None
|
||||
instances = _instance_dirs(root)
|
||||
signals = {
|
||||
"dashscope_duplicate_omission": 0,
|
||||
"provider_compaction_omission": 0,
|
||||
"bare_think_close": 0,
|
||||
}
|
||||
patches = {"empty_git_patch": 0, "present_git_patch": 0}
|
||||
llm = {
|
||||
"requests": 0,
|
||||
"responses": 0,
|
||||
"response_chunks": 0,
|
||||
"errors": 0,
|
||||
"status_429": 0,
|
||||
"status_5xx": 0,
|
||||
"timeout_errors": 0,
|
||||
}
|
||||
request_ts_by_id: dict[str, float] = {}
|
||||
first_chunk_ts_by_id: dict[str, float] = {}
|
||||
|
||||
for instance_dir in instances:
|
||||
transcript_text = ""
|
||||
transcript_path = instance_dir / "transcript.jsonl"
|
||||
if transcript_path.exists():
|
||||
transcript_text = transcript_path.read_text(
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
signals["dashscope_duplicate_omission"] += transcript_text.count(
|
||||
DASHSCOPE_DUPLICATE_MARKER
|
||||
)
|
||||
signals["provider_compaction_omission"] += transcript_text.count(
|
||||
PROVIDER_COMPACTION_MARKER
|
||||
)
|
||||
signals["bare_think_close"] += transcript_text.count(BARE_THINK_CLOSE_MARKER)
|
||||
|
||||
patch_path = instance_dir / "git.patch"
|
||||
if patch_path.exists():
|
||||
patches["present_git_patch"] += 1
|
||||
if not patch_path.read_text(encoding="utf-8", errors="replace").strip():
|
||||
patches["empty_git_patch"] += 1
|
||||
|
||||
for row in _iter_jsonl(instance_dir / "llm_calls.jsonl"):
|
||||
event = str(row.get("event") or "")
|
||||
if event == "llm.request":
|
||||
llm["requests"] += 1
|
||||
request_id = row.get("request_id")
|
||||
ts = _numeric_timestamp(row)
|
||||
if isinstance(request_id, str) and ts is not None:
|
||||
request_ts_by_id.setdefault(request_id, ts)
|
||||
elif event == "llm.response":
|
||||
llm["responses"] += 1
|
||||
elif event == "llm.response_chunk":
|
||||
llm["response_chunks"] += 1
|
||||
request_id = row.get("request_id")
|
||||
ts = _numeric_timestamp(row)
|
||||
if isinstance(request_id, str) and ts is not None:
|
||||
first_chunk_ts_by_id.setdefault(request_id, ts)
|
||||
elif event == "llm.error":
|
||||
llm["errors"] += 1
|
||||
|
||||
status_code = row.get("status_code")
|
||||
if status_code == 429:
|
||||
llm["status_429"] += 1
|
||||
if isinstance(status_code, int) and 500 <= status_code <= 599:
|
||||
llm["status_5xx"] += 1
|
||||
text = _row_text(row).lower()
|
||||
if event == "llm.error" and "timeout" in text:
|
||||
llm["timeout_errors"] += 1
|
||||
|
||||
latencies = [
|
||||
first_ts - request_ts
|
||||
for request_id, request_ts in request_ts_by_id.items()
|
||||
if (first_ts := first_chunk_ts_by_id.get(request_id)) is not None
|
||||
and first_ts >= request_ts
|
||||
]
|
||||
llm["first_chunk_latency_seconds"] = _latency_summary(latencies)
|
||||
|
||||
return {
|
||||
"artifact_root": str(root),
|
||||
"instances": len(instances),
|
||||
"predictions": _prediction_summary(prediction_path),
|
||||
"patches": patches,
|
||||
"signals": signals,
|
||||
"llm": llm,
|
||||
}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Analyze Qwen/DashScope provider-visible run artifact risks.",
|
||||
)
|
||||
parser.add_argument("artifact_root", type=Path)
|
||||
parser.add_argument("--predictions", type=Path, default=None)
|
||||
parser.add_argument("--pretty", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
summary = analyze_artifact_root(
|
||||
args.artifact_root,
|
||||
predictions_path=args.predictions,
|
||||
)
|
||||
indent = 2 if args.pretty else None
|
||||
print(json.dumps(summary, ensure_ascii=False, sort_keys=True, indent=indent))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify eval Docker image tags against a digest lock."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
data = json.load(handle)
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"lock file must contain a JSON object: {path}")
|
||||
return data
|
||||
|
||||
|
||||
def _read_instance_ids(paths: list[Path]) -> list[str]:
|
||||
instance_ids: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for path in paths:
|
||||
for raw_line in path.read_text(encoding="utf-8").splitlines():
|
||||
instance_id = raw_line.strip()
|
||||
if not instance_id or instance_id.startswith("#") or instance_id in seen:
|
||||
continue
|
||||
seen.add(instance_id)
|
||||
instance_ids.append(instance_id)
|
||||
return instance_ids
|
||||
|
||||
|
||||
def _records_by_instance(lock: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
records = lock.get("records")
|
||||
if not isinstance(records, list):
|
||||
raise ValueError("lock file is missing records[]")
|
||||
indexed: dict[str, dict[str, Any]] = {}
|
||||
for record in records:
|
||||
if not isinstance(record, dict):
|
||||
raise ValueError("lock records must be JSON objects")
|
||||
instance_id = record.get("instance_id")
|
||||
if not isinstance(instance_id, str) or not instance_id:
|
||||
raise ValueError("lock record missing instance_id")
|
||||
indexed[instance_id] = record
|
||||
return indexed
|
||||
|
||||
|
||||
def _inspect_image_id(image_ref: str) -> tuple[int, str, str]:
|
||||
proc = subprocess.run(
|
||||
["docker", "inspect", image_ref, "--format", "{{.Id}}"],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
return proc.returncode, proc.stdout.strip(), proc.stderr.strip()
|
||||
|
||||
|
||||
def _expected_instances(
|
||||
records: dict[str, dict[str, Any]],
|
||||
instance_files: list[Path],
|
||||
) -> list[str]:
|
||||
if instance_files:
|
||||
return _read_instance_ids(instance_files)
|
||||
return sorted(records)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--lock",
|
||||
required=True,
|
||||
type=Path,
|
||||
help="Path to the image lock JSON.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--instance-file",
|
||||
action="append",
|
||||
default=[],
|
||||
type=Path,
|
||||
help="Instance id file to verify. May be repeated. Defaults to all lock records.",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
try:
|
||||
lock = _read_json(args.lock)
|
||||
records = _records_by_instance(lock)
|
||||
instance_ids = _expected_instances(records, args.instance_file)
|
||||
except Exception as exc:
|
||||
print(f"invalid_lock: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
errors = 0
|
||||
for instance_id in instance_ids:
|
||||
record = records.get(instance_id)
|
||||
if record is None:
|
||||
print(f"missing_lock_record: {instance_id}", file=sys.stderr)
|
||||
errors += 1
|
||||
continue
|
||||
image_ref = record.get("image_ref")
|
||||
expected_id = record.get("image_id")
|
||||
if not isinstance(image_ref, str) or not isinstance(expected_id, str):
|
||||
print(f"invalid_lock_record: {instance_id}", file=sys.stderr)
|
||||
errors += 1
|
||||
continue
|
||||
returncode, actual_id, stderr = _inspect_image_id(image_ref)
|
||||
if returncode != 0:
|
||||
print(f"missing_image: {instance_id} {image_ref} {stderr}", file=sys.stderr)
|
||||
errors += 1
|
||||
continue
|
||||
if actual_id != expected_id:
|
||||
print(
|
||||
f"digest_mismatch: {instance_id} {image_ref} "
|
||||
f"expected={expected_id} actual={actual_id}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
errors += 1
|
||||
|
||||
if errors:
|
||||
print(f"checked={len(instance_ids)} errors={errors}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(f"checked={len(instance_ids)} errors=0 tag={lock.get('tag', '')}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Assert LLM treatment delivery from per-instance llm_calls.jsonl records.
|
||||
|
||||
Experiment arms must verify that configured treatments actually reached the
|
||||
provider before a run is scored; any delivery mismatch makes the decision
|
||||
``invalid``. This scans every ``llm.request`` record in each instance's
|
||||
llm_calls.jsonl and asserts, on all requests:
|
||||
|
||||
- ``metadata.request_proof.proof_budget`` equals --expected-proof-budget
|
||||
- ``payload.reasoning.effort`` equals --expected-reasoning-effort
|
||||
(OpenRouter/GLM adapter: effort string; payloads carry no numeric budget)
|
||||
- ``payload.thinking_budget`` equals --expected-thinking-budget
|
||||
(DashScope/Qwen adapter: numeric budget)
|
||||
|
||||
The engine has a designed one-shot recovery that retries a failed provider
|
||||
call with thinking disabled; the request shape is adapter-specific
|
||||
(provider/openai.py): OpenRouter/GLM emits ``payload.reasoning = {"enabled":
|
||||
false}``; DashScope/Qwen emits ``payload.enable_thinking = false`` with no
|
||||
reasoning key and no thinking_budget. Such requests are counted separately
|
||||
and excluded from the reasoning-effort and thinking-budget assertions, but
|
||||
any occurrence beyond --allow-reasoning-fallbacks (default 0) is an error.
|
||||
The proof-budget assertion still applies to them. (Detection assumes a
|
||||
thinking-enabled arm; a deliberately thinking-off DashScope arm would count
|
||||
every request as a fallback.)
|
||||
|
||||
Separately, an httpx stream timeout with no stream event triggers a
|
||||
non-stream retry of the same budget-coordinated payload
|
||||
(``_complete_non_stream``); its record carries ``metadata.fallback_from ==
|
||||
"stream_timeout"`` and no ``request_proof`` block at all. These records are
|
||||
excluded from the proof-budget assertion (the payload assertions still
|
||||
apply), reported per instance as ``stream_fallbacks``, and never gated —
|
||||
the treatment itself was delivered unchanged.
|
||||
|
||||
One stdout line per instance reports the request count and distinct observed
|
||||
values. Exit is non-zero on any mismatch, unreadable request record, or
|
||||
instance with zero ``llm.request`` records.
|
||||
|
||||
Known limit: lines are prefiltered on the substring ``"llm.request"`` before
|
||||
JSON parsing, so a request line truncated within its first ~200 bytes (before
|
||||
the ``event`` key) is skipped silently rather than counted as unparsed. Tail
|
||||
truncation from a killed run cuts inside the large ``payload`` field instead,
|
||||
which still matches the prefilter and lands in the unparsed-error path — and a
|
||||
killed run is already invalid under the rc!=0 rule.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _instance_dirs(run_dir: Path) -> list[Path]:
|
||||
if (run_dir / "llm_calls.jsonl").is_file():
|
||||
return [run_dir]
|
||||
return sorted(child for child in run_dir.iterdir() if child.is_dir())
|
||||
|
||||
|
||||
class _InstanceScan:
|
||||
def __init__(self) -> None:
|
||||
self.requests = 0
|
||||
self.unparsed = 0
|
||||
self.reasoning_fallbacks = 0
|
||||
self.stream_fallbacks = 0
|
||||
self.proof_budgets: set[object] = set()
|
||||
self.efforts: set[object] = set()
|
||||
self.thinking_budgets: set[object] = set()
|
||||
|
||||
|
||||
def _scan_llm_requests(path: Path) -> _InstanceScan:
|
||||
scan = _InstanceScan()
|
||||
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
||||
for line in handle:
|
||||
if '"llm.request"' not in line:
|
||||
continue
|
||||
try:
|
||||
record = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
scan.unparsed += 1
|
||||
continue
|
||||
if not isinstance(record, dict) or record.get("event") != "llm.request":
|
||||
continue
|
||||
scan.requests += 1
|
||||
metadata = record.get("metadata")
|
||||
metadata = metadata if isinstance(metadata, dict) else {}
|
||||
payload = record.get("payload")
|
||||
payload = payload if isinstance(payload, dict) else {}
|
||||
if metadata.get("fallback_from") == "stream_timeout":
|
||||
# Non-stream retry of a stream timeout re-sends the same
|
||||
# budget-coordinated payload, but its record carries no
|
||||
# request_proof metadata (provider/openai.py record_request).
|
||||
scan.stream_fallbacks += 1
|
||||
else:
|
||||
request_proof = metadata.get("request_proof")
|
||||
scan.proof_budgets.add(
|
||||
request_proof.get("proof_budget") if isinstance(request_proof, dict) else None
|
||||
)
|
||||
reasoning = payload.get("reasoning")
|
||||
if reasoning == {"enabled": False} or payload.get("enable_thinking") is False:
|
||||
# Exact shapes the engine's one-shot thinking-disable recovery
|
||||
# emits (provider/openai.py): OpenRouter reasoning={"enabled":
|
||||
# false}; DashScope enable_thinking=false with no reasoning key
|
||||
# and no thinking_budget. Anything else is a treatment value.
|
||||
scan.reasoning_fallbacks += 1
|
||||
else:
|
||||
scan.efforts.add(reasoning.get("effort") if isinstance(reasoning, dict) else None)
|
||||
scan.thinking_budgets.add(payload.get("thinking_budget"))
|
||||
return scan
|
||||
|
||||
|
||||
def _distinct(values: set[object]) -> str:
|
||||
return ",".join(sorted("null" if value is None else str(value) for value in values))
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--run-dir",
|
||||
action="append",
|
||||
required=True,
|
||||
type=Path,
|
||||
help=(
|
||||
"Run directory holding per-instance subdirectories with llm_calls.jsonl "
|
||||
"(or a single instance directory). May be repeated."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--expected-proof-budget",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Expected metadata.request_proof.proof_budget on every llm.request.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--expected-reasoning-effort",
|
||||
default=None,
|
||||
help="Expected payload.reasoning.effort string on every llm.request.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--expected-thinking-budget",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Expected numeric payload.thinking_budget on every llm.request.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--allow-reasoning-fallbacks",
|
||||
type=int,
|
||||
default=0,
|
||||
help=(
|
||||
"Max engine thinking-disable recovery requests tolerated per "
|
||||
"instance (OpenRouter payload.reasoning.enabled == false, or "
|
||||
"DashScope payload.enable_thinking == false)."
|
||||
),
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if (
|
||||
args.expected_proof_budget is None
|
||||
and args.expected_reasoning_effort is None
|
||||
and args.expected_thinking_budget is None
|
||||
):
|
||||
parser.error("at least one --expected-* assertion is required")
|
||||
|
||||
checked = 0
|
||||
errors = 0
|
||||
for run_dir in args.run_dir:
|
||||
if not run_dir.is_dir():
|
||||
print(f"missing_run_dir: {run_dir}", file=sys.stderr)
|
||||
errors += 1
|
||||
continue
|
||||
instance_dirs = _instance_dirs(run_dir)
|
||||
if not instance_dirs:
|
||||
print(f"no_instances: {run_dir}", file=sys.stderr)
|
||||
errors += 1
|
||||
continue
|
||||
for instance_dir in instance_dirs:
|
||||
checked += 1
|
||||
instance_id = instance_dir.name
|
||||
calls_path = instance_dir / "llm_calls.jsonl"
|
||||
if not calls_path.is_file():
|
||||
print(f"missing_llm_calls: {instance_id}", file=sys.stderr)
|
||||
errors += 1
|
||||
continue
|
||||
scan = _scan_llm_requests(calls_path)
|
||||
print(
|
||||
f"instance={instance_id} requests={scan.requests} "
|
||||
f"proof_budget={_distinct(scan.proof_budgets)} "
|
||||
f"reasoning_effort={_distinct(scan.efforts)} "
|
||||
f"reasoning_fallbacks={scan.reasoning_fallbacks} "
|
||||
f"stream_fallbacks={scan.stream_fallbacks} "
|
||||
f"thinking_budget={_distinct(scan.thinking_budgets)}"
|
||||
)
|
||||
if scan.unparsed:
|
||||
print(
|
||||
f"unparsed_request_lines: {instance_id} count={scan.unparsed}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
errors += 1
|
||||
if scan.requests == 0:
|
||||
print(f"no_llm_requests: {instance_id}", file=sys.stderr)
|
||||
errors += 1
|
||||
continue
|
||||
if scan.reasoning_fallbacks > args.allow_reasoning_fallbacks:
|
||||
print(
|
||||
f"reasoning_fallback_exceeded: {instance_id} "
|
||||
f"count={scan.reasoning_fallbacks} "
|
||||
f"allowed={args.allow_reasoning_fallbacks}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
errors += 1
|
||||
if args.expected_proof_budget is not None and scan.proof_budgets != {
|
||||
args.expected_proof_budget
|
||||
}:
|
||||
print(
|
||||
f"proof_budget_mismatch: {instance_id} "
|
||||
f"expected={args.expected_proof_budget} "
|
||||
f"actual={_distinct(scan.proof_budgets)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
errors += 1
|
||||
if args.expected_reasoning_effort is not None and scan.efforts != {
|
||||
args.expected_reasoning_effort
|
||||
}:
|
||||
print(
|
||||
f"reasoning_effort_mismatch: {instance_id} "
|
||||
f"expected={args.expected_reasoning_effort} "
|
||||
f"actual={_distinct(scan.efforts)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
errors += 1
|
||||
if args.expected_thinking_budget is not None and scan.thinking_budgets != {
|
||||
args.expected_thinking_budget
|
||||
}:
|
||||
print(
|
||||
f"thinking_budget_mismatch: {instance_id} "
|
||||
f"expected={args.expected_thinking_budget} "
|
||||
f"actual={_distinct(scan.thinking_budgets)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
errors += 1
|
||||
|
||||
if errors:
|
||||
print(f"checked={checked} errors={errors}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(f"checked={checked} errors=0")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,416 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared helpers for the OpenSquilla experiment ledger."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
DEFAULT_LEDGER_ROOT = Path("./.experiments-ledger")
|
||||
RUNNER_RELATIVE_PATH = Path("scripts/run_tool_policy_validation_stdin_keys.sh")
|
||||
|
||||
# Docker needles for run supervision; overridable per run via the top-level
|
||||
# manifest keys ``container_name_prefix`` and ``eval_image_needle``.
|
||||
CONTAINER_NAME_PREFIX = "opensquilla-swe-"
|
||||
EVAL_IMAGE_NEEDLE = "sweb.eval."
|
||||
|
||||
# Default provider secret env var required per model family; overridable via
|
||||
# the ``--required-secret-env`` CLI flag on exp_init.
|
||||
DEFAULT_REQUIRED_SECRET_ENV = {
|
||||
"qwen": "DASHSCOPE_API_KEY",
|
||||
"glm": "OPENROUTER_API_KEY",
|
||||
}
|
||||
|
||||
EXP_ID_RE = re.compile(r"^[a-z0-9][a-z0-9._-]*$")
|
||||
SECRET_KEY_RE = re.compile(r"(api[_-]?key|token|secret|password|credential)", re.I)
|
||||
TRACKED_ENV_PREFIXES = ("OPENSQUILLA_",)
|
||||
|
||||
|
||||
class LedgerError(RuntimeError):
|
||||
"""Raised for user-correctable ledger command failures."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GitInfo:
|
||||
path: str
|
||||
branch: str
|
||||
head: str
|
||||
short_head: str
|
||||
dirty_count: int
|
||||
dirty_summary: list[str]
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return datetime.now(UTC).astimezone().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def ledger_root_from_env() -> Path:
|
||||
return Path(
|
||||
os.environ.get(
|
||||
"OPENSQUILLA_EXPERIMENT_LEDGER_ROOT",
|
||||
os.environ.get("OPENSQUILLA_SWE_EXPERIMENT_LEDGER_ROOT", DEFAULT_LEDGER_ROOT),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def validate_exp_id(exp_id: str) -> str:
|
||||
if not EXP_ID_RE.fullmatch(exp_id):
|
||||
raise LedgerError("exp_id must match [a-z0-9][a-z0-9._-]*")
|
||||
return exp_id
|
||||
|
||||
|
||||
def ensure_ledger_layout(root: Path) -> None:
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
(root / "runs").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def ledger_lock(root: Path) -> Iterator[None]:
|
||||
ensure_ledger_layout(root)
|
||||
lock_path = root / ".lock"
|
||||
with lock_path.open("a", encoding="utf-8") as handle:
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def atomic_write_text(path: Path, text: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
"w",
|
||||
encoding="utf-8",
|
||||
dir=str(path.parent),
|
||||
prefix=f".{path.name}.",
|
||||
delete=False,
|
||||
) as handle:
|
||||
handle.write(text)
|
||||
tmp = Path(handle.name)
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def atomic_write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
atomic_write_text(path, json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def read_json(path: Path, default: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {} if default is None else dict(default)
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return {} if default is None else dict(default)
|
||||
return data if isinstance(data, dict) else ({} if default is None else dict(default))
|
||||
|
||||
|
||||
def read_json_strict(path: Path, *, label: str = "JSON file") -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
raise LedgerError(f"missing {label}: {path}")
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise LedgerError(f"invalid {label}: {path}: {exc}") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise LedgerError(f"{label} must contain a JSON object: {path}")
|
||||
return data
|
||||
|
||||
|
||||
def append_jsonl(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(payload, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def run_command(args: list[str], *, cwd: Path | None = None) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(args, cwd=cwd, text=True, capture_output=True, check=False)
|
||||
|
||||
|
||||
def git_info(path: Path, *, max_dirty_lines: int = 20) -> GitInfo:
|
||||
if not path.exists():
|
||||
raise LedgerError(f"path does not exist: {path}")
|
||||
head = _git_stdout(path, ["rev-parse", "HEAD"])
|
||||
short_head = _git_stdout(path, ["rev-parse", "--short", "HEAD"])
|
||||
branch_proc = run_command(["git", "branch", "--show-current"], cwd=path)
|
||||
branch = branch_proc.stdout.strip() if branch_proc.returncode == 0 else ""
|
||||
dirty_proc = run_command(["git", "status", "--short"], cwd=path)
|
||||
if dirty_proc.returncode != 0:
|
||||
raise LedgerError(f"git status failed for {path}: {dirty_proc.stderr.strip()}")
|
||||
dirty_lines = [line for line in dirty_proc.stdout.splitlines() if line.strip()]
|
||||
return GitInfo(
|
||||
path=str(path),
|
||||
branch=branch,
|
||||
head=head,
|
||||
short_head=short_head,
|
||||
dirty_count=len(dirty_lines),
|
||||
dirty_summary=dirty_lines[:max_dirty_lines],
|
||||
)
|
||||
|
||||
|
||||
def _git_stdout(path: Path, args: list[str]) -> str:
|
||||
proc = run_command(["git", *args], cwd=path)
|
||||
if proc.returncode != 0:
|
||||
raise LedgerError(f"git {' '.join(args)} failed for {path}: {proc.stderr.strip()}")
|
||||
return proc.stdout.strip()
|
||||
|
||||
|
||||
def copy_snapshot(src: Path, dst_dir: Path) -> dict[str, str]:
|
||||
if not src.is_file():
|
||||
raise LedgerError(f"snapshot source must be a file: {src}")
|
||||
dst_dir.mkdir(parents=True, exist_ok=True)
|
||||
dst = dst_dir / src.name
|
||||
shutil.copy2(src, dst)
|
||||
return {"source": str(src), "snapshot": str(dst), "sha256": sha256_file(src)}
|
||||
|
||||
|
||||
def parse_key_value_file(path: Path) -> dict[str, str]:
|
||||
result: dict[str, str] = {}
|
||||
if not path.exists():
|
||||
return result
|
||||
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
result[key.strip()] = value.strip()
|
||||
return result
|
||||
|
||||
|
||||
def parse_env_overrides(items: list[str]) -> dict[str, dict[str, Any]]:
|
||||
parsed: dict[str, dict[str, Any]] = {}
|
||||
for item in items:
|
||||
if "=" not in item:
|
||||
raise LedgerError(f"--env must use KEY=VALUE form: {item}")
|
||||
key, value = item.split("=", 1)
|
||||
key = key.strip()
|
||||
if not key:
|
||||
raise LedgerError("--env key cannot be empty")
|
||||
parsed[key] = redact_env_value(key, value)
|
||||
return parsed
|
||||
|
||||
|
||||
def redact_env_value(key: str, value: str | None = None) -> dict[str, Any]:
|
||||
if SECRET_KEY_RE.search(key):
|
||||
return {"required": True, "provided_at_init": bool(value), "redacted": True}
|
||||
if key.startswith(TRACKED_ENV_PREFIXES):
|
||||
return {"value": value or "", "redacted": False}
|
||||
return {"value": value or "", "redacted": False}
|
||||
|
||||
|
||||
def env_exports_for_command(env: dict[str, dict[str, Any]]) -> list[str]:
|
||||
exports: list[str] = []
|
||||
for key, meta in sorted(env.items()):
|
||||
if meta.get("redacted"):
|
||||
continue
|
||||
value = str(meta.get("value", ""))
|
||||
exports.append(f"export {key}={sh_quote(value)}")
|
||||
return exports
|
||||
|
||||
|
||||
def sh_quote(value: str) -> str:
|
||||
return "'" + value.replace("'", "'\"'\"'") + "'"
|
||||
|
||||
|
||||
def required_secret_env(
|
||||
run_mode: str, mapping: dict[str, str] | None = None
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
if mapping is None:
|
||||
mapping = DEFAULT_REQUIRED_SECRET_ENV
|
||||
required: dict[str, dict[str, Any]] = {}
|
||||
if run_mode != "glm_only":
|
||||
key = mapping["qwen"]
|
||||
required[key] = redact_env_value(key)
|
||||
if run_mode != "qwen_only":
|
||||
key = mapping["glm"]
|
||||
required[key] = redact_env_value(key)
|
||||
return required
|
||||
|
||||
|
||||
def read_first_existing_report(paths_file: Path) -> list[str]:
|
||||
if not paths_file.exists():
|
||||
return []
|
||||
reports = []
|
||||
for line in paths_file.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
candidate = line.strip()
|
||||
if candidate and Path(candidate).is_file():
|
||||
reports.append(candidate)
|
||||
return reports
|
||||
|
||||
|
||||
def collect_eval_metrics(report_paths: list[str]) -> dict[str, Any]:
|
||||
totals = {
|
||||
"total_instances": 0,
|
||||
"submitted_instances": 0,
|
||||
"completed_instances": 0,
|
||||
"resolved_instances": 0,
|
||||
"unresolved_instances": 0,
|
||||
"empty_patch_instances": 0,
|
||||
"error_instances": 0,
|
||||
}
|
||||
resolved_ids: list[str] = []
|
||||
empty_patch_ids: list[str] = []
|
||||
error_ids: list[str] = []
|
||||
reports: list[dict[str, Any]] = []
|
||||
for path_str in report_paths:
|
||||
path = Path(path_str)
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
continue
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
for key in totals:
|
||||
value = data.get(key)
|
||||
if isinstance(value, int):
|
||||
totals[key] += value
|
||||
resolved_ids.extend(_string_list(data.get("resolved_ids")))
|
||||
empty_patch_ids.extend(_string_list(data.get("empty_patch_ids")))
|
||||
error_ids.extend(_string_list(data.get("error_ids")))
|
||||
reports.append({"path": str(path), "schema_version": data.get("schema_version", "")})
|
||||
return {
|
||||
**totals,
|
||||
"resolved_ids": sorted(set(resolved_ids)),
|
||||
"empty_patch_ids": sorted(set(empty_patch_ids)),
|
||||
"error_ids": sorted(set(error_ids)),
|
||||
"report_count": len(reports),
|
||||
"reports": reports,
|
||||
}
|
||||
|
||||
|
||||
def _string_list(value: Any) -> list[str]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [item for item in value if isinstance(item, str)]
|
||||
|
||||
|
||||
def active_processes() -> list[dict[str, Any]]:
|
||||
proc = run_command(["ps", "-eo", "pid=,args="])
|
||||
if proc.returncode != 0:
|
||||
return []
|
||||
needles = (
|
||||
"run_tool_policy_validation",
|
||||
"run_infer.py",
|
||||
"run_eval.py",
|
||||
"swebench.harness.run_evaluation",
|
||||
)
|
||||
rows = []
|
||||
self_pid = os.getpid()
|
||||
for line in proc.stdout.splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
pid_text, _, args = stripped.partition(" ")
|
||||
try:
|
||||
pid = int(pid_text)
|
||||
except ValueError:
|
||||
continue
|
||||
if pid == self_pid:
|
||||
continue
|
||||
if any(needle in args for needle in needles):
|
||||
rows.append({"pid": pid, "args": args[:500]})
|
||||
return rows
|
||||
|
||||
|
||||
def active_swe_containers(manifest: dict[str, Any] | None = None) -> list[str]:
|
||||
config = manifest if isinstance(manifest, dict) else {}
|
||||
container_prefix = str(config.get("container_name_prefix") or CONTAINER_NAME_PREFIX)
|
||||
eval_image_needle = str(config.get("eval_image_needle") or EVAL_IMAGE_NEEDLE)
|
||||
proc = run_command(["docker", "ps", "--format", "{{.Names}}"])
|
||||
if proc.returncode != 0:
|
||||
return []
|
||||
return [
|
||||
line.strip()
|
||||
for line in proc.stdout.splitlines()
|
||||
if line.startswith(container_prefix) or line.startswith(eval_image_needle)
|
||||
]
|
||||
|
||||
|
||||
def status_label_from_dirty(info: GitInfo) -> str:
|
||||
return "clean" if info.dirty_count == 0 else f"dirty {info.dirty_count}"
|
||||
|
||||
|
||||
def exp_dir(root: Path, exp_id: str) -> Path:
|
||||
return root / "runs" / validate_exp_id(exp_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Contamination registry (quarantined runs)
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# ``contaminations.json`` at the ledger root maps a contamination class (e.g.
|
||||
# ``tool_result_compaction_defect``) to the artifact batch names whose
|
||||
# results are confounded by an infra defect. Baseline and A/B tooling must
|
||||
# exclude quarantined artifacts; ``exp_status`` warns when a recorded
|
||||
# baseline references one.
|
||||
|
||||
CONTAMINATIONS_FILENAME = "contaminations.json"
|
||||
|
||||
|
||||
def contaminations_path(root: Path) -> Path:
|
||||
return root / CONTAMINATIONS_FILENAME
|
||||
|
||||
|
||||
def load_contaminations(root: Path) -> dict[str, Any]:
|
||||
data = read_json(contaminations_path(root), default={})
|
||||
classes = data.get("classes")
|
||||
if not isinstance(classes, dict):
|
||||
data["classes"] = {}
|
||||
data.setdefault("version", 1)
|
||||
return data
|
||||
|
||||
|
||||
def artifact_basename(artifact: str | Path) -> str:
|
||||
"""Normalize an artifact path or batch name to its bare batch name."""
|
||||
return str(artifact).rstrip("/").rsplit("/", 1)[-1]
|
||||
|
||||
|
||||
def contamination_class_for(
|
||||
root: Path,
|
||||
artifact: str | Path,
|
||||
contaminations: dict[str, Any] | None = None,
|
||||
) -> str | None:
|
||||
"""Return the contamination class covering ``artifact``, or None if clean."""
|
||||
name = artifact_basename(artifact)
|
||||
if not name:
|
||||
return None
|
||||
data = contaminations if contaminations is not None else load_contaminations(root)
|
||||
classes = data.get("classes")
|
||||
if not isinstance(classes, dict):
|
||||
return None
|
||||
for class_name, payload in sorted(classes.items()):
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
names = payload.get("artifact_names")
|
||||
if isinstance(names, list) and name in names:
|
||||
return class_name
|
||||
return None
|
||||
|
||||
|
||||
def run_dir_contamination_classes(root: Path, exp_id: str) -> list[str]:
|
||||
"""Return contamination classes stamped on a ledger run dir (empty if clean)."""
|
||||
if not exp_id or not EXP_ID_RE.fullmatch(exp_id):
|
||||
return []
|
||||
stamp = read_json(root / "runs" / exp_id / "contamination.json")
|
||||
classes = stamp.get("classes")
|
||||
if not isinstance(classes, dict):
|
||||
return []
|
||||
return sorted(name for name in classes if isinstance(name, str))
|
||||
@@ -0,0 +1,684 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Finalize an OpenSquilla experiment from handoff artifacts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from exp_common import (
|
||||
LedgerError,
|
||||
append_jsonl,
|
||||
atomic_write_json,
|
||||
atomic_write_text,
|
||||
collect_eval_metrics,
|
||||
contamination_class_for,
|
||||
exp_dir,
|
||||
ledger_lock,
|
||||
ledger_root_from_env,
|
||||
now_iso,
|
||||
parse_key_value_file,
|
||||
read_first_existing_report,
|
||||
read_json,
|
||||
read_json_strict,
|
||||
run_dir_contamination_classes,
|
||||
sha256_file,
|
||||
validate_exp_id,
|
||||
)
|
||||
|
||||
DECISIONS = {"adopted", "rejected", "observe", "inconclusive", "invalid", "stopped"}
|
||||
NO_VALID_EVAL_DECISIONS = {"invalid", "stopped"}
|
||||
AGENT_ENV_DELIVERY_VARS = frozenset(
|
||||
{
|
||||
"OPENSQUILLA_DASHSCOPE_THINKING_BUDGET",
|
||||
"OPENSQUILLA_FINAL_DIFF_CONTRACT_MODE",
|
||||
"OPENSQUILLA_FINALIZE_EVIDENCE_GATE",
|
||||
"OPENSQUILLA_PATCH_EVIDENCE_PROTOCOL",
|
||||
"OPENSQUILLA_PROVIDER_COMPACTION_TINY_GUARD_CHARS",
|
||||
"OPENSQUILLA_PROVIDER_COMPACTION_PROTECT_RECENT_ASSISTANT",
|
||||
"OPENSQUILLA_PROVIDER_CONTEXT_BLOCK_FEEDBACK",
|
||||
"OPENSQUILLA_IDENTICAL_REQUEST_LOOP_BREAK",
|
||||
"OPENSQUILLA_PLACEHOLDER_ESCALATION_THRESHOLD",
|
||||
"OPENSQUILLA_DEADLINE_WRAPUP_MARGIN_SECONDS",
|
||||
"OPENSQUILLA_TOOL_REPEAT_NUDGE_THRESHOLD",
|
||||
"OPENSQUILLA_TOOL_REPEAT_NUDGE_TOOLS",
|
||||
"OPENSQUILLA_PROVIDER_HISTORY_DEDUP",
|
||||
"OPENSQUILLA_PROVIDER_HISTORY_DEDUP_MIN_REPEATS",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--exp-id", required=True)
|
||||
parser.add_argument("--batch-dir", type=Path, required=True)
|
||||
parser.add_argument("--decision", required=True, choices=sorted(DECISIONS))
|
||||
parser.add_argument("--decision-reason", required=True)
|
||||
parser.add_argument("--mechanism", default="")
|
||||
parser.add_argument("--baseline-model", choices=["qwen", "glm"], default="")
|
||||
parser.add_argument("--overwrite-decision", action="store_true")
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
finalize(args)
|
||||
except LedgerError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
return 0
|
||||
|
||||
|
||||
def finalize(args: argparse.Namespace) -> None:
|
||||
exp_id = validate_exp_id(args.exp_id)
|
||||
ledger_root = ledger_root_from_env()
|
||||
run_dir = exp_dir(ledger_root, exp_id)
|
||||
manifest = read_json_strict(run_dir / "manifest.json", label="experiment manifest")
|
||||
if (run_dir / "decision.md").exists() and not args.overwrite_decision:
|
||||
raise LedgerError("decision already exists; pass --overwrite-decision to replace")
|
||||
if not args.batch_dir.is_dir():
|
||||
raise LedgerError(f"batch dir does not exist: {args.batch_dir}")
|
||||
|
||||
artifacts = collect_artifacts(args.batch_dir)
|
||||
validate_batch_matches_manifest(manifest, artifacts)
|
||||
artifacts["env_delivery"] = collect_env_delivery(manifest, artifacts)
|
||||
metrics = collect_metrics(artifacts)
|
||||
eval_valid = (
|
||||
metrics["eval_report_count"] > 0
|
||||
and not metrics["nonzero_eval_exit_codes"]
|
||||
and not metrics["nonzero_infer_exit_codes"]
|
||||
and not metrics["nonzero_other_exit_codes"]
|
||||
)
|
||||
if not eval_valid and args.decision not in NO_VALID_EVAL_DECISIONS:
|
||||
raise LedgerError(
|
||||
"missing/nonzero infer or eval results can only be finalized as invalid or stopped"
|
||||
)
|
||||
env_delivery_errors = artifacts.get("env_delivery", {}).get("errors", [])
|
||||
if env_delivery_errors and args.decision not in NO_VALID_EVAL_DECISIONS:
|
||||
raise LedgerError(
|
||||
"manifest-pinned runtime env was not delivered to agent instances: "
|
||||
+ "; ".join(str(error) for error in env_delivery_errors)
|
||||
)
|
||||
if args.decision == "adopted" and args.baseline_model:
|
||||
contamination_classes = _batch_contamination_classes(ledger_root, exp_id, artifacts)
|
||||
if contamination_classes:
|
||||
raise LedgerError(
|
||||
"cannot adopt as baseline: batch is quarantined ("
|
||||
+ ", ".join(contamination_classes)
|
||||
+ "); re-baseline on clean runs"
|
||||
)
|
||||
|
||||
finished_at = now_iso()
|
||||
decision_payload = {
|
||||
"exp_id": exp_id,
|
||||
"decision": args.decision,
|
||||
"reason": args.decision_reason,
|
||||
"mechanism": args.mechanism,
|
||||
"baseline_model": args.baseline_model,
|
||||
"decided_at": finished_at,
|
||||
}
|
||||
with ledger_lock(ledger_root):
|
||||
atomic_write_json(run_dir / "artifacts.json", artifacts)
|
||||
atomic_write_json(run_dir / "metrics.json", metrics)
|
||||
atomic_write_text(run_dir / "analysis.md", render_analysis(manifest, artifacts, metrics))
|
||||
atomic_write_text(run_dir / "decision.md", render_decision(decision_payload, metrics))
|
||||
update_current(ledger_root, exp_id, args.decision, metrics, finished_at)
|
||||
if args.decision == "adopted" and args.baseline_model:
|
||||
update_baseline(
|
||||
ledger_root,
|
||||
args.baseline_model,
|
||||
exp_id,
|
||||
manifest,
|
||||
metrics,
|
||||
args.decision_reason,
|
||||
)
|
||||
if args.mechanism:
|
||||
update_mechanism(
|
||||
ledger_root,
|
||||
args.mechanism,
|
||||
args.decision,
|
||||
exp_id,
|
||||
args.decision_reason,
|
||||
)
|
||||
append_jsonl(
|
||||
ledger_root / "experiments.jsonl",
|
||||
{
|
||||
"time": finished_at,
|
||||
"exp_id": exp_id,
|
||||
"event": "finalized",
|
||||
"decision": args.decision,
|
||||
"resolved": metrics["resolved_instances"],
|
||||
"total": metrics["total_instances"],
|
||||
"empty": metrics["empty_patch_instances"],
|
||||
"batch_dir": str(args.batch_dir),
|
||||
},
|
||||
)
|
||||
print(json.dumps({"exp_id": exp_id, "decision": args.decision, "metrics": metrics}, indent=2))
|
||||
|
||||
|
||||
def _batch_contamination_classes(
|
||||
ledger_root: Path, exp_id: str, artifacts: dict[str, Any]
|
||||
) -> list[str]:
|
||||
"""Return contamination classes covering this batch, by name or by stamped run dir."""
|
||||
classes = set(run_dir_contamination_classes(ledger_root, exp_id))
|
||||
batch_dir = str(artifacts.get("batch_dir") or "").strip()
|
||||
if batch_dir:
|
||||
batch_class = contamination_class_for(ledger_root, batch_dir)
|
||||
if batch_class:
|
||||
classes.add(batch_class)
|
||||
return sorted(classes)
|
||||
|
||||
|
||||
def collect_artifacts(batch_dir: Path) -> dict[str, Any]:
|
||||
manifest_txt = batch_dir / "manifest.txt"
|
||||
batch_manifest = parse_key_value_file(manifest_txt)
|
||||
report_paths: list[str] = []
|
||||
for path_file in sorted(batch_dir.glob("*-eval.report_paths.txt")):
|
||||
report_paths.extend(read_first_existing_report(path_file))
|
||||
exit_codes = {}
|
||||
for exit_file in sorted(batch_dir.glob("*.exit_code")):
|
||||
text = exit_file.read_text(encoding="utf-8", errors="replace").strip()
|
||||
try:
|
||||
value: int | str = int(text)
|
||||
except ValueError:
|
||||
value = text
|
||||
exit_codes[exit_file.name] = value
|
||||
return {
|
||||
"batch_dir": str(batch_dir),
|
||||
"batch_manifest_path": str(manifest_txt) if manifest_txt.exists() else "",
|
||||
"batch_manifest": batch_manifest,
|
||||
"eval_report_paths": sorted(set(report_paths)),
|
||||
"exit_codes": exit_codes,
|
||||
"supervisor_log": str(batch_dir / "supervisor.log")
|
||||
if (batch_dir / "supervisor.log").exists()
|
||||
else "",
|
||||
}
|
||||
|
||||
|
||||
def collect_env_delivery(
|
||||
manifest: dict[str, Any],
|
||||
artifacts: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
expected = _manifest_agent_env_expectations(manifest)
|
||||
delivery: dict[str, Any] = {
|
||||
"expected": expected,
|
||||
"checked_instance_count": 0,
|
||||
"run_dirs": [],
|
||||
"per_var": {
|
||||
name: {
|
||||
"expected": value,
|
||||
"matched": 0,
|
||||
"missing": 0,
|
||||
"mismatch": 0,
|
||||
}
|
||||
for name, value in expected.items()
|
||||
},
|
||||
"errors": [],
|
||||
}
|
||||
if not expected:
|
||||
return delivery
|
||||
|
||||
run_dirs = _agent_run_dirs(artifacts)
|
||||
delivery["run_dirs"] = [str(path) for path in run_dirs]
|
||||
if not run_dirs:
|
||||
delivery["errors"].append(
|
||||
"no agent run dirs resolved from batch manifest; env delivery could not be "
|
||||
"verified for expected vars: " + ", ".join(sorted(expected))
|
||||
)
|
||||
return delivery
|
||||
|
||||
metadata_paths: list[Path] = []
|
||||
for run_dir in run_dirs:
|
||||
if not run_dir.is_dir():
|
||||
delivery["errors"].append(f"run artifact dir not found: {run_dir}")
|
||||
continue
|
||||
metadata_paths.extend(sorted(run_dir.glob("*/metadata.json")))
|
||||
if not metadata_paths:
|
||||
delivery["errors"].append("no instance metadata found for env delivery check")
|
||||
return delivery
|
||||
|
||||
for metadata_path in metadata_paths:
|
||||
delivery["checked_instance_count"] += 1
|
||||
try:
|
||||
payload = json.loads(metadata_path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError, UnicodeDecodeError):
|
||||
delivery["errors"].append(f"metadata is not valid JSON: {metadata_path}")
|
||||
continue
|
||||
forwarded_env = (
|
||||
((payload.get("agent") or {}).get("controls") or {}).get(
|
||||
"progress_watchdog_env"
|
||||
)
|
||||
or {}
|
||||
)
|
||||
if not isinstance(forwarded_env, dict):
|
||||
delivery["errors"].append(
|
||||
f"progress_watchdog_env is not an object: {metadata_path}"
|
||||
)
|
||||
forwarded_env = {}
|
||||
for name, expected_value in expected.items():
|
||||
stats = delivery["per_var"][name]
|
||||
if name not in forwarded_env:
|
||||
stats["missing"] += 1
|
||||
elif str(forwarded_env.get(name)) != expected_value:
|
||||
stats["mismatch"] += 1
|
||||
else:
|
||||
stats["matched"] += 1
|
||||
|
||||
for name, stats in delivery["per_var"].items():
|
||||
missing = int(stats["missing"])
|
||||
mismatch = int(stats["mismatch"])
|
||||
if missing or mismatch:
|
||||
delivery["errors"].append(
|
||||
f"{name} expected {stats['expected']!r}: "
|
||||
f"{missing} missing, {mismatch} mismatched across "
|
||||
f"{delivery['checked_instance_count']} metadata files"
|
||||
)
|
||||
return delivery
|
||||
|
||||
|
||||
def _manifest_agent_env_expectations(manifest: dict[str, Any]) -> dict[str, str]:
|
||||
env = manifest.get("config", {}).get("env", {})
|
||||
if not isinstance(env, dict):
|
||||
return {}
|
||||
expected: dict[str, str] = {}
|
||||
for name, payload in env.items():
|
||||
if name not in AGENT_ENV_DELIVERY_VARS:
|
||||
continue
|
||||
if not isinstance(payload, dict) or payload.get("redacted"):
|
||||
continue
|
||||
value = payload.get("value")
|
||||
if value is not None:
|
||||
expected[name] = str(value)
|
||||
return expected
|
||||
|
||||
|
||||
def _agent_run_dirs(artifacts: dict[str, Any]) -> list[Path]:
|
||||
batch_dir_text = str(artifacts.get("batch_dir") or "").strip()
|
||||
if not batch_dir_text:
|
||||
return []
|
||||
batch_dir = Path(batch_dir_text)
|
||||
batch_manifest = artifacts.get("batch_manifest") or {}
|
||||
run_mode = str(batch_manifest.get("run_mode") or "")
|
||||
keys: list[tuple[str, str]] = []
|
||||
if run_mode != "glm_only":
|
||||
keys.extend(
|
||||
[
|
||||
("qwen_ml_run_id", "ml_ids"),
|
||||
("qwen_verified_run_id", "verified_ids"),
|
||||
]
|
||||
)
|
||||
if run_mode != "qwen_only":
|
||||
keys.extend(
|
||||
[
|
||||
("glm_ml_run_id", "ml_ids"),
|
||||
("glm_verified_run_id", "verified_ids"),
|
||||
]
|
||||
)
|
||||
roots: list[Path] = []
|
||||
for key, ids_key in keys:
|
||||
if not str(batch_manifest.get(ids_key) or "").strip():
|
||||
continue
|
||||
run_id = str(batch_manifest.get(key) or "")
|
||||
if run_id:
|
||||
roots.append(batch_dir.parent / run_id)
|
||||
return roots
|
||||
|
||||
|
||||
def validate_batch_matches_manifest(manifest: dict[str, Any], artifacts: dict[str, Any]) -> None:
|
||||
batch_manifest = artifacts.get("batch_manifest", {})
|
||||
if not batch_manifest:
|
||||
raise LedgerError("batch manifest.txt is missing or empty")
|
||||
|
||||
errors: list[str] = []
|
||||
_expect_equal(
|
||||
errors,
|
||||
"opensquilla_source_head",
|
||||
batch_manifest.get("opensquilla_source_head"),
|
||||
manifest.get("source", {}).get("head"),
|
||||
)
|
||||
_expect_equal(
|
||||
errors,
|
||||
"handoff_head",
|
||||
batch_manifest.get("handoff_head"),
|
||||
manifest.get("handoff", {}).get("head"),
|
||||
)
|
||||
_expect_equal(
|
||||
errors,
|
||||
"condition_label",
|
||||
batch_manifest.get("condition_label"),
|
||||
manifest.get("config", {}).get("condition_label"),
|
||||
)
|
||||
_expect_equal(
|
||||
errors,
|
||||
"run_mode",
|
||||
batch_manifest.get("run_mode"),
|
||||
manifest.get("execution", {}).get("run_mode"),
|
||||
)
|
||||
for key in ("qwen_workers", "glm_workers", "eval_workers"):
|
||||
_expect_equal(
|
||||
errors,
|
||||
key,
|
||||
batch_manifest.get(key),
|
||||
str(manifest.get("execution", {}).get(key, "")),
|
||||
)
|
||||
|
||||
run_mode = str(manifest.get("execution", {}).get("run_mode", ""))
|
||||
if run_mode != "glm_only":
|
||||
_expect_equal(
|
||||
errors,
|
||||
"qwen_config_sha256",
|
||||
batch_manifest.get("qwen_config_sha256"),
|
||||
manifest.get("config", {}).get("qwen_config", {}).get("sha256"),
|
||||
)
|
||||
if run_mode != "qwen_only":
|
||||
_expect_equal(
|
||||
errors,
|
||||
"glm_config_sha256",
|
||||
batch_manifest.get("glm_config_sha256"),
|
||||
manifest.get("config", {}).get("glm_config", {}).get("sha256"),
|
||||
)
|
||||
|
||||
_expect_one_of(
|
||||
errors,
|
||||
"ml_instance_file",
|
||||
batch_manifest.get("ml_instance_file"),
|
||||
_path_candidates(manifest.get("slice", {}).get("ml", {})),
|
||||
)
|
||||
_expect_one_of(
|
||||
errors,
|
||||
"verified_instance_file",
|
||||
batch_manifest.get("verified_instance_file"),
|
||||
_path_candidates(manifest.get("slice", {}).get("verified", {})),
|
||||
)
|
||||
|
||||
_verify_slice_content(
|
||||
errors,
|
||||
"ml_instance_file",
|
||||
batch_manifest.get("ml_instance_file"),
|
||||
manifest.get("slice", {}).get("ml", {}),
|
||||
)
|
||||
_verify_slice_content(
|
||||
errors,
|
||||
"verified_instance_file",
|
||||
batch_manifest.get("verified_instance_file"),
|
||||
manifest.get("slice", {}).get("verified", {}),
|
||||
)
|
||||
_verify_batch_selected_id_count(
|
||||
errors,
|
||||
"ml_ids",
|
||||
batch_manifest.get("ml_ids"),
|
||||
manifest.get("slice", {}).get("ml", {}),
|
||||
)
|
||||
_verify_batch_selected_id_count(
|
||||
errors,
|
||||
"verified_ids",
|
||||
batch_manifest.get("verified_ids"),
|
||||
manifest.get("slice", {}).get("verified", {}),
|
||||
)
|
||||
|
||||
_verify_runner_sha(errors, batch_manifest, manifest)
|
||||
|
||||
if errors:
|
||||
raise LedgerError("batch does not match experiment manifest: " + "; ".join(errors))
|
||||
|
||||
|
||||
def _expect_equal(errors: list[str], label: str, actual: str | None, expected: Any) -> None:
|
||||
expected_text = "" if expected is None else str(expected)
|
||||
actual_text = "" if actual is None else str(actual)
|
||||
if not actual_text:
|
||||
errors.append(f"{label} missing")
|
||||
elif actual_text != expected_text:
|
||||
errors.append(f"{label}={actual_text!r} expected {expected_text!r}")
|
||||
|
||||
|
||||
def _expect_one_of(
|
||||
errors: list[str],
|
||||
label: str,
|
||||
actual: str | None,
|
||||
expected_values: set[str],
|
||||
) -> None:
|
||||
actual_text = "" if actual is None else str(actual)
|
||||
if not actual_text:
|
||||
errors.append(f"{label} missing")
|
||||
elif actual_text not in expected_values:
|
||||
expected = ", ".join(sorted(expected_values))
|
||||
errors.append(f"{label}={actual_text!r} expected one of [{expected}]")
|
||||
|
||||
|
||||
def _path_candidates(payload: dict[str, Any]) -> set[str]:
|
||||
return {str(payload.get(key)) for key in ("source", "snapshot") if payload.get(key)}
|
||||
|
||||
|
||||
def _verify_slice_content(
|
||||
errors: list[str],
|
||||
label: str,
|
||||
actual_path: str | None,
|
||||
slice_payload: dict[str, Any],
|
||||
) -> None:
|
||||
"""Confirm the batch's instance file matches the manifest by content, not just path.
|
||||
|
||||
The batch manifest.txt records only the instance-file path; a path match alone does
|
||||
not prove the file's contents (or slice size) are the ones the manifest pinned.
|
||||
"""
|
||||
path_text = "" if actual_path is None else str(actual_path)
|
||||
if not path_text:
|
||||
return # missing path already reported by _expect_one_of
|
||||
path = Path(path_text)
|
||||
if not path.is_file():
|
||||
errors.append(f"{label} not readable for content check: {path}")
|
||||
return
|
||||
expected_sha = str(slice_payload.get("sha256") or "")
|
||||
if expected_sha and sha256_file(path) != expected_sha:
|
||||
errors.append(f"{label} sha256 changed since manifest creation")
|
||||
expected_count = slice_payload.get("count")
|
||||
if isinstance(expected_count, int):
|
||||
if expected_count == 0:
|
||||
return
|
||||
actual_count = _count_instances(path)
|
||||
if actual_count != expected_count:
|
||||
errors.append(
|
||||
f"{label} instance count {actual_count} != manifest {expected_count}"
|
||||
)
|
||||
|
||||
|
||||
def _count_instances(path: Path) -> int:
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
return sum(1 for line in text.splitlines() if line.strip())
|
||||
|
||||
|
||||
def _verify_batch_selected_id_count(
|
||||
errors: list[str],
|
||||
label: str,
|
||||
actual_ids: str | None,
|
||||
slice_payload: dict[str, Any],
|
||||
) -> None:
|
||||
expected_count = slice_payload.get("count")
|
||||
if not isinstance(expected_count, int):
|
||||
return
|
||||
actual = 0 if actual_ids is None else len(str(actual_ids).split())
|
||||
if actual != expected_count:
|
||||
errors.append(f"{label} selected count {actual} != manifest {expected_count}")
|
||||
|
||||
|
||||
def _verify_runner_sha(
|
||||
errors: list[str],
|
||||
batch_manifest: dict[str, Any],
|
||||
manifest: dict[str, Any],
|
||||
) -> None:
|
||||
"""Opportunistically confirm the batch was produced by the manifest-pinned runner.
|
||||
|
||||
The handoff runner does not emit a runner sha into manifest.txt today, so this is a
|
||||
no-op unless a ``runner_sha256``/``handoff_runner_sha256`` field is present. Runner
|
||||
integrity at launch time is enforced separately in exp_run.py.
|
||||
"""
|
||||
expected = str(manifest.get("config", {}).get("runner", {}).get("sha256") or "")
|
||||
if not expected:
|
||||
return
|
||||
actual = (
|
||||
batch_manifest.get("runner_sha256")
|
||||
or batch_manifest.get("handoff_runner_sha256")
|
||||
or ""
|
||||
)
|
||||
actual_text = str(actual)
|
||||
if actual_text and actual_text != expected:
|
||||
errors.append("runner_sha256 does not match manifest runner")
|
||||
|
||||
|
||||
def collect_metrics(artifacts: dict[str, Any]) -> dict[str, Any]:
|
||||
eval_metrics = collect_eval_metrics(artifacts.get("eval_report_paths", []))
|
||||
exit_codes = artifacts.get("exit_codes", {})
|
||||
nonzero = {name: value for name, value in exit_codes.items() if value != 0}
|
||||
nonzero_eval = {
|
||||
name: value for name, value in nonzero.items() if name.endswith("-eval.exit_code")
|
||||
}
|
||||
nonzero_infer = {
|
||||
name: value for name, value in nonzero.items() if name.endswith(".infer.exit_code")
|
||||
}
|
||||
# Any other nonzero exit file (nonstandard/unexpected name) must still gate validity;
|
||||
# silently ignoring it could let a broken run be finalized as adopted/rejected.
|
||||
nonzero_other = {
|
||||
name: value
|
||||
for name, value in nonzero.items()
|
||||
if name not in nonzero_eval and name not in nonzero_infer
|
||||
}
|
||||
total = int(eval_metrics.get("total_instances") or 0)
|
||||
resolved = int(eval_metrics.get("resolved_instances") or 0)
|
||||
empty = int(eval_metrics.get("empty_patch_instances") or 0)
|
||||
return {
|
||||
**eval_metrics,
|
||||
"eval_report_count": int(eval_metrics.get("report_count") or 0),
|
||||
"env_delivery_error_count": len(
|
||||
artifacts.get("env_delivery", {}).get("errors", [])
|
||||
),
|
||||
"resolved_rate": (resolved / total) if total else None,
|
||||
"empty_rate": (empty / total) if total else None,
|
||||
"nonzero_eval_exit_codes": nonzero_eval,
|
||||
"nonzero_infer_exit_codes": nonzero_infer,
|
||||
"nonzero_other_exit_codes": nonzero_other,
|
||||
}
|
||||
|
||||
|
||||
def render_analysis(
|
||||
manifest: dict[str, Any],
|
||||
artifacts: dict[str, Any],
|
||||
metrics: dict[str, Any],
|
||||
) -> str:
|
||||
resolved = f"{metrics.get('resolved_instances', 0)}/{metrics.get('total_instances', 0)}"
|
||||
env_delivery = artifacts.get("env_delivery") or {}
|
||||
env_lines: list[str] = []
|
||||
if env_delivery.get("expected"):
|
||||
env_lines = [
|
||||
"- Runtime env delivery checked: "
|
||||
f"{env_delivery.get('checked_instance_count', 0)} metadata files",
|
||||
f"- Runtime env delivery errors: {len(env_delivery.get('errors', []))}",
|
||||
]
|
||||
for error in env_delivery.get("errors", []):
|
||||
env_lines.append(f" - {error}")
|
||||
return "\n".join(
|
||||
[
|
||||
f"# SWE Experiment Analysis: {manifest.get('exp_id')}",
|
||||
"",
|
||||
f"- Question: {manifest.get('question', '')}",
|
||||
f"- Batch: `{artifacts.get('batch_dir', '')}`",
|
||||
f"- Source HEAD: `{manifest.get('source', {}).get('head', '')}`",
|
||||
f"- Handoff HEAD: `{manifest.get('handoff', {}).get('head', '')}`",
|
||||
f"- Eval reports: {metrics.get('eval_report_count', 0)}",
|
||||
f"- Resolved: {resolved}",
|
||||
f"- Empty patches: {metrics.get('empty_patch_instances', 0)}",
|
||||
f"- Errors: {metrics.get('error_instances', 0)}",
|
||||
*env_lines,
|
||||
"",
|
||||
"This report is generated from the manifest and batch artifacts; "
|
||||
"raw traces remain in place.",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def render_decision(decision: dict[str, Any], metrics: dict[str, Any]) -> str:
|
||||
resolved = f"{metrics.get('resolved_instances', 0)}/{metrics.get('total_instances', 0)}"
|
||||
return "\n".join(
|
||||
[
|
||||
f"# Decision: {decision['decision']}",
|
||||
"",
|
||||
f"- Experiment: `{decision['exp_id']}`",
|
||||
f"- Reason: {decision['reason']}",
|
||||
f"- Resolved: {resolved}",
|
||||
f"- Empty patches: {metrics.get('empty_patch_instances', 0)}",
|
||||
f"- Decided at: {decision['decided_at']}",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def update_current(
|
||||
ledger_root: Path,
|
||||
exp_id: str,
|
||||
decision: str,
|
||||
metrics: dict[str, Any],
|
||||
updated_at: str,
|
||||
) -> None:
|
||||
current = read_json(ledger_root / "current.json")
|
||||
if current.get("active_experiment") == exp_id:
|
||||
current["active_experiment"] = None
|
||||
current.update(
|
||||
{
|
||||
"updated_at": updated_at,
|
||||
"last_experiment": exp_id,
|
||||
"last_decision": decision,
|
||||
"last_result": {
|
||||
"resolved": metrics.get("resolved_instances", 0),
|
||||
"total": metrics.get("total_instances", 0),
|
||||
"empty": metrics.get("empty_patch_instances", 0),
|
||||
},
|
||||
}
|
||||
)
|
||||
atomic_write_json(ledger_root / "current.json", current)
|
||||
|
||||
|
||||
def update_baseline(
|
||||
ledger_root: Path,
|
||||
model: str,
|
||||
exp_id: str,
|
||||
manifest: dict[str, Any],
|
||||
metrics: dict[str, Any],
|
||||
reason: str,
|
||||
) -> None:
|
||||
baselines = read_json(ledger_root / "baselines.json")
|
||||
baselines[model] = {
|
||||
"current_best": {
|
||||
"exp_id": exp_id,
|
||||
"label": manifest.get("config", {}).get("condition_label", exp_id),
|
||||
"source_head": manifest.get("source", {}).get("head", ""),
|
||||
"resolved": metrics.get("resolved_instances", 0),
|
||||
"total": metrics.get("total_instances", 0),
|
||||
"empty": metrics.get("empty_patch_instances", 0),
|
||||
"reason": reason,
|
||||
}
|
||||
}
|
||||
atomic_write_json(ledger_root / "baselines.json", baselines)
|
||||
|
||||
|
||||
def update_mechanism(
|
||||
ledger_root: Path,
|
||||
mechanism: str,
|
||||
decision: str,
|
||||
exp_id: str,
|
||||
reason: str,
|
||||
) -> None:
|
||||
mechanisms = read_json(ledger_root / "mechanisms.json")
|
||||
mechanisms[mechanism] = {
|
||||
"status": decision,
|
||||
"exp_id": exp_id,
|
||||
"reason": reason,
|
||||
"updated_at": now_iso(),
|
||||
}
|
||||
atomic_write_json(ledger_root / "mechanisms.json", mechanisms)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,292 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create a reproducible OpenSquilla experiment manifest."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import stat
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from exp_common import (
|
||||
DEFAULT_REQUIRED_SECRET_ENV,
|
||||
RUNNER_RELATIVE_PATH,
|
||||
LedgerError,
|
||||
append_jsonl,
|
||||
atomic_write_json,
|
||||
atomic_write_text,
|
||||
copy_snapshot,
|
||||
env_exports_for_command,
|
||||
exp_dir,
|
||||
git_info,
|
||||
ledger_lock,
|
||||
ledger_root_from_env,
|
||||
now_iso,
|
||||
parse_env_overrides,
|
||||
required_secret_env,
|
||||
sh_quote,
|
||||
sha256_file,
|
||||
validate_exp_id,
|
||||
)
|
||||
|
||||
RUN_MODES = {"qwen_only", "glm_only", "both"}
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--exp-id", required=True)
|
||||
parser.add_argument("--question", required=True)
|
||||
parser.add_argument("--hypothesis", default="")
|
||||
parser.add_argument("--condition-label", required=True)
|
||||
parser.add_argument("--run-mode", required=True, choices=sorted(RUN_MODES))
|
||||
parser.add_argument("--source-root", type=Path, required=True)
|
||||
parser.add_argument("--handoff-root", type=Path, required=True)
|
||||
parser.add_argument("--qwen-config", type=Path, required=True)
|
||||
parser.add_argument("--glm-config", type=Path, required=True)
|
||||
parser.add_argument("--ml-instance-file", type=Path, required=True)
|
||||
parser.add_argument("--verified-instance-file", type=Path, required=True)
|
||||
parser.add_argument("--ml-count", type=int, required=True)
|
||||
parser.add_argument("--verified-count", type=int, required=True)
|
||||
parser.add_argument("--qwen-workers", type=int, required=True)
|
||||
parser.add_argument("--glm-workers", type=int, required=True)
|
||||
parser.add_argument("--eval-workers", type=int, required=True)
|
||||
parser.add_argument("--env", action="append", default=[])
|
||||
parser.add_argument(
|
||||
"--required-secret-env",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="MODEL=ENV_VAR",
|
||||
help="Override the required provider secret env var for a model (qwen=..., glm=...).",
|
||||
)
|
||||
parser.add_argument("--decision-gate", action="append", default=[])
|
||||
parser.add_argument("--allow-handoff-dirty", action="store_true")
|
||||
parser.add_argument("--resume-existing", action="store_true")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
create_experiment(args)
|
||||
except LedgerError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
return 0
|
||||
|
||||
|
||||
def create_experiment(args: argparse.Namespace) -> None:
|
||||
exp_id = validate_exp_id(args.exp_id)
|
||||
ledger_root = ledger_root_from_env()
|
||||
run_dir = exp_dir(ledger_root, exp_id)
|
||||
with ledger_lock(ledger_root):
|
||||
if run_dir.exists() and not args.resume_existing and not args.dry_run:
|
||||
raise LedgerError(f"experiment already exists: {run_dir}")
|
||||
|
||||
source = git_info(args.source_root)
|
||||
handoff = git_info(args.handoff_root)
|
||||
if source.dirty_count:
|
||||
raise LedgerError("source repo is dirty; refusing to create experiment manifest")
|
||||
if handoff.dirty_count and not args.allow_handoff_dirty:
|
||||
raise LedgerError("handoff repo is dirty; pass --allow-handoff-dirty to record it")
|
||||
|
||||
qwen_config = _require_config_file(args.qwen_config, "qwen")
|
||||
glm_config = _require_config_file(args.glm_config, "glm")
|
||||
ml_file = _require_file(args.ml_instance_file, "ml instance file")
|
||||
verified_file = _require_file(args.verified_instance_file, "verified instance file")
|
||||
runner = args.handoff_root / RUNNER_RELATIVE_PATH
|
||||
_require_file(runner, "handoff runner")
|
||||
|
||||
config_snapshot_dir = run_dir / "config_snapshot"
|
||||
instance_snapshot_dir = run_dir / "instance_snapshot"
|
||||
if args.dry_run:
|
||||
qwen_snapshot = _snapshot_preview(qwen_config)
|
||||
glm_snapshot = _snapshot_preview(glm_config)
|
||||
ml_snapshot = _snapshot_preview(ml_file)
|
||||
verified_snapshot = _snapshot_preview(verified_file)
|
||||
else:
|
||||
run_dir.mkdir(parents=True, exist_ok=True)
|
||||
qwen_snapshot = copy_snapshot(qwen_config, config_snapshot_dir / "qwen")
|
||||
glm_snapshot = copy_snapshot(glm_config, config_snapshot_dir / "glm")
|
||||
ml_snapshot = copy_snapshot(ml_file, instance_snapshot_dir / "ml")
|
||||
verified_snapshot = copy_snapshot(verified_file, instance_snapshot_dir / "verified")
|
||||
|
||||
env = required_secret_env(
|
||||
args.run_mode, _required_secret_env_overrides(args.required_secret_env)
|
||||
)
|
||||
env.update(parse_env_overrides(args.env))
|
||||
created_at = now_iso()
|
||||
manifest = {
|
||||
"exp_id": exp_id,
|
||||
"status": "planned",
|
||||
"question": args.question,
|
||||
"hypothesis": args.hypothesis,
|
||||
"source": source.__dict__,
|
||||
"handoff": {
|
||||
**handoff.__dict__,
|
||||
"dirty_allowed": bool(args.allow_handoff_dirty),
|
||||
},
|
||||
"model": _model_metadata(args.run_mode, env),
|
||||
"config": {
|
||||
"condition_label": args.condition_label,
|
||||
"qwen_config": qwen_snapshot,
|
||||
"glm_config": glm_snapshot,
|
||||
"runner": {"path": str(runner), "sha256": sha256_file(runner)},
|
||||
"env": env,
|
||||
},
|
||||
"slice": {
|
||||
"ml": {**ml_snapshot, "count": args.ml_count},
|
||||
"verified": {**verified_snapshot, "count": args.verified_count},
|
||||
},
|
||||
"execution": {
|
||||
"run_mode": args.run_mode,
|
||||
"qwen_workers": args.qwen_workers,
|
||||
"glm_workers": args.glm_workers,
|
||||
"eval_workers": args.eval_workers,
|
||||
"command_path": str(run_dir / "command.sh"),
|
||||
},
|
||||
"artifacts": {},
|
||||
"decision_gate": {"items": args.decision_gate},
|
||||
"created_at": created_at,
|
||||
"evidence_level": "manifested",
|
||||
}
|
||||
|
||||
command = render_command(args, manifest)
|
||||
if not args.dry_run:
|
||||
atomic_write_json(run_dir / "manifest.json", manifest)
|
||||
atomic_write_text(run_dir / "command.sh", command)
|
||||
_make_executable(run_dir / "command.sh")
|
||||
atomic_write_json(
|
||||
run_dir / "preflight.json",
|
||||
{
|
||||
"created_at": created_at,
|
||||
"source_dirty_count": source.dirty_count,
|
||||
"handoff_dirty_count": handoff.dirty_count,
|
||||
"config_hashes": {
|
||||
"qwen": qwen_snapshot["sha256"],
|
||||
"glm": glm_snapshot["sha256"],
|
||||
},
|
||||
},
|
||||
)
|
||||
append_jsonl(
|
||||
ledger_root / "experiments.jsonl",
|
||||
{
|
||||
"time": created_at,
|
||||
"exp_id": exp_id,
|
||||
"event": "created",
|
||||
"run_dir": str(run_dir),
|
||||
"condition_label": args.condition_label,
|
||||
},
|
||||
)
|
||||
print(json.dumps({"exp_id": exp_id, "run_dir": str(run_dir)}, indent=2))
|
||||
|
||||
|
||||
def _required_secret_env_overrides(items: list[str]) -> dict[str, str]:
|
||||
mapping = dict(DEFAULT_REQUIRED_SECRET_ENV)
|
||||
for item in items:
|
||||
model, sep, name = item.partition("=")
|
||||
model = model.strip()
|
||||
name = name.strip()
|
||||
if not sep or model not in DEFAULT_REQUIRED_SECRET_ENV or not name:
|
||||
raise LedgerError(
|
||||
"--required-secret-env must use MODEL=ENV_VAR with MODEL in "
|
||||
+ "/".join(sorted(DEFAULT_REQUIRED_SECRET_ENV))
|
||||
)
|
||||
mapping[model] = name
|
||||
return mapping
|
||||
|
||||
|
||||
def _require_config_file(path: Path, label: str) -> Path:
|
||||
if path.is_dir():
|
||||
raise LedgerError(f"{label} config must be a config.toml file, got directory: {path}")
|
||||
return _require_file(path, f"{label} config")
|
||||
|
||||
|
||||
def _require_file(path: Path, label: str) -> Path:
|
||||
if not path.is_file():
|
||||
raise LedgerError(f"missing {label}: {path}")
|
||||
return path
|
||||
|
||||
|
||||
def _snapshot_preview(path: Path) -> dict[str, str]:
|
||||
return {"source": str(path), "snapshot": "", "sha256": sha256_file(path)}
|
||||
|
||||
|
||||
def _model_metadata(run_mode: str, env: dict[str, dict[str, Any]]) -> dict[str, Any]:
|
||||
# Thinking levels reflect the pinned env treatment when present; the
|
||||
# defaults mirror what an unpinned run actually gets (the batch runner's
|
||||
# GLM_THINKING:-xhigh fallback, the qwen config.toml thinking_level).
|
||||
return {
|
||||
"run_mode": run_mode,
|
||||
"qwen": {
|
||||
"enabled": run_mode != "glm_only",
|
||||
"provider": "dashscope",
|
||||
"model": "qwen3.6-flash",
|
||||
"thinking": _pinned_env_value(env, "QWEN_THINKING", "high"),
|
||||
"cache": "on",
|
||||
},
|
||||
"glm": {
|
||||
"enabled": run_mode != "qwen_only",
|
||||
"provider": "openrouter",
|
||||
"model": "z-ai/glm-5.1",
|
||||
"thinking": _pinned_env_value(env, "GLM_THINKING", "xhigh"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _pinned_env_value(env: dict[str, dict[str, Any]], key: str, default: str) -> str:
|
||||
meta = env.get(key)
|
||||
if not isinstance(meta, dict) or meta.get("redacted"):
|
||||
return default
|
||||
value = meta.get("value")
|
||||
return value if value else default
|
||||
|
||||
|
||||
def render_command(args: argparse.Namespace, manifest: dict[str, Any]) -> str:
|
||||
qwen_config_dir = Path(_snapshot_or_source(manifest["config"]["qwen_config"])).parent
|
||||
glm_config_dir = Path(_snapshot_or_source(manifest["config"]["glm_config"])).parent
|
||||
ml_instance_file = _snapshot_or_source(manifest["slice"]["ml"])
|
||||
verified_instance_file = _snapshot_or_source(manifest["slice"]["verified"])
|
||||
env_exports = [
|
||||
f"export OPENSQUILLA_SOURCE_REPO={sh_quote(str(args.source_root))}",
|
||||
f"export RUN_MODE={sh_quote(args.run_mode)}",
|
||||
f"export CONDITION_LABEL={sh_quote(args.condition_label)}",
|
||||
f"export QWEN_CONFIG_DIR={sh_quote(str(qwen_config_dir))}",
|
||||
f"export GLM_CONFIG_DIR={sh_quote(str(glm_config_dir))}",
|
||||
f"export ML_INSTANCE_FILE={sh_quote(ml_instance_file)}",
|
||||
f"export VERIFIED_INSTANCE_FILE={sh_quote(verified_instance_file)}",
|
||||
f"export ML_COUNT={args.ml_count}",
|
||||
f"export VERIFIED_COUNT={args.verified_count}",
|
||||
f"export QWEN_WORKERS={args.qwen_workers}",
|
||||
f"export GLM_WORKERS={args.glm_workers}",
|
||||
f"export EVAL_WORKERS={args.eval_workers}",
|
||||
]
|
||||
env_exports.extend(env_exports_for_command(manifest["config"]["env"]))
|
||||
return "\n".join(
|
||||
[
|
||||
"#!/usr/bin/env bash",
|
||||
"set -euo pipefail",
|
||||
"# Secrets are intentionally not embedded. Provide provider keys via",
|
||||
"# environment variables or stdin as expected by the handoff runner.",
|
||||
f"cd {sh_quote(str(args.handoff_root))}",
|
||||
*env_exports,
|
||||
f"{sh_quote(str(args.handoff_root / RUNNER_RELATIVE_PATH))}",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _snapshot_or_source(payload: dict[str, Any]) -> str:
|
||||
snapshot = str(payload.get("snapshot") or "")
|
||||
return snapshot or str(payload["source"])
|
||||
|
||||
|
||||
def _make_executable(path: Path) -> None:
|
||||
mode = path.stat().st_mode
|
||||
path.chmod(mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,261 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Register and query quarantined (contaminated) runs in the experiment ledger.
|
||||
|
||||
A contamination class tags artifact batches whose results are confounded by an
|
||||
infra defect (e.g. a provider-compaction marker leak). Registration is
|
||||
idempotent: names merge into ``contaminations.json`` at the ledger root,
|
||||
matching ledger run dirs get a ``contamination.json`` stamp, and an event is
|
||||
appended to ``experiments.jsonl``. Baseline tooling and ``exp_status`` read the
|
||||
registry to warn when a quarantined artifact backs a baseline.
|
||||
|
||||
Usage:
|
||||
exp_quarantine.py register --contamination-class NAME --names-file F \
|
||||
--description TEXT [--evidence PATH] [--boundary-commit C ...]
|
||||
exp_quarantine.py check ARTIFACT [ARTIFACT ...]
|
||||
exp_quarantine.py list
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from exp_common import (
|
||||
LedgerError,
|
||||
append_jsonl,
|
||||
artifact_basename,
|
||||
atomic_write_json,
|
||||
contamination_class_for,
|
||||
contaminations_path,
|
||||
ledger_lock,
|
||||
ledger_root_from_env,
|
||||
load_contaminations,
|
||||
now_iso,
|
||||
read_json,
|
||||
)
|
||||
|
||||
CLASS_NAME_RE_HELP = "lowercase snake_case, e.g. tool_result_compaction_defect"
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
register = sub.add_parser("register", help="register/merge a contamination class")
|
||||
register.add_argument(
|
||||
"--contamination-class",
|
||||
required=True,
|
||||
help=f"class name ({CLASS_NAME_RE_HELP})",
|
||||
)
|
||||
register.add_argument(
|
||||
"--names-file",
|
||||
required=True,
|
||||
type=Path,
|
||||
help="JSON file containing a list of artifact batch names",
|
||||
)
|
||||
register.add_argument("--description", required=True)
|
||||
register.add_argument(
|
||||
"--evidence",
|
||||
default="",
|
||||
help="path or URL of the audit/report that established the contamination",
|
||||
)
|
||||
register.add_argument(
|
||||
"--boundary-commit",
|
||||
action="append",
|
||||
default=[],
|
||||
help="fix-boundary commit (repeatable); runs at or before these are affected",
|
||||
)
|
||||
|
||||
check = sub.add_parser("check", help="check artifact names/paths against the registry")
|
||||
check.add_argument("artifacts", nargs="+")
|
||||
|
||||
sub.add_parser("list", help="summarize registered contamination classes")
|
||||
return parser
|
||||
|
||||
|
||||
def _load_names(path: Path) -> list[str]:
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise LedgerError(f"cannot read names file {path}: {exc}") from exc
|
||||
if not isinstance(data, list) or not all(isinstance(item, str) for item in data):
|
||||
raise LedgerError(f"names file must be a JSON list of strings: {path}")
|
||||
# Normalize before filtering: a raw value like "/" or "///" is non-empty
|
||||
# pre-normalization but collapses to "" via artifact_basename, and an
|
||||
# empty name would substring-match (or exact-match) every run.
|
||||
names = sorted(
|
||||
{
|
||||
normalized
|
||||
for item in data
|
||||
if item.strip()
|
||||
for normalized in [artifact_basename(item)]
|
||||
if normalized
|
||||
}
|
||||
)
|
||||
if not names:
|
||||
raise LedgerError(f"names file is empty: {path}")
|
||||
return names
|
||||
|
||||
|
||||
def _candidate_names_in_payload(payload: Any) -> set[str]:
|
||||
"""Recursively collect basename-normalized string values from a JSON payload."""
|
||||
names: set[str] = set()
|
||||
if isinstance(payload, str):
|
||||
normalized = artifact_basename(payload)
|
||||
if normalized:
|
||||
names.add(normalized)
|
||||
elif isinstance(payload, dict):
|
||||
for value in payload.values():
|
||||
names.update(_candidate_names_in_payload(value))
|
||||
elif isinstance(payload, list):
|
||||
for item in payload:
|
||||
names.update(_candidate_names_in_payload(item))
|
||||
return names
|
||||
|
||||
|
||||
def _stamp_run_dirs(root: Path, class_name: str, names: list[str]) -> list[str]:
|
||||
"""Stamp ledger run dirs whose artifacts reference a quarantined batch name.
|
||||
|
||||
Matching is by exact basename, consistent with ``contamination_class_for``.
|
||||
A prior substring-based check over the raw JSON text could both false-
|
||||
positive (a quarantined name that is a substring of an unrelated, longer
|
||||
name) and, worse, treat an empty name as a substring of everything.
|
||||
"""
|
||||
stamped: list[str] = []
|
||||
runs_dir = root / "runs"
|
||||
if not runs_dir.is_dir():
|
||||
return stamped
|
||||
name_set = set(names)
|
||||
for run_dir in sorted(runs_dir.iterdir()):
|
||||
if not run_dir.is_dir():
|
||||
continue
|
||||
matched: set[str] = set()
|
||||
for source in ("artifacts.json", "manifest.json"):
|
||||
payload = read_json(run_dir / source)
|
||||
if not payload:
|
||||
continue
|
||||
matched.update(name_set & _candidate_names_in_payload(payload))
|
||||
if not matched:
|
||||
continue
|
||||
stamp_path = run_dir / "contamination.json"
|
||||
existing = read_json(stamp_path)
|
||||
classes = existing.get("classes") if isinstance(existing.get("classes"), dict) else {}
|
||||
previous = classes.get(class_name, {})
|
||||
previous_names = (
|
||||
set(previous.get("matched_artifact_names", []))
|
||||
if isinstance(previous, dict)
|
||||
else set()
|
||||
)
|
||||
classes[class_name] = {
|
||||
"matched_artifact_names": sorted(previous_names | matched),
|
||||
"stamped_at": (
|
||||
previous.get("stamped_at")
|
||||
if isinstance(previous, dict) and previous.get("stamped_at")
|
||||
else now_iso()
|
||||
),
|
||||
}
|
||||
atomic_write_json(stamp_path, {"classes": classes})
|
||||
stamped.append(run_dir.name)
|
||||
return stamped
|
||||
|
||||
|
||||
def cmd_register(args: argparse.Namespace) -> int:
|
||||
root = ledger_root_from_env()
|
||||
class_name = str(args.contamination_class).strip()
|
||||
if not class_name:
|
||||
raise LedgerError("--contamination-class must be non-empty")
|
||||
names = _load_names(args.names_file)
|
||||
with ledger_lock(root):
|
||||
data = load_contaminations(root)
|
||||
classes = data["classes"]
|
||||
entry = classes.get(class_name)
|
||||
if not isinstance(entry, dict):
|
||||
entry = {"registered_at": now_iso()}
|
||||
existing_names = set(entry.get("artifact_names", []))
|
||||
merged = sorted(existing_names | set(names))
|
||||
new_names = sorted(set(names) - existing_names)
|
||||
entry.update(
|
||||
{
|
||||
"description": str(args.description),
|
||||
"evidence": str(args.evidence),
|
||||
"boundary_commits": sorted({str(c) for c in args.boundary_commit}),
|
||||
"artifact_names": merged,
|
||||
"updated_at": now_iso(),
|
||||
}
|
||||
)
|
||||
classes[class_name] = entry
|
||||
data["updated_at"] = now_iso()
|
||||
atomic_write_json(contaminations_path(root), data)
|
||||
stamped = _stamp_run_dirs(root, class_name, merged)
|
||||
append_jsonl(
|
||||
root / "experiments.jsonl",
|
||||
{
|
||||
"event": "contamination_registered",
|
||||
"contamination_class": class_name,
|
||||
"artifact_names_total": len(merged),
|
||||
"artifact_names_new": len(new_names),
|
||||
"stamped_run_dirs": stamped,
|
||||
"evidence": str(args.evidence),
|
||||
"boundary_commits": sorted({str(c) for c in args.boundary_commit}),
|
||||
"time": now_iso(),
|
||||
},
|
||||
)
|
||||
print(
|
||||
f"registered {class_name}: {len(merged)} artifact names "
|
||||
f"({len(new_names)} new), stamped {len(stamped)} ledger run dirs"
|
||||
)
|
||||
for run_name in stamped:
|
||||
print(f" stamped: runs/{run_name}/contamination.json")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_check(args: argparse.Namespace) -> int:
|
||||
root = ledger_root_from_env()
|
||||
contaminations = load_contaminations(root)
|
||||
dirty = 0
|
||||
for artifact in args.artifacts:
|
||||
contamination_class = contamination_class_for(root, artifact, contaminations)
|
||||
if contamination_class:
|
||||
dirty += 1
|
||||
print(f"QUARANTINED\t{artifact_basename(artifact)}\t{contamination_class}")
|
||||
else:
|
||||
print(f"clean\t{artifact_basename(artifact)}")
|
||||
return 1 if dirty else 0
|
||||
|
||||
|
||||
def cmd_list(_: argparse.Namespace) -> int:
|
||||
root = ledger_root_from_env()
|
||||
classes = load_contaminations(root).get("classes", {})
|
||||
if not classes:
|
||||
print("no contamination classes registered")
|
||||
return 0
|
||||
for name, payload in sorted(classes.items()):
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
count = len(payload.get("artifact_names", []))
|
||||
boundary = ",".join(payload.get("boundary_commits", [])) or "-"
|
||||
print(f"{name}\truns={count}\tboundary={boundary}")
|
||||
description = payload.get("description")
|
||||
if description:
|
||||
print(f" {description}")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
if args.command == "register":
|
||||
return cmd_register(args)
|
||||
if args.command == "check":
|
||||
return cmd_check(args)
|
||||
return cmd_list(args)
|
||||
except LedgerError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run an OpenSquilla experiment from an existing ledger manifest."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from exp_common import (
|
||||
LedgerError,
|
||||
append_jsonl,
|
||||
atomic_write_json,
|
||||
exp_dir,
|
||||
git_info,
|
||||
ledger_lock,
|
||||
ledger_root_from_env,
|
||||
now_iso,
|
||||
read_json,
|
||||
read_json_strict,
|
||||
sha256_file,
|
||||
validate_exp_id,
|
||||
)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--exp-id", required=True)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
return run_experiment(args.exp_id)
|
||||
except LedgerError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
def run_experiment(exp_id: str) -> int:
|
||||
exp_id = validate_exp_id(exp_id)
|
||||
ledger_root = ledger_root_from_env()
|
||||
run_dir = exp_dir(ledger_root, exp_id)
|
||||
manifest_path = run_dir / "manifest.json"
|
||||
manifest = read_json_strict(manifest_path, label="experiment manifest")
|
||||
_verify_manifest_inputs(manifest)
|
||||
command_path = Path(manifest["execution"]["command_path"])
|
||||
if not command_path.is_file():
|
||||
raise LedgerError(f"missing command.sh: {command_path}")
|
||||
|
||||
started_at = now_iso()
|
||||
with ledger_lock(ledger_root):
|
||||
atomic_write_json(
|
||||
run_dir / "live_status.json",
|
||||
{"status": "running", "started_at": started_at, "exp_id": exp_id},
|
||||
)
|
||||
current = read_json(ledger_root / "current.json")
|
||||
current.update(
|
||||
{
|
||||
"updated_at": started_at,
|
||||
"active_experiment": exp_id,
|
||||
"active_run_dir": str(run_dir),
|
||||
}
|
||||
)
|
||||
atomic_write_json(ledger_root / "current.json", current)
|
||||
append_jsonl(
|
||||
ledger_root / "experiments.jsonl",
|
||||
{"time": started_at, "exp_id": exp_id, "event": "started", "run_dir": str(run_dir)},
|
||||
)
|
||||
|
||||
status = "finished"
|
||||
return_code = 0
|
||||
failure = ""
|
||||
try:
|
||||
proc = subprocess.run([str(command_path)], cwd=run_dir, check=False)
|
||||
return_code = proc.returncode
|
||||
except KeyboardInterrupt:
|
||||
status = "interrupted"
|
||||
return_code = 130
|
||||
failure = "keyboard_interrupt"
|
||||
except Exception as exc: # pragma: no cover - defensive ledger cleanup path
|
||||
status = "failed"
|
||||
return_code = 2
|
||||
failure = str(exc)
|
||||
finally:
|
||||
_record_completion(
|
||||
ledger_root=ledger_root,
|
||||
run_dir=run_dir,
|
||||
exp_id=exp_id,
|
||||
started_at=started_at,
|
||||
status=status,
|
||||
return_code=return_code,
|
||||
failure=failure,
|
||||
)
|
||||
return return_code
|
||||
|
||||
|
||||
def _record_completion(
|
||||
*,
|
||||
ledger_root: Path,
|
||||
run_dir: Path,
|
||||
exp_id: str,
|
||||
started_at: str,
|
||||
status: str,
|
||||
return_code: int,
|
||||
failure: str,
|
||||
) -> None:
|
||||
finished_at = now_iso()
|
||||
payload: dict[str, Any] = {
|
||||
"status": status,
|
||||
"started_at": started_at,
|
||||
"finished_at": finished_at,
|
||||
"return_code": return_code,
|
||||
"exp_id": exp_id,
|
||||
}
|
||||
if failure:
|
||||
payload["failure"] = failure
|
||||
with ledger_lock(ledger_root):
|
||||
atomic_write_json(run_dir / "live_status.json", payload)
|
||||
current = read_json(ledger_root / "current.json")
|
||||
if current.get("active_experiment") == exp_id:
|
||||
current["active_experiment"] = None
|
||||
current.update(
|
||||
{
|
||||
"updated_at": finished_at,
|
||||
"last_experiment": exp_id,
|
||||
"last_return_code": return_code,
|
||||
"last_status": status,
|
||||
}
|
||||
)
|
||||
atomic_write_json(ledger_root / "current.json", current)
|
||||
append_jsonl(
|
||||
ledger_root / "experiments.jsonl",
|
||||
{
|
||||
"time": finished_at,
|
||||
"exp_id": exp_id,
|
||||
"event": status,
|
||||
"return_code": return_code,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _verify_manifest_inputs(manifest: dict[str, Any]) -> None:
|
||||
source = manifest.get("source", {})
|
||||
current_source = git_info(Path(source["path"]))
|
||||
if current_source.dirty_count:
|
||||
raise LedgerError("source repo is dirty; refusing to run experiment")
|
||||
if current_source.head != source.get("head"):
|
||||
raise LedgerError("source HEAD changed since manifest creation")
|
||||
for section in ("qwen_config", "glm_config"):
|
||||
payload = manifest.get("config", {}).get(section, {})
|
||||
_verify_payload_hash(section, payload)
|
||||
for section in ("ml", "verified"):
|
||||
payload = manifest.get("slice", {}).get(section, {})
|
||||
_verify_payload_hash(f"{section} instance file", payload)
|
||||
_verify_runner(manifest.get("config", {}).get("runner", {}))
|
||||
|
||||
|
||||
def _verify_runner(runner: dict[str, Any]) -> None:
|
||||
expected = str(runner.get("sha256") or "")
|
||||
path_str = str(runner.get("path") or "")
|
||||
if not expected or not path_str:
|
||||
return
|
||||
path = Path(path_str)
|
||||
if not path.is_file():
|
||||
raise LedgerError(f"handoff runner missing: {path}")
|
||||
if sha256_file(path) != expected:
|
||||
raise LedgerError("handoff runner changed since manifest creation")
|
||||
|
||||
|
||||
def _verify_payload_hash(label: str, payload: dict[str, Any]) -> None:
|
||||
expected = payload.get("sha256")
|
||||
if not expected:
|
||||
return
|
||||
checked = False
|
||||
for key in ("snapshot", "source"):
|
||||
path_str = str(payload.get(key) or "")
|
||||
if not path_str:
|
||||
continue
|
||||
path = Path(path_str)
|
||||
if path.is_file():
|
||||
checked = True
|
||||
if sha256_file(path) != expected:
|
||||
raise LedgerError(f"{label} {key} hash changed since manifest creation")
|
||||
if not checked:
|
||||
raise LedgerError(f"{label} source and snapshot are both missing")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Print the current OpenSquilla experiment ledger status."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from exp_common import (
|
||||
LedgerError,
|
||||
active_processes,
|
||||
active_swe_containers,
|
||||
contamination_class_for,
|
||||
exp_dir,
|
||||
git_info,
|
||||
ledger_root_from_env,
|
||||
load_contaminations,
|
||||
read_json,
|
||||
run_dir_contamination_classes,
|
||||
)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--source-root", type=Path, required=True)
|
||||
parser.add_argument("--handoff-root", type=Path, required=True)
|
||||
parser.add_argument("--json", action="store_true")
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
status = collect_status(args.source_root, args.handoff_root)
|
||||
except LedgerError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
if args.json:
|
||||
print(json.dumps(status, indent=2, sort_keys=True))
|
||||
else:
|
||||
print(render_text(status))
|
||||
return 0
|
||||
|
||||
|
||||
def collect_status(source_root: Path, handoff_root: Path) -> dict[str, Any]:
|
||||
ledger_root = ledger_root_from_env()
|
||||
current = read_json(ledger_root / "current.json")
|
||||
baselines = read_json(ledger_root / "baselines.json")
|
||||
mechanisms = read_json(ledger_root / "mechanisms.json")
|
||||
source = git_info(source_root)
|
||||
handoff = git_info(handoff_root)
|
||||
processes = active_processes()
|
||||
warnings = []
|
||||
active_exp_id = current.get("active_experiment")
|
||||
active_manifest = None
|
||||
active_manifest_payload: dict[str, Any] = {}
|
||||
active_live_status: dict[str, Any] | None = None
|
||||
if active_exp_id:
|
||||
run_dir = exp_dir(ledger_root, str(active_exp_id))
|
||||
manifest_path = run_dir / "manifest.json"
|
||||
if manifest_path.exists():
|
||||
active_manifest = str(manifest_path)
|
||||
active_manifest_payload = read_json(manifest_path)
|
||||
else:
|
||||
warnings.append(f"active experiment manifest missing: {active_exp_id}")
|
||||
live_status_path = run_dir / "live_status.json"
|
||||
if live_status_path.exists():
|
||||
active_live_status = read_json(live_status_path)
|
||||
containers = active_swe_containers(active_manifest_payload)
|
||||
if active_exp_id and not processes and not containers:
|
||||
warnings.append(
|
||||
f"active experiment {active_exp_id} but no live runner processes or SWE "
|
||||
"containers; verify live_status.json (stale active state?)"
|
||||
)
|
||||
qwen_baseline = _extract_baseline(baselines, "qwen")
|
||||
glm_baseline = _extract_baseline(baselines, "glm")
|
||||
for label, baseline in (("qwen", qwen_baseline), ("glm", glm_baseline)):
|
||||
head = baseline.get("source_head")
|
||||
if (
|
||||
head
|
||||
and not str(source.head).startswith(str(head))
|
||||
and not str(head).startswith(source.short_head)
|
||||
):
|
||||
warnings.append(f"{label} baseline source_head differs from current source HEAD")
|
||||
contaminations = load_contaminations(ledger_root)
|
||||
for label, baseline in (("qwen", qwen_baseline), ("glm", glm_baseline)):
|
||||
contamination_classes: list[str] = []
|
||||
artifact = baseline.get("artifact")
|
||||
if artifact:
|
||||
artifact_class = contamination_class_for(
|
||||
ledger_root, str(artifact), contaminations
|
||||
)
|
||||
if artifact_class:
|
||||
contamination_classes.append(artifact_class)
|
||||
baseline_exp_id = baseline.get("exp_id")
|
||||
if baseline_exp_id:
|
||||
contamination_classes.extend(
|
||||
run_dir_contamination_classes(ledger_root, str(baseline_exp_id))
|
||||
)
|
||||
if contamination_classes:
|
||||
joined = ", ".join(sorted(set(contamination_classes)))
|
||||
warnings.append(
|
||||
f"{label} baseline is quarantined ({joined}); "
|
||||
"re-baseline on clean runs"
|
||||
)
|
||||
return {
|
||||
"ledger_root": str(ledger_root),
|
||||
"source": source.__dict__,
|
||||
"handoff": handoff.__dict__,
|
||||
"active_experiment": active_exp_id,
|
||||
"active_manifest": active_manifest,
|
||||
"active_live_status": active_live_status,
|
||||
"processes": processes,
|
||||
"containers": containers,
|
||||
"qwen_baseline": qwen_baseline,
|
||||
"glm_baseline": glm_baseline,
|
||||
"mechanisms": mechanisms,
|
||||
"warnings": [*current.get("warnings", []), *warnings]
|
||||
if isinstance(current.get("warnings", []), list)
|
||||
else warnings,
|
||||
"current": current,
|
||||
}
|
||||
|
||||
|
||||
def _extract_baseline(baselines: dict[str, Any], model: str) -> dict[str, Any]:
|
||||
value = baselines.get(model, {})
|
||||
if isinstance(value, dict) and isinstance(value.get("current_best"), dict):
|
||||
return dict(value["current_best"])
|
||||
if isinstance(value, dict) and isinstance(value.get("latest_guard"), dict):
|
||||
return dict(value["latest_guard"])
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
return {}
|
||||
|
||||
|
||||
def render_text(status: dict[str, Any]) -> str:
|
||||
source = status["source"]
|
||||
handoff = status["handoff"]
|
||||
lines = [
|
||||
f"Ledger: {status['ledger_root']}",
|
||||
f"Source: {_dirty_label(source)} {source['short_head']} {source['branch']}",
|
||||
f"Handoff: {_dirty_label(handoff)} {handoff['short_head']} {handoff['branch']}",
|
||||
]
|
||||
active = status.get("active_experiment")
|
||||
if active:
|
||||
lines.append(f"Active experiment: {active}")
|
||||
if status.get("active_manifest"):
|
||||
lines.append(f"Manifest: {status['active_manifest']}")
|
||||
live = status.get("active_live_status") or {}
|
||||
if live:
|
||||
lines.append(f"Live status: {live.get('status', 'unknown')}")
|
||||
else:
|
||||
lines.append("Active experiment: none")
|
||||
lines.append(f"Active runner processes: {len(status['processes'])}")
|
||||
lines.append(f"Active SWE containers: {len(status['containers'])}")
|
||||
qwen = status.get("qwen_baseline") or {}
|
||||
glm = status.get("glm_baseline") or {}
|
||||
lines.append(f"Qwen baseline: {format_baseline(qwen)}")
|
||||
lines.append(f"GLM baseline: {format_baseline(glm)}")
|
||||
rejected = _mechanisms_by_status(
|
||||
status.get("mechanisms", {}),
|
||||
{"rejected", "rejected_for_qwen"},
|
||||
)
|
||||
observe = _mechanisms_by_status(status.get("mechanisms", {}), {"observe"})
|
||||
if rejected:
|
||||
lines.append("Rejected: " + ", ".join(rejected[:8]))
|
||||
if observe:
|
||||
lines.append("Observe-only: " + ", ".join(observe[:8]))
|
||||
warnings = status.get("warnings") or []
|
||||
if warnings:
|
||||
lines.append("Warnings:")
|
||||
lines.extend(f" - {item}" for item in warnings)
|
||||
lines.append("Next: create or run from manifest; do not infer config from chat.")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _dirty_label(info: dict[str, Any]) -> str:
|
||||
if int(info.get("dirty_count") or 0) == 0:
|
||||
return "clean"
|
||||
return f"dirty {info['dirty_count']}"
|
||||
|
||||
|
||||
def format_baseline(baseline: dict[str, Any]) -> str:
|
||||
if not baseline:
|
||||
return "not recorded"
|
||||
result = baseline.get("result")
|
||||
if result:
|
||||
return str(result)
|
||||
resolved = baseline.get("resolved")
|
||||
total = baseline.get("total")
|
||||
empty = baseline.get("empty")
|
||||
label = baseline.get("label") or baseline.get("exp_id") or "baseline"
|
||||
if resolved is not None and total is not None:
|
||||
suffix = f" empty={empty}" if empty is not None else ""
|
||||
return f"{label} = {resolved}/{total}{suffix}"
|
||||
return label
|
||||
|
||||
|
||||
def _mechanisms_by_status(mechanisms: dict[str, Any], statuses: set[str]) -> list[str]:
|
||||
result = []
|
||||
for name, payload in mechanisms.items():
|
||||
if isinstance(payload, dict) and payload.get("status") in statuses:
|
||||
result.append(name)
|
||||
return sorted(result)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,295 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline transcript replay for the finalize-time red-evidence gate.
|
||||
|
||||
Feeds recorded run transcripts through the exact same pure tracker the
|
||||
live agent loop uses (``opensquilla.engine.finalize_evidence_gate``) and
|
||||
reports whether the gate would have challenged the run's final state. This
|
||||
lets the gate be validated offline: it should fire on runs whose transcripts
|
||||
show red self-evidence at finalization while staying quiet on runs that
|
||||
finished green.
|
||||
|
||||
Usage:
|
||||
# Single runs, one JSON report line per run dir
|
||||
python scripts/experiments/replay_finalize_gate.py RUN_DIR [RUN_DIR ...]
|
||||
|
||||
# Driver mode over a replay-set manifest
|
||||
python scripts/experiments/replay_finalize_gate.py --sets replay_sets.json
|
||||
|
||||
The replay-set manifest maps set names to ``[label, run_dir]`` pairs; the set
|
||||
named ``positives`` is expected to fire, all other sets are controls.
|
||||
|
||||
Each RUN_DIR must contain ``transcript.jsonl`` (persisted session messages)
|
||||
and is expected to contain ``git.patch`` (final workspace diff; missing or
|
||||
blank means no diff, which suppresses the gate exactly as in the live loop).
|
||||
|
||||
Known live-vs-replay divergences:
|
||||
|
||||
- ``has_workspace_diff`` comes from the harness-cleaned ``git.patch``; the
|
||||
live gate uses ``git status --porcelain --untracked-files=all``, which also
|
||||
sees untracked scratch files. Replay therefore under-fires on runs whose
|
||||
only diff was untracked files.
|
||||
- The live gate evaluates only when the model emits a zero-tool-call
|
||||
assistant message (a finalize attempt). Transcripts that end mid-loop
|
||||
(iteration cap, abort) never reach the gate live, so replay reports them
|
||||
as ``should_challenge: false`` with ``suppressed_reason:
|
||||
no_finalize_attempt_at_transcript_end`` (raw triggers stay in the report
|
||||
for diagnostics).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from opensquilla.engine.finalize_evidence_gate import (
|
||||
EXECUTION_TOOL_NAMES,
|
||||
WRITE_TOOL_NAMES,
|
||||
FinalizeEvidenceTracker,
|
||||
execution_signals_from_result,
|
||||
)
|
||||
|
||||
_ANCHOR_MARKERS = (
|
||||
"failed",
|
||||
"failure",
|
||||
"error",
|
||||
"exception",
|
||||
"traceback",
|
||||
"assert",
|
||||
"expected",
|
||||
"actual",
|
||||
)
|
||||
|
||||
|
||||
def _failure_anchor_lines(text: str, limit: int = 3) -> list[str]:
|
||||
"""Lightweight stand-in for the live loop's anchor extraction.
|
||||
|
||||
Anchors only decorate challenge messages and dedup keys; they do not
|
||||
affect whether a trigger fires, so replay uses a simplified extraction.
|
||||
"""
|
||||
|
||||
anchors: list[str] = []
|
||||
for raw_line in text.splitlines():
|
||||
line = " ".join(raw_line.strip().split())
|
||||
if not line:
|
||||
continue
|
||||
lowered = line.lower()
|
||||
if "no failures" in lowered or "no errors" in lowered:
|
||||
continue
|
||||
if not any(marker in lowered for marker in _ANCHOR_MARKERS):
|
||||
continue
|
||||
anchors.append(line[:220])
|
||||
if len(anchors) >= limit:
|
||||
break
|
||||
return anchors
|
||||
|
||||
|
||||
def _string_arg(arguments: dict[str, Any] | None, *names: str) -> str | None:
|
||||
if not isinstance(arguments, dict):
|
||||
return None
|
||||
for name in names:
|
||||
value = arguments.get(name)
|
||||
if isinstance(value, str) and value.strip():
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _command_for_tool_call(tool_name: str, arguments: dict[str, Any] | None) -> str | None:
|
||||
# Mirrors Agent._execution_command_for_progress.
|
||||
if tool_name == "execute_code":
|
||||
return _string_arg(arguments, "code")
|
||||
return _string_arg(arguments, "command", "cmd")
|
||||
|
||||
|
||||
def _content_text(content: Any) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for block in content:
|
||||
if isinstance(block, dict) and block.get("type") == "text":
|
||||
parts.append(str(block.get("text") or ""))
|
||||
return "\n".join(parts)
|
||||
return ""
|
||||
|
||||
|
||||
def replay_run(run_dir: Path) -> dict[str, Any]:
|
||||
"""Replay one run directory; returns a JSON-safe report."""
|
||||
|
||||
transcript_path = run_dir / "transcript.jsonl"
|
||||
if not transcript_path.is_file():
|
||||
return {"run_dir": str(run_dir), "error": "missing transcript.jsonl"}
|
||||
|
||||
git_patch = run_dir / "git.patch"
|
||||
has_workspace_diff = git_patch.is_file() and bool(
|
||||
git_patch.read_text(errors="replace").strip()
|
||||
)
|
||||
|
||||
tracker = FinalizeEvidenceTracker()
|
||||
pending_calls: dict[str, tuple[str, dict[str, Any] | None]] = {}
|
||||
iteration = 0
|
||||
parse_errors = 0
|
||||
last_assistant_had_tool_calls: bool | None = None
|
||||
|
||||
with transcript_path.open(encoding="utf-8", errors="replace") as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
parse_errors += 1
|
||||
continue
|
||||
message = entry.get("message")
|
||||
if not isinstance(message, dict):
|
||||
continue
|
||||
role = message.get("role")
|
||||
if role == "assistant":
|
||||
content = message.get("content")
|
||||
saw_tool_call = False
|
||||
if isinstance(content, list):
|
||||
for block in content:
|
||||
if isinstance(block, dict) and block.get("type") == "toolCall":
|
||||
saw_tool_call = True
|
||||
call_id = str(block.get("id") or "")
|
||||
pending_calls[call_id] = (
|
||||
str(block.get("name") or ""),
|
||||
block.get("arguments")
|
||||
if isinstance(block.get("arguments"), dict)
|
||||
else None,
|
||||
)
|
||||
if saw_tool_call:
|
||||
iteration += 1
|
||||
last_assistant_had_tool_calls = saw_tool_call
|
||||
continue
|
||||
if role != "toolResult":
|
||||
continue
|
||||
tool_name = str(message.get("toolName") or "")
|
||||
call_id = str(message.get("toolCallId") or "")
|
||||
# Look up without popping: an approval retry emits TWO toolResult
|
||||
# messages for the same call id (approval_pending, then the real
|
||||
# retried outcome); the second must still see the arguments.
|
||||
_, arguments = pending_calls.get(call_id, (tool_name, None))
|
||||
is_error = bool(message.get("isError"))
|
||||
if tool_name in WRITE_TOOL_NAMES:
|
||||
tracker.observe_write(
|
||||
_string_arg(arguments, "path", "file_path"),
|
||||
is_error=is_error,
|
||||
iteration=iteration,
|
||||
scratch=(tool_name == "write_scratch"),
|
||||
)
|
||||
elif tool_name in EXECUTION_TOOL_NAMES:
|
||||
command = _command_for_tool_call(tool_name, arguments)
|
||||
if command is None:
|
||||
continue
|
||||
content_text = _content_text(message.get("content"))
|
||||
execution_status = message.get("executionStatus")
|
||||
red, exit_code, timed_out, status_reason = execution_signals_from_result(
|
||||
tool_name=tool_name,
|
||||
content_text=content_text,
|
||||
execution_status=(
|
||||
execution_status if isinstance(execution_status, dict) else None
|
||||
),
|
||||
is_error=is_error,
|
||||
)
|
||||
tracker.observe_execution(
|
||||
command,
|
||||
red=red,
|
||||
exit_code=exit_code,
|
||||
timed_out=timed_out,
|
||||
status_reason=status_reason,
|
||||
failure_anchors=_failure_anchor_lines(content_text) if red else (),
|
||||
iteration=iteration,
|
||||
)
|
||||
|
||||
observation = tracker.build_observation(has_workspace_diff=has_workspace_diff)
|
||||
# Live parity: the gate only runs on a zero-tool-call assistant message.
|
||||
# A transcript that ends mid-loop (iteration cap, abort) never reached it.
|
||||
finalize_attempt_at_end = last_assistant_had_tool_calls is False
|
||||
report = {
|
||||
"run_dir": str(run_dir),
|
||||
"instance": run_dir.name,
|
||||
"iterations": iteration,
|
||||
"parse_errors": parse_errors,
|
||||
"has_workspace_diff": has_workspace_diff,
|
||||
"has_workspace_diff_source": "git.patch",
|
||||
"finalize_attempt_at_end": finalize_attempt_at_end,
|
||||
**observation.to_event_details(),
|
||||
}
|
||||
if not finalize_attempt_at_end:
|
||||
report["should_challenge"] = False
|
||||
report["suppressed_reason"] = "no_finalize_attempt_at_transcript_end"
|
||||
return report
|
||||
|
||||
|
||||
def _print_report(report: dict[str, Any]) -> None:
|
||||
print(json.dumps(report, ensure_ascii=False))
|
||||
|
||||
|
||||
def _run_sets(sets_path: Path) -> int:
|
||||
manifest = json.loads(sets_path.read_text())
|
||||
summary: dict[str, dict[str, Any]] = {}
|
||||
exit_code = 0
|
||||
for set_name, entries in manifest.items():
|
||||
expected_fire = set_name == "positives"
|
||||
fired: list[str] = []
|
||||
quiet: list[str] = []
|
||||
errored: list[str] = []
|
||||
trigger_counts: dict[str, int] = {}
|
||||
for entry in entries:
|
||||
label, run_dir = entry[0], Path(entry[1])
|
||||
report = replay_run(run_dir)
|
||||
report["set"] = set_name
|
||||
report["label"] = label
|
||||
_print_report(report)
|
||||
if report.get("error"):
|
||||
errored.append(label)
|
||||
continue
|
||||
if report.get("should_challenge"):
|
||||
fired.append(label)
|
||||
for trigger in report.get("triggers") or []:
|
||||
trigger_counts[trigger] = trigger_counts.get(trigger, 0) + 1
|
||||
else:
|
||||
quiet.append(label)
|
||||
summary[set_name] = {
|
||||
"expected_fire": expected_fire,
|
||||
"total": len(entries),
|
||||
"fired": len(fired),
|
||||
"quiet": len(quiet),
|
||||
"errored": len(errored),
|
||||
"fired_labels": fired if expected_fire else fired,
|
||||
"trigger_counts": trigger_counts,
|
||||
}
|
||||
print("\n=== finalize-evidence-gate replay summary ===", file=sys.stderr)
|
||||
for set_name, stats in summary.items():
|
||||
rate = stats["fired"] / stats["total"] if stats["total"] else 0.0
|
||||
print(
|
||||
f"{set_name}: fired {stats['fired']}/{stats['total']} ({rate:.0%})"
|
||||
f" errored={stats['errored']} triggers={stats['trigger_counts']}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
if stats["expected_fire"] and stats["fired"] != stats["total"]:
|
||||
missing = stats["total"] - stats["fired"] - stats["errored"]
|
||||
print(f" positives not fired: {missing}", file=sys.stderr)
|
||||
return exit_code
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("run_dirs", nargs="*", type=Path, help="Run directories to replay")
|
||||
parser.add_argument("--sets", type=Path, help="Replay-set manifest (replay_sets.json)")
|
||||
args = parser.parse_args(argv)
|
||||
if args.sets:
|
||||
return _run_sets(args.sets)
|
||||
if not args.run_dirs:
|
||||
parser.error("provide RUN_DIR arguments or --sets")
|
||||
for run_dir in args.run_dirs:
|
||||
_print_report(replay_run(run_dir))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,418 @@
|
||||
# install_source.ps1 - user-local OpenSquilla installer (no admin).
|
||||
#
|
||||
# Installer contract:
|
||||
# - installs into a user-owned prefix (never Program Files or system32)
|
||||
# - prefers uv tool install; falls back to pip --user; errors clearly if neither exists
|
||||
# - defaults to the "recommended" runtime profile (memory + bundled v4 router)
|
||||
# and allows `$env:OPENSQUILLA_INSTALL_PROFILE="core"` to opt back down
|
||||
# - on Windows, best-effort installs Microsoft Visual C++ Redistributable
|
||||
# before the recommended router profile because onnxruntime requires it
|
||||
# - prints a post-install banner documenting the default bind
|
||||
# (127.0.0.1:18791) and the explicit opt-in required to expose the gateway
|
||||
# on the network (-Listen 0.0.0.0 or $env:OPENSQUILLA_LISTEN="0.0.0.0")
|
||||
# - adds an extra WARNING when the operator requested network exposure at
|
||||
# install time via $env:OPENSQUILLA_LISTEN="0.0.0.0"
|
||||
#
|
||||
# Dry-run: set $env:OPENSQUILLA_INSTALL_DRY_RUN="1" to print the install plan +
|
||||
# banner without touching the system.
|
||||
|
||||
param(
|
||||
[string]$Profile = "",
|
||||
[string[]]$Extras = @()
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
# --- prefix resolution ------------------------------------------------------
|
||||
|
||||
if ($env:OPENSQUILLA_PREFIX) {
|
||||
$prefix = $env:OPENSQUILLA_PREFIX
|
||||
} elseif ($env:LOCALAPPDATA) {
|
||||
$prefix = Join-Path $env:LOCALAPPDATA 'opensquilla'
|
||||
} else {
|
||||
$prefix = Join-Path $HOME '.local'
|
||||
}
|
||||
|
||||
$dryRun = $env:OPENSQUILLA_INSTALL_DRY_RUN -eq '1'
|
||||
$script:isWindowsHost = if (Get-Variable IsWindows -ErrorAction SilentlyContinue) {
|
||||
$IsWindows
|
||||
} else {
|
||||
$env:OS -eq 'Windows_NT'
|
||||
}
|
||||
$profile = if ($Profile) {
|
||||
$Profile
|
||||
} elseif ($env:OPENSQUILLA_INSTALL_PROFILE) {
|
||||
$env:OPENSQUILLA_INSTALL_PROFILE
|
||||
} else {
|
||||
'recommended'
|
||||
}
|
||||
|
||||
$validExtras = @(
|
||||
'matrix',
|
||||
'matrix-e2e',
|
||||
'document-extras'
|
||||
)
|
||||
|
||||
function Split-InstallExtras {
|
||||
param([string[]]$Values)
|
||||
|
||||
$items = New-Object System.Collections.Generic.List[string]
|
||||
foreach ($value in $Values) {
|
||||
if (-not $value) {
|
||||
continue
|
||||
}
|
||||
foreach ($part in ($value -split '[,\s]+')) {
|
||||
$item = $part.Trim()
|
||||
if ($item -and -not $items.Contains($item)) {
|
||||
$items.Add($item)
|
||||
}
|
||||
}
|
||||
}
|
||||
return $items.ToArray()
|
||||
}
|
||||
|
||||
$extraInputs = @()
|
||||
if ($env:OPENSQUILLA_INSTALL_EXTRAS) {
|
||||
$extraInputs += $env:OPENSQUILLA_INSTALL_EXTRAS
|
||||
}
|
||||
$extraInputs += $Extras
|
||||
$installExtras = @(Split-InstallExtras $extraInputs)
|
||||
|
||||
$unknownExtras = @($installExtras | Where-Object { $_ -notin $validExtras })
|
||||
if ($unknownExtras.Count -gt 0) {
|
||||
Write-Error "install_source.ps1: unsupported extras: $($unknownExtras -join ', '). Supported extras: $($validExtras -join ', ')."
|
||||
exit 1
|
||||
}
|
||||
|
||||
switch ($profile) {
|
||||
'core' { $targetExtras = @() }
|
||||
'minimal' { $profile = 'core'; $targetExtras = @() }
|
||||
'recommended' { $targetExtras = @('recommended') }
|
||||
default {
|
||||
Write-Error "install_source.ps1: unsupported OPENSQUILLA_INSTALL_PROFILE='$profile'. Supported profiles: core, recommended."
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
$targetExtras += $installExtras
|
||||
$installTarget = if ($targetExtras.Count -gt 0) {
|
||||
".[$($targetExtras -join ',')]"
|
||||
} else {
|
||||
'.'
|
||||
}
|
||||
|
||||
function Test-SquillaRouterAssets {
|
||||
param(
|
||||
[switch]$WarnOnly
|
||||
)
|
||||
|
||||
if ($profile -ne 'recommended') {
|
||||
return
|
||||
}
|
||||
|
||||
$modelRoot = 'src/opensquilla/squilla_router/models'
|
||||
$required = @(
|
||||
"$modelRoot/v4.2_phase3_inference/lgbm_main.bin",
|
||||
"$modelRoot/v4.2_phase3_inference/router.runtime.yaml",
|
||||
"$modelRoot/v4.2_phase3_inference/mlp/model.onnx",
|
||||
"$modelRoot/v4.2_phase3_inference/features/tfidf.pkl",
|
||||
"$modelRoot/v4.2_phase3_inference/bge_onnx/model.onnx"
|
||||
)
|
||||
$pointerLine = 'version https://git-lfs.github.com/spec/v1'
|
||||
$missing = New-Object System.Collections.Generic.List[string]
|
||||
$pointers = New-Object System.Collections.Generic.List[string]
|
||||
|
||||
foreach ($path in $required) {
|
||||
if (-not (Test-Path $path -PathType Leaf)) {
|
||||
$missing.Add($path)
|
||||
continue
|
||||
}
|
||||
$firstLine = Get-Content -Path $path -TotalCount 1 -ErrorAction SilentlyContinue
|
||||
if ($firstLine -eq $pointerLine) {
|
||||
$pointers.Add($path)
|
||||
}
|
||||
}
|
||||
|
||||
if ($missing.Count -gt 0 -or $pointers.Count -gt 0) {
|
||||
if ($WarnOnly) {
|
||||
Write-Host 'install_source.ps1: dry-run note — real recommended install would fail until bundled squilla-router v4 assets are available in this checkout.'
|
||||
}
|
||||
else {
|
||||
Write-Error 'install_source.ps1: bundled squilla-router v4 assets are unavailable in this checkout.'
|
||||
}
|
||||
if ($missing.Count -gt 0) {
|
||||
$message = "install_source.ps1: missing squilla-router assets: $($missing -join ', ')"
|
||||
if ($WarnOnly) { Write-Host $message } else { Write-Error $message }
|
||||
}
|
||||
if ($pointers.Count -gt 0) {
|
||||
$message = "install_source.ps1: Git LFS pointer files detected: $($pointers -join ', ')"
|
||||
if ($WarnOnly) { Write-Host $message } else { Write-Error $message }
|
||||
}
|
||||
$lfsMessage = 'install_source.ps1: run `git lfs install` once, then `git lfs pull --include="src/opensquilla/squilla_router/models/**"`.'
|
||||
$coreMessage = 'install_source.ps1: or retry with `$env:OPENSQUILLA_INSTALL_PROFILE="core"` for the minimal runtime.'
|
||||
if ($WarnOnly) {
|
||||
Write-Host $lfsMessage
|
||||
Write-Host $coreMessage
|
||||
return
|
||||
}
|
||||
Write-Error $lfsMessage
|
||||
Write-Error $coreMessage
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
function Test-WindowsVCRedistInstalled {
|
||||
if (-not $script:isWindowsHost) {
|
||||
return $true
|
||||
}
|
||||
|
||||
$runtimeKeys = @(
|
||||
'HKLM:\SOFTWARE\Microsoft\VisualStudio\14.0\VC\Runtimes\x64',
|
||||
'HKLM:\SOFTWARE\WOW6432Node\Microsoft\VisualStudio\14.0\VC\Runtimes\x64'
|
||||
)
|
||||
foreach ($key in $runtimeKeys) {
|
||||
if (-not (Test-Path $key)) {
|
||||
continue
|
||||
}
|
||||
$runtime = Get-ItemProperty -Path $key -ErrorAction SilentlyContinue
|
||||
if ($runtime -and $runtime.Installed -eq 1 -and $runtime.Major -ge 14) {
|
||||
return $true
|
||||
}
|
||||
}
|
||||
return $false
|
||||
}
|
||||
|
||||
function Install-WindowsVCRedistIfNeeded {
|
||||
if (-not $script:isWindowsHost -or $profile -ne 'recommended') {
|
||||
return
|
||||
}
|
||||
if ($env:OPENSQUILLA_SKIP_VC_REDIST -eq '1') {
|
||||
Write-Host 'install_source.ps1: skipping Microsoft Visual C++ Redistributable check because OPENSQUILLA_SKIP_VC_REDIST=1.'
|
||||
return
|
||||
}
|
||||
if (Test-WindowsVCRedistInstalled) {
|
||||
Write-Host 'install_source.ps1: Microsoft Visual C++ Redistributable is already installed.'
|
||||
return
|
||||
}
|
||||
|
||||
$redistUrl = 'https://aka.ms/vs/17/release/vc_redist.x64.exe'
|
||||
$winget = Get-Command winget -ErrorAction SilentlyContinue
|
||||
if ($winget) {
|
||||
Write-Host 'install_source.ps1: Microsoft Visual C++ Redistributable not detected; installing with winget.'
|
||||
$wingetArgs = @(
|
||||
'install',
|
||||
'--id',
|
||||
'Microsoft.VCRedist.2015+.x64',
|
||||
'--exact',
|
||||
'--silent',
|
||||
'--accept-package-agreements',
|
||||
'--accept-source-agreements'
|
||||
)
|
||||
& winget @wingetArgs
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
Write-Host 'install_source.ps1: Microsoft Visual C++ Redistributable installation completed.'
|
||||
return
|
||||
}
|
||||
Write-Warning "install_source.ps1: winget could not install Microsoft Visual C++ Redistributable (exit $LASTEXITCODE)."
|
||||
}
|
||||
|
||||
Write-Warning 'OpenSquilla: Microsoft Visual C++ Redistributable 2015-2022 x64 is required for the bundled ONNX router.'
|
||||
Write-Warning 'OpenSquilla can still start with safe router fallback, but bundled ONNX model routing is disabled until this runtime is installed.'
|
||||
Write-Warning "If automatic installation fails, install it manually: $redistUrl"
|
||||
Write-Warning 'After installing, reopen PowerShell and restart OpenSquilla.'
|
||||
}
|
||||
|
||||
# --- installer selection ----------------------------------------------------
|
||||
|
||||
$installer = $null
|
||||
$installArgs = @()
|
||||
|
||||
# Probe the ambient python version once (used only for the pip fallback gate).
|
||||
$pythonCmd = Get-Command python -ErrorAction SilentlyContinue
|
||||
$pythonOk = $false
|
||||
if ($pythonCmd) {
|
||||
& python -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 12) else 1)' 2>$null
|
||||
$pythonOk = ($LASTEXITCODE -eq 0)
|
||||
}
|
||||
|
||||
if (Get-Command uv -ErrorAction SilentlyContinue) {
|
||||
$installer = 'uv'
|
||||
$installArgs = @('tool', 'install', '--python', '3.12', '--force', '--reinstall-package', 'opensquilla', $installTarget)
|
||||
} elseif ($pythonOk) {
|
||||
$installer = 'pip'
|
||||
$installArgs = @('-m', 'pip', 'install', '--user', $installTarget)
|
||||
} else {
|
||||
# No uv, and the ambient python is missing or older than 3.12. Do NOT
|
||||
# silently pip-install onto an unsupported interpreter: a broken
|
||||
# opensquilla makes coding mode fall back to manual edits. Fail loud.
|
||||
$pyver = if ($pythonCmd) { (& python -V 2>&1) } else { 'none' }
|
||||
Write-Error "install_source.ps1: cannot install - uv not found and python ($pyver) is older than 3.12. OpenSquilla requires Python >= 3.12. Install uv (it brings its own 3.12): 'irm https://astral.sh/uv/install.ps1 | iex', then re-run scripts/install_source.ps1."
|
||||
exit 1
|
||||
}
|
||||
|
||||
$installCmd = if ($installer -eq 'uv') {
|
||||
"uv $($installArgs -join ' ')"
|
||||
} else {
|
||||
"python $($installArgs -join ' ')"
|
||||
}
|
||||
|
||||
# --- banner -----------------------------------------------------------------
|
||||
|
||||
function Write-Banner {
|
||||
@"
|
||||
----------------------------------------------------------------------------
|
||||
OpenSquilla installed via $installer -> $prefix (profile: $profile)
|
||||
Extras: $(if ($installExtras.Count -gt 0) { $installExtras -join ', ' } else { 'none' })
|
||||
|
||||
Default gateway bind: 127.0.0.1:18791 (loopback only)
|
||||
Network exposure is opt-in only. To expose the gateway on the network you
|
||||
must use one of:
|
||||
- CLI flag: opensquilla gateway run --listen 0.0.0.0
|
||||
- Env var: `$env:OPENSQUILLA_LISTEN="0.0.0.0"; opensquilla gateway run
|
||||
|
||||
Reminder: only expose 0.0.0.0 behind a trusted reverse proxy or VPN. The
|
||||
gateway's first-class auth assumes loopback-scope by default.
|
||||
----------------------------------------------------------------------------
|
||||
"@ | Write-Host
|
||||
}
|
||||
|
||||
function Write-ListenWarning {
|
||||
@"
|
||||
WARNING: you have selected network-exposed default - ensure you
|
||||
understand the blast radius. The gateway will bind to 0.0.0.0 and be
|
||||
reachable from every interface on this host.
|
||||
"@ | Write-Host
|
||||
}
|
||||
|
||||
# --- post-install PATH sanity (parity with install_source.sh) --------------
|
||||
|
||||
function Resolve-EntrypointDir {
|
||||
# Determine where the just-installed `opensquilla`/`gateway` entry points
|
||||
# landed, so we can warn when that directory is not on PATH. uv tool
|
||||
# install drops entry points in `uv tool dir --bin`; pip --user puts them
|
||||
# in the interpreter's Scripts dir. Both live outside the default PATH on
|
||||
# a clean Windows host - the exact failure mode `opensquilla onboard`
|
||||
# hits right after a "successful" install. Parity with install_source.sh,
|
||||
# which does the same absolute-path lookup on POSIX.
|
||||
if ($installer -eq 'uv') {
|
||||
$uvBin = $null
|
||||
try {
|
||||
$line = (& uv tool dir --bin 2>$null | Select-Object -First 1)
|
||||
if ($line) { $uvBin = $line.Trim() }
|
||||
} catch { }
|
||||
if ($uvBin -and (Test-Path (Join-Path $uvBin 'opensquilla.exe') -PathType Leaf)) {
|
||||
return $uvBin
|
||||
}
|
||||
$fallback = Join-Path $HOME '.local\bin'
|
||||
if (Test-Path (Join-Path $fallback 'opensquilla.exe') -PathType Leaf) {
|
||||
return $fallback
|
||||
}
|
||||
return $null
|
||||
} else {
|
||||
$scriptsDir = $null
|
||||
try {
|
||||
$line = (& python -c "import sysconfig; print(sysconfig.get_path('scripts'))" 2>$null | Select-Object -First 1)
|
||||
if ($line) { $scriptsDir = $line.Trim() }
|
||||
} catch { }
|
||||
if ($scriptsDir -and (Test-Path (Join-Path $scriptsDir 'opensquilla.exe') -PathType Leaf)) {
|
||||
return $scriptsDir
|
||||
}
|
||||
return $null
|
||||
}
|
||||
}
|
||||
|
||||
function Test-DirOnUserPath {
|
||||
param([string]$Dir)
|
||||
if (-not $Dir) { return $false }
|
||||
$userPath = [Environment]::GetEnvironmentVariable('Path', 'User')
|
||||
if (-not $userPath) { return $false }
|
||||
$target = $Dir.TrimEnd('\')
|
||||
foreach ($entry in ($userPath -split ';')) {
|
||||
if ([string]::IsNullOrWhiteSpace($entry)) { continue }
|
||||
if ([string]::Equals($entry.TrimEnd('\'), $target, [System.StringComparison]::OrdinalIgnoreCase)) {
|
||||
return $true
|
||||
}
|
||||
}
|
||||
return $false
|
||||
}
|
||||
|
||||
function Write-PathHint {
|
||||
# Verify the just-installed entry point is reachable from a fresh shell.
|
||||
# install_source.sh runs the same smoke check on POSIX; this brings the
|
||||
# PowerShell installer to parity so a "successful" install does not leave
|
||||
# the user with an unresolvable `opensquilla` command (see issue #500).
|
||||
$entryDir = Resolve-EntrypointDir
|
||||
if (-not $entryDir) {
|
||||
Write-Warning 'install_source.ps1: could not locate the installed `opensquilla` entry point to verify PATH.'
|
||||
Write-Warning "install_source.ps1: if `opensquilla` is not recognized, run 'uv tool update-shell' and open a new terminal."
|
||||
return
|
||||
}
|
||||
if (Test-DirOnUserPath -Dir $entryDir) {
|
||||
Write-Host "install_source.ps1: entry points are on PATH ($entryDir)."
|
||||
return
|
||||
}
|
||||
Write-Warning "install_source.ps1: entry points are NOT on PATH: $entryDir"
|
||||
Write-Warning 'install_source.ps1: `opensquilla` will not be found in a new terminal until this is fixed.'
|
||||
Write-Warning 'install_source.ps1: fix it with one of:'
|
||||
Write-Warning ' uv tool update-shell # uv official PATH configurator (recommended)'
|
||||
$oneLiner = '[Environment]::SetEnvironmentVariable(''Path'', [Environment]::GetEnvironmentVariable(''Path'',''User'') + '';{0}'', ''User'')' -f $entryDir
|
||||
Write-Warning " $oneLiner # or add this dir to user PATH manually"
|
||||
Write-Warning "install_source.ps1: then open a new terminal and run 'opensquilla onboard'."
|
||||
}
|
||||
|
||||
if ($dryRun) {
|
||||
Write-Host "install_source.ps1: dry-run — would run: $installCmd"
|
||||
Write-Host "install_source.ps1: dry-run — prefix: $prefix"
|
||||
Test-SquillaRouterAssets -WarnOnly
|
||||
Write-Banner
|
||||
if ($env:OPENSQUILLA_LISTEN -eq '0.0.0.0') {
|
||||
Write-ListenWarning
|
||||
}
|
||||
exit 0
|
||||
}
|
||||
|
||||
# --- execute ---------------------------------------------------------------
|
||||
|
||||
Install-WindowsVCRedistIfNeeded
|
||||
Test-SquillaRouterAssets
|
||||
|
||||
Write-Host "install_source.ps1: installing via $installer into prefix $prefix"
|
||||
Write-Host "install_source.ps1: running: $installCmd"
|
||||
if ($installer -eq 'uv') {
|
||||
& uv @installArgs
|
||||
} else {
|
||||
& python @installArgs
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Error "install_source.ps1: install command failed with exit code $LASTEXITCODE."
|
||||
Write-Error 'install_source.ps1: Close any running OpenSquilla gateway or shell using the existing tool environment, then retry.'
|
||||
exit $LASTEXITCODE
|
||||
}
|
||||
|
||||
# Write an install receipt to aid `opensquilla uninstall`. Best-effort.
|
||||
try {
|
||||
$receiptHome = if ($env:OPENSQUILLA_STATE_DIR) { $env:OPENSQUILLA_STATE_DIR } else { Join-Path $HOME '.opensquilla' }
|
||||
$receiptMethod = if ($installer -eq 'uv') { 'uv-tool' } else { 'pip' }
|
||||
New-Item -ItemType Directory -Force -Path $receiptHome | Out-Null
|
||||
$receipt = [ordered]@{
|
||||
version = 1
|
||||
install_method = $receiptMethod
|
||||
installed_at = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')
|
||||
entrypoints = @()
|
||||
owned_paths = @()
|
||||
data_root = $receiptHome
|
||||
}
|
||||
$receipt | ConvertTo-Json | Set-Content -Path (Join-Path $receiptHome 'install-receipt.json') -Encoding utf8
|
||||
} catch {
|
||||
# Receipt is optional; never fail the install over it.
|
||||
}
|
||||
|
||||
# Smoke-check the just-installed entry point is reachable from a fresh shell.
|
||||
# Runs after install only (dry-run exits above), matching install_source.sh.
|
||||
Write-PathHint
|
||||
|
||||
Write-Banner
|
||||
if ($env:OPENSQUILLA_LISTEN -eq '0.0.0.0') {
|
||||
Write-ListenWarning
|
||||
}
|
||||
Executable
+314
@@ -0,0 +1,314 @@
|
||||
#!/usr/bin/env bash
|
||||
# install_source.sh - user-local OpenSquilla installer (no sudo).
|
||||
#
|
||||
# Installer contract:
|
||||
# - installs into a user-owned prefix (never /usr/local, /opt, or admin paths)
|
||||
# - prefers uv tool install; falls back to pip --user; errors clearly if neither exists
|
||||
# - defaults to the "recommended" runtime profile (memory + bundled v4 router)
|
||||
# and allows `OPENSQUILLA_INSTALL_PROFILE=core` to opt back down
|
||||
# - prints a post-install banner documenting the default bind
|
||||
# (127.0.0.1:18791) and the explicit opt-in required to expose the gateway
|
||||
# on the network (--listen 0.0.0.0 or OPENSQUILLA_LISTEN=0.0.0.0)
|
||||
# - adds an extra WARNING when the operator requested network exposure at
|
||||
# install time via OPENSQUILLA_LISTEN=0.0.0.0
|
||||
#
|
||||
# Dry-run: export OPENSQUILLA_INSTALL_DRY_RUN=1 to print the install plan + banner
|
||||
# without touching the system.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
cli_profile=""
|
||||
cli_extras=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--profile)
|
||||
cli_profile="${2:?install_source.sh: --profile requires a value}"
|
||||
shift 2
|
||||
;;
|
||||
--profile=*)
|
||||
cli_profile="${1#*=}"
|
||||
shift
|
||||
;;
|
||||
--extras)
|
||||
cli_extras="${2:?install_source.sh: --extras requires a value}"
|
||||
shift 2
|
||||
;;
|
||||
--extras=*)
|
||||
cli_extras="${1#*=}"
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
cat <<HELP
|
||||
Usage: bash scripts/install_source.sh [--profile recommended|core] [--extras name[,name]]
|
||||
|
||||
Environment equivalents:
|
||||
OPENSQUILLA_INSTALL_PROFILE=recommended|core
|
||||
OPENSQUILLA_INSTALL_EXTRAS=matrix
|
||||
OPENSQUILLA_INSTALL_DRY_RUN=1
|
||||
HELP
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "install_source.sh: unknown argument '$1'." >&2
|
||||
echo "install_source.sh: run 'bash scripts/install_source.sh --help' for usage." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# --- prefix resolution ------------------------------------------------------
|
||||
|
||||
if [[ -n "${OPENSQUILLA_PREFIX:-}" ]]; then
|
||||
prefix="${OPENSQUILLA_PREFIX}"
|
||||
elif [[ -n "${XDG_DATA_HOME:-}" ]]; then
|
||||
prefix="${XDG_DATA_HOME}/opensquilla"
|
||||
else
|
||||
prefix="${HOME}/.local"
|
||||
fi
|
||||
|
||||
dry_run="${OPENSQUILLA_INSTALL_DRY_RUN:-0}"
|
||||
profile="${cli_profile:-${OPENSQUILLA_INSTALL_PROFILE:-recommended}}"
|
||||
|
||||
valid_extras=" matrix matrix-e2e document-extras "
|
||||
extras_csv="${OPENSQUILLA_INSTALL_EXTRAS:-}"
|
||||
if [[ -n "${cli_extras}" ]]; then
|
||||
extras_csv="${extras_csv}${extras_csv:+,}${cli_extras}"
|
||||
fi
|
||||
extras_csv="${extras_csv// /,}"
|
||||
raw_extras=()
|
||||
if [[ -n "${extras_csv}" ]]; then
|
||||
IFS=',' read -r -a raw_extras <<< "${extras_csv}"
|
||||
fi
|
||||
install_extras=()
|
||||
if (( ${#raw_extras[@]} > 0 )); then
|
||||
for extra in "${raw_extras[@]}"; do
|
||||
[[ -n "${extra}" ]] || continue
|
||||
if [[ "${valid_extras}" != *" ${extra} "* ]]; then
|
||||
echo "install_source.sh: unsupported extra '${extra}'." >&2
|
||||
echo "install_source.sh: supported extras:${valid_extras}" >&2
|
||||
exit 1
|
||||
fi
|
||||
duplicate=0
|
||||
if (( ${#install_extras[@]} > 0 )); then
|
||||
for existing in "${install_extras[@]}"; do
|
||||
if [[ "${existing}" == "${extra}" ]]; then
|
||||
duplicate=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [[ "${duplicate}" -eq 0 ]]; then
|
||||
install_extras+=("${extra}")
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
case "${profile}" in
|
||||
core|minimal)
|
||||
profile="core"
|
||||
target_extras=()
|
||||
;;
|
||||
recommended)
|
||||
target_extras=(recommended)
|
||||
;;
|
||||
*)
|
||||
echo "install_source.sh: unsupported OPENSQUILLA_INSTALL_PROFILE='${profile}'." >&2
|
||||
echo "install_source.sh: supported profiles: core, recommended" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
if (( ${#install_extras[@]} > 0 )); then
|
||||
target_extras+=("${install_extras[@]}")
|
||||
fi
|
||||
if (( ${#target_extras[@]} > 0 )); then
|
||||
joined_extras="$(IFS=,; echo "${target_extras[*]}")"
|
||||
install_target=".[${joined_extras}]"
|
||||
else
|
||||
install_target="."
|
||||
fi
|
||||
|
||||
check_squilla_router_assets() {
|
||||
local mode="${1:-strict}"
|
||||
if [[ "${profile}" != "recommended" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local model_root="src/opensquilla/squilla_router/models"
|
||||
local pointer_line="version https://git-lfs.github.com/spec/v1"
|
||||
local required=(
|
||||
"${model_root}/v4.2_phase3_inference/lgbm_main.bin"
|
||||
"${model_root}/v4.2_phase3_inference/router.runtime.yaml"
|
||||
"${model_root}/v4.2_phase3_inference/mlp/model.onnx"
|
||||
"${model_root}/v4.2_phase3_inference/features/tfidf.pkl"
|
||||
"${model_root}/v4.2_phase3_inference/bge_onnx/model.onnx"
|
||||
)
|
||||
local missing=()
|
||||
local pointers=()
|
||||
local path=""
|
||||
for path in "${required[@]}"; do
|
||||
if [[ ! -f "${path}" ]]; then
|
||||
missing+=("${path}")
|
||||
continue
|
||||
fi
|
||||
if LC_ALL=C grep -q -m 1 -F -x "${pointer_line}" "${path}" 2>/dev/null; then
|
||||
pointers+=("${path}")
|
||||
fi
|
||||
done
|
||||
if (( ${#missing[@]} > 0 || ${#pointers[@]} > 0 )); then
|
||||
if [[ "${mode}" == "warn" ]]; then
|
||||
echo "install_source.sh: dry-run note — real recommended install would fail until bundled squilla-router v4 assets are available in this checkout." >&2
|
||||
else
|
||||
echo "install_source.sh: bundled squilla-router v4 assets are unavailable in this checkout." >&2
|
||||
fi
|
||||
if (( ${#missing[@]} > 0 )); then
|
||||
echo "install_source.sh: missing assets: ${missing[*]}" >&2
|
||||
fi
|
||||
if (( ${#pointers[@]} > 0 )); then
|
||||
echo "install_source.sh: Git LFS pointer files detected: ${pointers[*]}" >&2
|
||||
fi
|
||||
echo 'install_source.sh: run `git lfs install` once, then:' >&2
|
||||
echo 'install_source.sh: git lfs pull --include="src/opensquilla/squilla_router/models/**"' >&2
|
||||
echo 'install_source.sh: or retry with OPENSQUILLA_INSTALL_PROFILE=core for the minimal runtime.' >&2
|
||||
if [[ "${mode}" == "warn" ]]; then
|
||||
return 0
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# --- installer selection ----------------------------------------------------
|
||||
|
||||
installer=""
|
||||
install_args=()
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
installer="uv"
|
||||
install_args=(uv tool install --python 3.12 --force --reinstall-package opensquilla "${install_target}")
|
||||
elif command -v python3 >/dev/null 2>&1 \
|
||||
&& python3 -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 12) else 1)'; then
|
||||
installer="pip"
|
||||
install_args=(python3 -m pip install --user "${install_target}")
|
||||
else
|
||||
# No uv, and the ambient python3 is missing or older than 3.12. Do NOT
|
||||
# silently pip-install onto an unsupported interpreter: that leaves a
|
||||
# broken `opensquilla` on PATH and makes coding mode fall back to manual
|
||||
# edits. Fail loud and point at uv, which provisions its own 3.12.
|
||||
if command -v python3 >/dev/null 2>&1; then
|
||||
_ambient_py="$(python3 -V 2>&1)"
|
||||
else
|
||||
_ambient_py="none"
|
||||
fi
|
||||
echo "install_source.sh: cannot install - uv not found and python3 (${_ambient_py}) is older than 3.12." >&2
|
||||
echo "install_source.sh: OpenSquilla requires Python >= 3.12 (pyproject 'requires-python')." >&2
|
||||
echo "install_source.sh: easiest fix - install uv; it brings its own 3.12, no system Python needed:" >&2
|
||||
echo "install_source.sh: curl -LsSf https://astral.sh/uv/install.sh | sh" >&2
|
||||
echo "install_source.sh: then re-run: bash scripts/install_source.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
install_cmd="${install_args[*]}"
|
||||
|
||||
# --- banner -----------------------------------------------------------------
|
||||
|
||||
print_banner() {
|
||||
cat <<BANNER
|
||||
----------------------------------------------------------------------------
|
||||
OpenSquilla installed via ${installer} -> ${prefix} (profile: ${profile})
|
||||
Extras: $(if (( ${#install_extras[@]} > 0 )); then IFS=,; echo "${install_extras[*]}"; else echo "none"; fi)
|
||||
|
||||
Default gateway bind: 127.0.0.1:18791 (loopback only)
|
||||
Network exposure is opt-in only. To expose the gateway on the network you
|
||||
must use one of:
|
||||
- CLI flag: opensquilla gateway run --listen 0.0.0.0
|
||||
- Env var: OPENSQUILLA_LISTEN=0.0.0.0 opensquilla gateway run
|
||||
|
||||
Reminder: only expose 0.0.0.0 behind a trusted reverse proxy or VPN. The
|
||||
gateway's first-class auth assumes loopback-scope by default.
|
||||
----------------------------------------------------------------------------
|
||||
BANNER
|
||||
}
|
||||
|
||||
print_listen_warning() {
|
||||
cat <<WARNING
|
||||
WARNING: you have selected network-exposed default - ensure you
|
||||
understand the blast radius. The gateway will bind to 0.0.0.0 and be
|
||||
reachable from every interface on this host.
|
||||
WARNING
|
||||
}
|
||||
|
||||
verify_install() {
|
||||
# Catch a broken/partial install now, not mid-task. A non-runnable
|
||||
# code-task is exactly what makes coding mode silently degrade.
|
||||
# Prefer the JUST-installed binary over any stale `opensquilla` earlier
|
||||
# on PATH (uv tool / pip --user land outside the default PATH).
|
||||
local bin=""
|
||||
if [[ "${installer}" == "uv" ]]; then
|
||||
local uv_bin
|
||||
uv_bin="$(uv tool dir --bin 2>/dev/null || true)"
|
||||
[[ -n "${uv_bin}" && -x "${uv_bin}/opensquilla" ]] && bin="${uv_bin}/opensquilla"
|
||||
fi
|
||||
if [[ -z "${bin}" && -x "${HOME}/.local/bin/opensquilla" ]]; then
|
||||
bin="${HOME}/.local/bin/opensquilla"
|
||||
fi
|
||||
if [[ -z "${bin}" ]] && command -v opensquilla >/dev/null 2>&1; then
|
||||
bin="opensquilla"
|
||||
fi
|
||||
# Coding mode requires `opensquilla code-task`, so verify THAT, not just --version.
|
||||
if [[ -n "${bin}" ]] && "${bin}" code-task --help >/dev/null 2>&1; then
|
||||
echo "install_source.sh: verified - 'opensquilla code-task' is runnable"
|
||||
else
|
||||
echo "install_source.sh: WARNING - 'opensquilla code-task' is not runnable yet." >&2
|
||||
echo "install_source.sh: run 'uv tool update-shell' (or open a new shell), then: opensquilla code-task --help" >&2
|
||||
fi
|
||||
command -v git >/dev/null 2>&1 || echo "install_source.sh: WARNING - 'git' not found; code-task cannot clone repositories without it." >&2
|
||||
command -v node >/dev/null 2>&1 || echo "install_source.sh: note - 'node' not found (only needed for code-task build-mode apps)." >&2
|
||||
}
|
||||
|
||||
if [[ "${dry_run}" = "1" ]]; then
|
||||
echo "install_source.sh: dry-run — would run: ${install_cmd}"
|
||||
echo "install_source.sh: dry-run — prefix: ${prefix}"
|
||||
check_squilla_router_assets warn
|
||||
print_banner
|
||||
if [[ "${OPENSQUILLA_LISTEN:-}" = "0.0.0.0" ]]; then
|
||||
print_listen_warning
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --- execute ---------------------------------------------------------------
|
||||
|
||||
check_squilla_router_assets
|
||||
|
||||
echo "install_source.sh: installing via ${installer} into prefix ${prefix}"
|
||||
echo "install_source.sh: running: ${install_cmd}"
|
||||
"${install_args[@]}"
|
||||
|
||||
verify_install
|
||||
|
||||
# Write an install receipt to aid `opensquilla uninstall`. Best-effort.
|
||||
write_install_receipt() {
|
||||
home="${OPENSQUILLA_STATE_DIR:-${HOME}/.opensquilla}"
|
||||
receipt="${home}/install-receipt.json"
|
||||
installed_at="$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo "")"
|
||||
if [[ "${installer}" == "uv" ]]; then
|
||||
method="uv-tool"
|
||||
else
|
||||
method="pip"
|
||||
fi
|
||||
mkdir -p "${home}" 2>/dev/null || return 0
|
||||
cat >"${receipt}" 2>/dev/null <<RECEIPT || return 0
|
||||
{
|
||||
"version": 1,
|
||||
"install_method": "${method}",
|
||||
"installed_at": "${installed_at}",
|
||||
"entrypoints": [],
|
||||
"owned_paths": [],
|
||||
"data_root": "${home}"
|
||||
}
|
||||
RECEIPT
|
||||
chmod 600 "${receipt}" 2>/dev/null || true
|
||||
}
|
||||
write_install_receipt || true
|
||||
|
||||
print_banner
|
||||
if [[ "${OPENSQUILLA_LISTEN:-}" = "0.0.0.0" ]]; then
|
||||
print_listen_warning
|
||||
fi
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Live smoke for the openai_codex (ChatGPT OAuth) provider.
|
||||
|
||||
Gated on the Codex CLI auth file existing (``$CODEX_HOME/auth.json`` or
|
||||
``~/.codex/auth.json``); exits 2 with guidance when it does not. Runs one
|
||||
text turn and one tool-call turn against the real ChatGPT backend and
|
||||
prints a compact JSON verdict. No secrets are printed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from opensquilla.provider.codex_auth import CodexAuthError, codex_auth_path, load_codex_credentials
|
||||
from opensquilla.provider.openai_codex import OpenAICodexProvider
|
||||
from opensquilla.provider.types import (
|
||||
ChatConfig,
|
||||
DoneEvent,
|
||||
ErrorEvent,
|
||||
Message,
|
||||
ReasoningDeltaEvent,
|
||||
TextDeltaEvent,
|
||||
ToolDefinition,
|
||||
ToolInputSchema,
|
||||
ToolUseEndEvent,
|
||||
)
|
||||
|
||||
|
||||
async def _run_turn(
|
||||
provider: OpenAICodexProvider,
|
||||
prompt: str,
|
||||
*,
|
||||
tools: list[ToolDefinition] | None = None,
|
||||
timeout: float = 120.0,
|
||||
) -> dict[str, Any]:
|
||||
start = time.perf_counter()
|
||||
text_parts: list[str] = []
|
||||
tool_ends: list[dict[str, Any]] = []
|
||||
reasoning_chars = 0
|
||||
done: DoneEvent | None = None
|
||||
error: ErrorEvent | None = None
|
||||
async for event in provider.chat(
|
||||
[Message(role="user", content=prompt)],
|
||||
tools=tools,
|
||||
config=ChatConfig(max_tokens=512, timeout=timeout),
|
||||
):
|
||||
if isinstance(event, TextDeltaEvent):
|
||||
text_parts.append(event.text)
|
||||
elif isinstance(event, ReasoningDeltaEvent):
|
||||
reasoning_chars += len(event.text)
|
||||
elif isinstance(event, ToolUseEndEvent):
|
||||
tool_ends.append({"name": event.tool_name, "arguments": event.arguments})
|
||||
elif isinstance(event, DoneEvent):
|
||||
done = event
|
||||
elif isinstance(event, ErrorEvent):
|
||||
error = event
|
||||
return {
|
||||
"content": "".join(text_parts),
|
||||
"tool_calls": tool_ends,
|
||||
"reasoning_chars": reasoning_chars,
|
||||
"usage": (
|
||||
{
|
||||
"input_tokens": done.input_tokens,
|
||||
"output_tokens": done.output_tokens,
|
||||
"cached_tokens": done.cached_tokens,
|
||||
"reasoning_tokens": done.reasoning_tokens,
|
||||
"model": done.model,
|
||||
"stop_reason": done.stop_reason,
|
||||
}
|
||||
if done
|
||||
else None
|
||||
),
|
||||
"error": {"code": error.code, "message": error.message[:300]} if error else None,
|
||||
"latency_ms": int((time.perf_counter() - start) * 1000),
|
||||
}
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--model", default="gpt-5.5")
|
||||
parser.add_argument("--timeout", type=float, default=120.0)
|
||||
parser.add_argument("--skip-tools", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
credentials = load_codex_credentials()
|
||||
except CodexAuthError as exc:
|
||||
print(f"SKIP: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
account_hint = credentials.account_id[:8] if credentials.account_id else "(from JWT)"
|
||||
print(f"auth file: {codex_auth_path()} (account {account_hint}...)", file=sys.stderr)
|
||||
|
||||
provider = OpenAICodexProvider(model=args.model)
|
||||
report: dict[str, Any] = {"model": args.model}
|
||||
|
||||
marker = f"codex smoke {int(time.time())}"
|
||||
text = await _run_turn(
|
||||
provider, f"Reply exactly with: {marker}", timeout=args.timeout
|
||||
)
|
||||
text["marker_present"] = marker in text.pop("content", "")
|
||||
report["text_turn"] = text
|
||||
|
||||
if not args.skip_tools:
|
||||
tool = ToolDefinition(
|
||||
name="get_weather",
|
||||
description="Get current weather for a city.",
|
||||
input_schema=ToolInputSchema(
|
||||
properties={"city": {"type": "string"}}, required=["city"]
|
||||
),
|
||||
)
|
||||
tool_turn = await _run_turn(
|
||||
provider,
|
||||
"Use the get_weather tool to check the weather in Tokyo.",
|
||||
tools=[tool],
|
||||
timeout=args.timeout,
|
||||
)
|
||||
tool_turn.pop("content", None)
|
||||
report["tool_turn"] = tool_turn
|
||||
|
||||
print(json.dumps(report, indent=2, ensure_ascii=False))
|
||||
text_ok = bool(report["text_turn"].get("marker_present")) and not report["text_turn"]["error"]
|
||||
tool_ok = args.skip_tools or (
|
||||
bool(report.get("tool_turn", {}).get("tool_calls"))
|
||||
and not report.get("tool_turn", {}).get("error")
|
||||
)
|
||||
return 0 if text_ok and tool_ok else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()))
|
||||
@@ -0,0 +1,205 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Live end-to-end test of cross-provider router tiers (P6: R3 + gate + R2 flag).
|
||||
|
||||
Exercises the real turn-path helpers against real APIs — credential
|
||||
resolution, the execution gate, and ModelSelector.override_provider_config
|
||||
— then runs a chat turn on the SWITCHED provider and confirms attribution
|
||||
followed (active provider id + response model = the tier's provider, with
|
||||
the previous primary retained as a fallback).
|
||||
|
||||
Loads keys from ``.env`` in the cwd; needs at least two of
|
||||
OPENROUTER_API_KEY / DEEPSEEK_API_KEY / OPENAI_API_KEY. No secrets printed.
|
||||
Exit 0 iff every check passes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from opensquilla.engine.selector_override import (
|
||||
apply_model_override,
|
||||
cross_provider_tier_config,
|
||||
resolve_tier_provider_config,
|
||||
)
|
||||
from opensquilla.gateway.config import GatewayConfig, LlmProviderConfig, LlmProviderProfile
|
||||
from opensquilla.provider.registry import get_provider_spec
|
||||
from opensquilla.provider.selector import ModelSelector, ProviderConfig, SelectorConfig
|
||||
from opensquilla.provider.types import (
|
||||
ChatConfig,
|
||||
DoneEvent,
|
||||
ErrorEvent,
|
||||
Message,
|
||||
TextDeltaEvent,
|
||||
)
|
||||
|
||||
|
||||
def _load_env(path: Path = Path(".env")) -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
for line in path.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
key, value = line.split("=", 1)
|
||||
os.environ.setdefault(key.strip(), value.strip())
|
||||
|
||||
|
||||
def _cfg(
|
||||
*,
|
||||
primary: str,
|
||||
flag: bool,
|
||||
profiles: dict[str, LlmProviderProfile] | None = None,
|
||||
) -> GatewayConfig:
|
||||
cfg = GatewayConfig()
|
||||
cfg.llm = LlmProviderConfig(provider=primary, model="m", api_key="primary-key")
|
||||
cfg.squilla_router.cross_provider_tiers = flag
|
||||
cfg.llm_profiles = profiles or {}
|
||||
return cfg
|
||||
|
||||
|
||||
def _key_for(provider_id: str) -> str:
|
||||
return os.environ.get(get_provider_spec(provider_id).env_key, "").strip()
|
||||
|
||||
|
||||
async def _run_turn(provider: Any, prompt: str) -> dict[str, Any]:
|
||||
start = time.perf_counter()
|
||||
text: list[str] = []
|
||||
done: DoneEvent | None = None
|
||||
error: ErrorEvent | None = None
|
||||
async for event in provider.chat(
|
||||
[Message(role="user", content=prompt)],
|
||||
config=ChatConfig(max_tokens=64, timeout=90.0),
|
||||
):
|
||||
if isinstance(event, TextDeltaEvent):
|
||||
text.append(event.text)
|
||||
elif isinstance(event, DoneEvent):
|
||||
done = event
|
||||
elif isinstance(event, ErrorEvent):
|
||||
error = event
|
||||
return {
|
||||
"content": "".join(text),
|
||||
"response_model": done.model if done else None,
|
||||
"error": error.message[:120] if error else None,
|
||||
"latency_ms": int((time.perf_counter() - start) * 1000),
|
||||
}
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
_load_env()
|
||||
report: dict[str, Any] = {}
|
||||
verdicts: list[bool] = []
|
||||
|
||||
# 1. Gate: flag OFF -> tier config not built (turn stays on primary).
|
||||
off = cross_provider_tier_config(
|
||||
_cfg(primary="openrouter", flag=False),
|
||||
{"routing_applied": True, "routed_provider": "deepseek"},
|
||||
"deepseek-chat",
|
||||
active_provider_id="openrouter",
|
||||
)
|
||||
report["gate_flag_off"] = {"tier_config": off}
|
||||
verdicts.append(off is None)
|
||||
|
||||
# 2. Gate: flag ON but credentials unresolvable -> None + warning.
|
||||
os.environ.pop("MISTRAL_API_KEY", None)
|
||||
unresolvable = cross_provider_tier_config(
|
||||
_cfg(primary="openrouter", flag=True),
|
||||
{"routing_applied": True, "routed_provider": "mistral"},
|
||||
"mistral-large-latest",
|
||||
active_provider_id="openrouter",
|
||||
)
|
||||
report["gate_unresolvable_creds"] = {"tier_config": unresolvable}
|
||||
verdicts.append(unresolvable is None)
|
||||
|
||||
# 3. Credential resolution: env source, profile override, R2 flag.
|
||||
env_cfg = resolve_tier_provider_config(
|
||||
_cfg(primary="openrouter", flag=True), "deepseek", "deepseek-chat"
|
||||
)
|
||||
prof_cfg = resolve_tier_provider_config(
|
||||
_cfg(
|
||||
primary="openrouter",
|
||||
flag=True,
|
||||
profiles={"deepseek": LlmProviderProfile(api_key="profile-override-key")},
|
||||
),
|
||||
"deepseek",
|
||||
"deepseek-chat",
|
||||
)
|
||||
creds_ok = bool(
|
||||
env_cfg
|
||||
and env_cfg.api_key == os.environ.get("DEEPSEEK_API_KEY", "")
|
||||
and env_cfg.replay_provider_state is False
|
||||
and prof_cfg
|
||||
and prof_cfg.api_key == "profile-override-key"
|
||||
)
|
||||
report["credential_resolution"] = {
|
||||
"env_resolved": bool(env_cfg and env_cfg.api_key),
|
||||
"profile_overrides_env": bool(prof_cfg) and prof_cfg.api_key == "profile-override-key",
|
||||
"replay_provider_state_disabled": bool(env_cfg) and env_cfg.replay_provider_state is False,
|
||||
}
|
||||
verdicts.append(creds_ok)
|
||||
|
||||
# 4. Live cross-provider execution: each turn must run on the tier
|
||||
# provider (proven by response model) with the primary kept as
|
||||
# fallback. Only pairs whose keys are present are attempted.
|
||||
candidate_cases = [
|
||||
("openrouter", "deepseek/deepseek-v4-flash", "deepseek", "deepseek-chat"),
|
||||
("deepseek", "deepseek-chat", "openai", "gpt-4.1"),
|
||||
]
|
||||
report["live_cross_provider"] = []
|
||||
for primary_prov, primary_model, tier_prov, tier_model in candidate_cases:
|
||||
if not _key_for(primary_prov) or not _key_for(tier_prov):
|
||||
report["live_cross_provider"].append(
|
||||
{"primary": primary_prov, "tier_provider": tier_prov, "skipped": "missing key"}
|
||||
)
|
||||
continue
|
||||
selector = ModelSelector(
|
||||
SelectorConfig(
|
||||
primary=ProviderConfig(
|
||||
primary_prov, primary_model, api_key=_key_for(primary_prov)
|
||||
)
|
||||
)
|
||||
)
|
||||
cfg = _cfg(primary=primary_prov, flag=True)
|
||||
metadata: dict[str, Any] = {"routing_applied": True, "routed_provider": tier_prov}
|
||||
tier_config = cross_provider_tier_config(
|
||||
cfg, metadata, tier_model, active_provider_id=selector.active_provider_id
|
||||
)
|
||||
provider = apply_model_override(
|
||||
selector,
|
||||
tier_model,
|
||||
turn_metadata=metadata,
|
||||
realign_routed_model=False,
|
||||
tier_provider_config=tier_config,
|
||||
)
|
||||
primary_kept = selector.has_fallback() and selector.current_config is not None
|
||||
turn = await _run_turn(provider, "Reply with the single word: switched")
|
||||
case_ok = bool(
|
||||
selector.active_provider_id == tier_prov
|
||||
and metadata.get("routed_provider_applied") == tier_prov
|
||||
and turn["error"] is None
|
||||
and turn["response_model"]
|
||||
and primary_kept
|
||||
)
|
||||
report["live_cross_provider"].append(
|
||||
{
|
||||
"primary": primary_prov,
|
||||
"tier_provider": tier_prov,
|
||||
"active_provider_after_switch": selector.active_provider_id,
|
||||
"primary_kept_as_fallback": primary_kept,
|
||||
"turn": turn,
|
||||
"ok": case_ok,
|
||||
}
|
||||
)
|
||||
verdicts.append(case_ok)
|
||||
|
||||
report["all_passed"] = all(verdicts)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0 if report["all_passed"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()))
|
||||
@@ -0,0 +1,262 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Live smoke DeepSeek thinking-mode tool replay requirements."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from opensquilla.provider.model_catalog import ModelCatalog
|
||||
from opensquilla.provider.registry import get_provider_spec
|
||||
from opensquilla.provider.selector import ProviderConfig, _build_provider
|
||||
from opensquilla.provider.types import (
|
||||
ChatConfig,
|
||||
ContentBlockToolResult,
|
||||
ContentBlockToolUse,
|
||||
DoneEvent,
|
||||
ErrorEvent,
|
||||
Message,
|
||||
TextDeltaEvent,
|
||||
ToolDefinition,
|
||||
ToolInputSchema,
|
||||
ToolUseEndEvent,
|
||||
ToolUseStartEvent,
|
||||
)
|
||||
|
||||
|
||||
def _load_env_quietly(path: Path = Path(".env")) -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
for raw_line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
key = key.strip()
|
||||
value = value.strip().strip('"').strip("'")
|
||||
if key and key not in os.environ:
|
||||
os.environ[key] = value
|
||||
|
||||
|
||||
def _tool_def() -> ToolDefinition:
|
||||
return ToolDefinition(
|
||||
name="lookup_status",
|
||||
description="Return a deterministic status string for a service name.",
|
||||
input_schema=ToolInputSchema(
|
||||
type="object",
|
||||
properties={
|
||||
"service": {
|
||||
"type": "string",
|
||||
"description": "Service name to inspect.",
|
||||
}
|
||||
},
|
||||
required=["service"],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def _collect_call(
|
||||
provider: Any,
|
||||
messages: list[Message],
|
||||
*,
|
||||
tools: list[ToolDefinition],
|
||||
config: ChatConfig,
|
||||
) -> dict[str, Any]:
|
||||
text_parts: list[str] = []
|
||||
tool_events: list[dict[str, Any]] = []
|
||||
done: DoneEvent | None = None
|
||||
error: str = ""
|
||||
start = time.perf_counter()
|
||||
async for event in provider.chat(messages, tools=tools, config=config):
|
||||
if isinstance(event, TextDeltaEvent):
|
||||
text_parts.append(event.text)
|
||||
elif isinstance(event, ToolUseStartEvent):
|
||||
tool_events.append(
|
||||
{
|
||||
"event": "start",
|
||||
"tool_use_id": event.tool_use_id,
|
||||
"tool_name": event.tool_name,
|
||||
}
|
||||
)
|
||||
elif isinstance(event, ToolUseEndEvent):
|
||||
tool_events.append(
|
||||
{
|
||||
"event": "end",
|
||||
"tool_use_id": event.tool_use_id,
|
||||
"tool_name": event.tool_name,
|
||||
"arguments": event.arguments,
|
||||
}
|
||||
)
|
||||
elif isinstance(event, DoneEvent):
|
||||
done = event
|
||||
elif isinstance(event, ErrorEvent):
|
||||
error = event.message or event.code
|
||||
break
|
||||
latency_ms = int((time.perf_counter() - start) * 1000)
|
||||
return {
|
||||
"latency_ms": latency_ms,
|
||||
"text": "".join(text_parts),
|
||||
"tool_events": tool_events,
|
||||
"done": {
|
||||
"present": done is not None,
|
||||
"stop_reason": done.stop_reason if done else "",
|
||||
"input_tokens": done.input_tokens if done else 0,
|
||||
"output_tokens": done.output_tokens if done else 0,
|
||||
"reasoning_tokens": done.reasoning_tokens if done else 0,
|
||||
"reasoning_content_present": bool(done and done.reasoning_content),
|
||||
"reasoning_content_chars": len(done.reasoning_content or "") if done else 0,
|
||||
"model": done.model if done else "",
|
||||
},
|
||||
"_reasoning_content_raw": done.reasoning_content if done else None,
|
||||
"error": error,
|
||||
}
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--model", default="")
|
||||
parser.add_argument("--base-url", default="")
|
||||
parser.add_argument("--max-tokens", type=int, default=512)
|
||||
parser.add_argument("--output", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
_load_env_quietly()
|
||||
spec = get_provider_spec("deepseek")
|
||||
api_key = os.environ.get(spec.env_key, "").strip()
|
||||
model = args.model or os.environ.get("DEEPSEEK_MODEL", "").strip() or "deepseek-v4-pro"
|
||||
base_url = (
|
||||
args.base_url
|
||||
or os.environ.get("DEEPSEEK_BASE_URL", "").strip()
|
||||
or spec.default_base_url
|
||||
)
|
||||
marker = f"DEEPSEEK_TOOL_REPLAY_{int(time.time() * 1000)}"
|
||||
payload: dict[str, Any] = {
|
||||
"generated_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
"provider": "deepseek",
|
||||
"model": model,
|
||||
"base_url": base_url,
|
||||
"env_key": spec.env_key,
|
||||
"key_present": bool(api_key),
|
||||
"marker": marker,
|
||||
}
|
||||
if not api_key:
|
||||
payload["ok"] = False
|
||||
payload["error"] = f"{spec.env_key} is empty"
|
||||
else:
|
||||
provider = _build_provider(
|
||||
ProviderConfig(provider="deepseek", model=model, api_key=api_key, base_url=base_url)
|
||||
)
|
||||
caps = ModelCatalog().get_capabilities(model, provider_name="deepseek", base_url=base_url)
|
||||
config = ChatConfig(
|
||||
max_tokens=args.max_tokens,
|
||||
temperature=None,
|
||||
thinking=True,
|
||||
thinking_budget_tokens=4096,
|
||||
timeout=60.0,
|
||||
model_capabilities=caps,
|
||||
)
|
||||
tools = [_tool_def()]
|
||||
first_messages = [
|
||||
Message(
|
||||
role="user",
|
||||
content=(
|
||||
"You must call lookup_status exactly once before answering. "
|
||||
"Use service='payments'. Do not provide a final answer yet."
|
||||
),
|
||||
)
|
||||
]
|
||||
first = await _collect_call(provider, first_messages, tools=tools, config=config)
|
||||
tool_end = next(
|
||||
(event for event in first["tool_events"] if event.get("event") == "end"),
|
||||
None,
|
||||
)
|
||||
second: dict[str, Any] | None = None
|
||||
replay_messages_without_secret: list[dict[str, Any]] = []
|
||||
if tool_end and first["done"]["reasoning_content_present"]:
|
||||
reasoning_placeholder = "<reasoning_content omitted; present in request object>"
|
||||
assistant_msg = Message(
|
||||
role="assistant",
|
||||
content=[
|
||||
ContentBlockToolUse(
|
||||
id=tool_end["tool_use_id"],
|
||||
name=tool_end["tool_name"],
|
||||
input=tool_end.get("arguments") or {},
|
||||
)
|
||||
],
|
||||
# The actual field is passed to the provider; the artifact only
|
||||
# records its presence/length, not the hidden reasoning text.
|
||||
reasoning_content=first.get("_reasoning_content_raw") or "",
|
||||
)
|
||||
tool_result_msg = Message(
|
||||
role="user",
|
||||
content=[
|
||||
ContentBlockToolResult(
|
||||
tool_use_id=tool_end["tool_use_id"],
|
||||
content="payments status: healthy; queue depth: 0",
|
||||
)
|
||||
],
|
||||
)
|
||||
final_user = Message(
|
||||
role="user",
|
||||
content=f"Now answer exactly with {marker}.",
|
||||
)
|
||||
second_messages = [assistant_msg, tool_result_msg, final_user]
|
||||
replay_messages_without_secret = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_use_id": tool_end["tool_use_id"],
|
||||
"tool_name": tool_end["tool_name"],
|
||||
"reasoning_content": reasoning_placeholder,
|
||||
},
|
||||
{
|
||||
"role": "tool/user",
|
||||
"tool_use_id": tool_end["tool_use_id"],
|
||||
"content": "payments status: healthy; queue depth: 0",
|
||||
},
|
||||
{"role": "user", "content": f"Now answer exactly with {marker}."},
|
||||
]
|
||||
second = await _collect_call(provider, second_messages, tools=tools, config=config)
|
||||
first.pop("_reasoning_content_raw", None)
|
||||
if second:
|
||||
second.pop("_reasoning_content_raw", None)
|
||||
payload.update(
|
||||
{
|
||||
"ok": bool(
|
||||
tool_end
|
||||
and first["done"]["reasoning_content_present"]
|
||||
and second
|
||||
and not second.get("error")
|
||||
and marker in str(second.get("text") or "")
|
||||
),
|
||||
"first_call": first,
|
||||
"second_call": second,
|
||||
"replay_messages_without_secret": replay_messages_without_secret,
|
||||
"chat_config_without_secret": config.model_dump(mode="json"),
|
||||
"failure_reason": None,
|
||||
}
|
||||
)
|
||||
if not payload["ok"]:
|
||||
if not tool_end:
|
||||
payload["failure_reason"] = "first_call_did_not_emit_tool_call"
|
||||
elif not first["done"]["reasoning_content_present"]:
|
||||
payload["failure_reason"] = "first_call_missing_reasoning_content"
|
||||
elif not second:
|
||||
payload["failure_reason"] = "second_call_not_run"
|
||||
elif second.get("error"):
|
||||
payload["failure_reason"] = "second_call_error"
|
||||
else:
|
||||
payload["failure_reason"] = "second_call_marker_missing"
|
||||
output = Path(args.output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
@@ -0,0 +1,283 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Live long-context WebChat smoke.
|
||||
|
||||
Opt-in maintainer gate. Requires OPENROUTER_API_KEY and starts a temporary
|
||||
gateway against a temporary state dir. The smoke verifies that WebChat accepts
|
||||
and completes a turn whose current user input is far above the gateway soft
|
||||
context budget, instead of returning a synchronous context-overflow refusal.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
_THIS_REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(_THIS_REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_THIS_REPO_ROOT))
|
||||
|
||||
from scripts.smoke_v4_phase3_router import ( # noqa: E402
|
||||
REPO_ROOT,
|
||||
SRC_DIR,
|
||||
_free_port,
|
||||
_post_json,
|
||||
_read_json,
|
||||
_read_turn_call_records,
|
||||
_summarize_llm_request_context,
|
||||
_wait_for_assistant_reply,
|
||||
_write_live_gateway_config,
|
||||
)
|
||||
|
||||
|
||||
def _long_message(marker: str, chars: int) -> str:
|
||||
filler = "\n".join(
|
||||
f"long-context-line-{index:05d}: preserve liveness while compacting history."
|
||||
for index in range(max(chars // 72, 1))
|
||||
)
|
||||
return (
|
||||
f"Reply with one short sentence and include marker {marker}. Do not call tools.\n"
|
||||
f"{filler}\n"
|
||||
f"Final reminder: the reply must include {marker}."
|
||||
)
|
||||
|
||||
|
||||
def _enable_memory_distill_failure_simulation(config_path: Path) -> None:
|
||||
text = config_path.read_text(encoding="utf-8")
|
||||
text = text.replace(
|
||||
'debug = false\n',
|
||||
'debug = false\ncontext_budget_tokens = 256\n',
|
||||
1,
|
||||
)
|
||||
text = text.replace(
|
||||
'[memory]\nsource = "state"\n',
|
||||
(
|
||||
'[memory]\n'
|
||||
'source = "state"\n'
|
||||
'flush_enabled = true\n'
|
||||
'flush_timeout_seconds = 0.001\n'
|
||||
'flush_background_timeout_seconds = 0.001\n'
|
||||
'flush_compaction_requires_safe_receipt = false\n'
|
||||
'flush_compaction_safety_mode = "protect"\n'
|
||||
),
|
||||
1,
|
||||
)
|
||||
config_path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def _blocking_memory_errors(turns: list[dict[str, Any]]) -> list[str]:
|
||||
blocked: list[str] = []
|
||||
needles = ("flush failed", "bad json", "memory distill failed")
|
||||
for turn in turns:
|
||||
accepted = str(turn.get("accepted") or "").lower()
|
||||
if any(needle in accepted for needle in needles):
|
||||
blocked.append(str(turn.get("name") or turn.get("index") or "unknown"))
|
||||
return blocked
|
||||
|
||||
|
||||
def run_live_long_context_smoke(
|
||||
*,
|
||||
long_chars: int,
|
||||
timeout_seconds: float,
|
||||
simulate_memory_distill_failure: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
if not os.environ.get("OPENROUTER_API_KEY"):
|
||||
return {
|
||||
"name": "opensquilla_gateway_live_long_context_chat",
|
||||
"ok": False,
|
||||
"error": "OPENROUTER_API_KEY is required",
|
||||
}
|
||||
|
||||
port = _free_port()
|
||||
live_model = os.environ.get("OPENSQUILLA_LIVE_LLM_MODEL", "").strip()
|
||||
session_key = f"live-long-context:{int(time.time() * 1000)}"
|
||||
turns_spec = [
|
||||
{
|
||||
"message": (
|
||||
"Long-context baseline turn: reply with one short sentence. Do not call tools."
|
||||
),
|
||||
"intent": "new_chat",
|
||||
"name": "baseline",
|
||||
},
|
||||
{
|
||||
"message": _long_message("LONG_CONTEXT_CONTINUES", long_chars),
|
||||
"intent": "continue",
|
||||
"name": "oversized_current_input",
|
||||
},
|
||||
]
|
||||
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="opensquilla-live-long-context-",
|
||||
ignore_cleanup_errors=True,
|
||||
) as tmp:
|
||||
tmp_path = Path(tmp)
|
||||
config_path = tmp_path / "live-config.toml"
|
||||
turn_log_dir = tmp_path / "turn-calls"
|
||||
_write_live_gateway_config(config_path, live_model)
|
||||
if simulate_memory_distill_failure:
|
||||
_enable_memory_distill_failure_simulation(config_path)
|
||||
|
||||
env = os.environ.copy()
|
||||
env["PYTHONPATH"] = str(SRC_DIR) + os.pathsep + env.get("PYTHONPATH", "")
|
||||
env["OPENSQUILLA_GATEWAY_CONFIG_PATH"] = str(config_path)
|
||||
env["OPENSQUILLA_STATE_DIR"] = str(tmp_path / "state")
|
||||
env["OPENSQUILLA_MEMORY_DREAM_DISABLED"] = "1"
|
||||
env["OPENSQUILLA_SANDBOX_SANDBOX"] = "false"
|
||||
env["OPENSQUILLA_SANDBOX_SECURITY_GRADING"] = "false"
|
||||
env["OPENSQUILLA_TURN_CALL_LOG"] = "1"
|
||||
env["OPENSQUILLA_TURN_CALL_LOG_DIR"] = str(turn_log_dir)
|
||||
if simulate_memory_distill_failure:
|
||||
env["OPENSQUILLA_SESSION_FLUSH"] = "1"
|
||||
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"opensquilla.cli.main",
|
||||
"gateway",
|
||||
"run",
|
||||
"--port",
|
||||
str(port),
|
||||
"--bind",
|
||||
"127.0.0.1",
|
||||
],
|
||||
cwd=REPO_ROOT,
|
||||
env=env,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
|
||||
turns: list[dict[str, Any]] = []
|
||||
health: dict[str, Any] | None = None
|
||||
usage: dict[str, Any] = {}
|
||||
error: str | None = None
|
||||
stdout_tail = ""
|
||||
stderr_tail = ""
|
||||
try:
|
||||
deadline = time.monotonic() + 45
|
||||
while time.monotonic() < deadline:
|
||||
if proc.poll() is not None:
|
||||
stdout, stderr = proc.communicate(timeout=1)
|
||||
error = f"gateway exited early with code {proc.returncode}: {stderr or stdout}"
|
||||
break
|
||||
try:
|
||||
health = _read_json(f"http://127.0.0.1:{port}/health")
|
||||
break
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError):
|
||||
time.sleep(0.25)
|
||||
if health is None and error is None:
|
||||
error = "gateway did not become healthy before timeout"
|
||||
|
||||
assistant_count = 0
|
||||
if error is None:
|
||||
for index, spec in enumerate(turns_spec, start=1):
|
||||
accepted = _post_json(
|
||||
f"http://127.0.0.1:{port}/api/chat",
|
||||
{
|
||||
"sessionKey": session_key,
|
||||
"message": spec["message"],
|
||||
"intent": spec["intent"],
|
||||
},
|
||||
timeout=20.0,
|
||||
)
|
||||
if accepted.get("ok") is not True:
|
||||
error = f"turn {index} was not accepted: {accepted}"
|
||||
break
|
||||
assistant, history, turn_error = _wait_for_assistant_reply(
|
||||
port=port,
|
||||
session_key=session_key,
|
||||
previous_assistant_count=assistant_count,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
if turn_error:
|
||||
error = f"turn {index} failed: {turn_error}"
|
||||
break
|
||||
assistant_count += 1
|
||||
turns.append(
|
||||
{
|
||||
"index": index,
|
||||
"name": spec["name"],
|
||||
"accepted": accepted,
|
||||
"assistant_text": str((assistant or {}).get("text", "")).strip(),
|
||||
"history_message_count": len((history or {}).get("messages", [])),
|
||||
"message_chars": len(spec["message"]),
|
||||
}
|
||||
)
|
||||
if error is None:
|
||||
usage = _read_json(f"http://127.0.0.1:{port}/api/usage", timeout=5.0)
|
||||
finally:
|
||||
if proc.poll() is None:
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
stdout, stderr = proc.communicate(timeout=1)
|
||||
stdout_tail = (stdout or "")[-2000:]
|
||||
stderr_tail = (stderr or "")[-2000:]
|
||||
turn_call_records = _read_turn_call_records(turn_log_dir)
|
||||
|
||||
context_summary = _summarize_llm_request_context(
|
||||
turn_call_records,
|
||||
session_keys={session_key},
|
||||
)
|
||||
blocking_memory_errors = _blocking_memory_errors(turns)
|
||||
ok = (
|
||||
error is None
|
||||
and len(turns) == len(turns_spec)
|
||||
and all(turn.get("assistant_text") for turn in turns)
|
||||
and int(usage.get("totalTokens", 0) or 0) > 0
|
||||
and not blocking_memory_errors
|
||||
)
|
||||
return {
|
||||
"name": "opensquilla_gateway_live_long_context_chat",
|
||||
"ok": ok,
|
||||
"session_key": session_key,
|
||||
"model": live_model,
|
||||
"long_chars": long_chars,
|
||||
"simulate_memory_distill_failure": simulate_memory_distill_failure,
|
||||
"blocking_memory_errors": blocking_memory_errors,
|
||||
"health": health or {},
|
||||
"turns": turns,
|
||||
"usage": usage,
|
||||
"llm_request_context_summary": context_summary,
|
||||
"error": error,
|
||||
"stdout_tail": stdout_tail,
|
||||
"stderr_tail": stderr_tail,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--long-chars", type=int, default=350_000)
|
||||
parser.add_argument("--timeout-seconds", type=float, default=180.0)
|
||||
parser.add_argument(
|
||||
"--simulate-memory-distill-failure",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Lower the context budget and flush timeouts so pre-compaction "
|
||||
"memory distill cannot block WebChat sendability."
|
||||
),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = run_live_long_context_smoke(
|
||||
long_chars=max(args.long_chars, 1),
|
||||
timeout_seconds=max(args.timeout_seconds, 1.0),
|
||||
simulate_memory_distill_failure=args.simulate_memory_distill_failure,
|
||||
)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True))
|
||||
return 0 if result.get("ok") else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,461 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Live auto-propose E2E for meta-skill-creator.
|
||||
|
||||
This harness intentionally does not accept a user prompt. It verifies the
|
||||
unattended creator path used by cron and dream hooks: aggregate decision-log
|
||||
history, synthesize a candidate through meta-skill-creator, run its gates, and
|
||||
persist a proposal with auto_* provenance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _load_env_file(path: Path | None) -> None:
|
||||
if path is None or not path.is_file():
|
||||
return
|
||||
for raw in path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
key = key.strip()
|
||||
value = value.strip().strip("'\"")
|
||||
if key and key not in os.environ:
|
||||
os.environ[key] = value
|
||||
|
||||
|
||||
def _seed_history(log_dir: Path) -> Path:
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
now = datetime.now(UTC).isoformat()
|
||||
rows: list[dict[str, object]] = []
|
||||
chains = [
|
||||
(["history-explorer", "summarize"], 7),
|
||||
(["multi-search-engine", "summarize"], 4),
|
||||
(["weather", "summarize"], 2),
|
||||
]
|
||||
for chain, count in chains:
|
||||
for _ in range(count):
|
||||
rows.append({
|
||||
"ts": now,
|
||||
"agent_id": "main",
|
||||
"user_message": "recent decision history operational recap",
|
||||
"skills_invoked": chain,
|
||||
})
|
||||
path = log_dir / f"decisions-{datetime.now(UTC).strftime('%Y%m%d')}.jsonl"
|
||||
path.write_text(
|
||||
"".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
async def _run(args: argparse.Namespace) -> dict[str, Any]:
|
||||
home = args.home.expanduser().resolve()
|
||||
log_dir = args.log_dir.expanduser().resolve() if args.log_dir else home / "logs"
|
||||
proposals_dir = (
|
||||
args.proposals_dir.expanduser().resolve()
|
||||
if args.proposals_dir
|
||||
else home / "proposals"
|
||||
)
|
||||
workspace_dir = args.workspace.expanduser().resolve() if args.workspace else home / "workspace"
|
||||
home.mkdir(parents=True, exist_ok=True)
|
||||
workspace_dir.mkdir(parents=True, exist_ok=True)
|
||||
if args.seed_history:
|
||||
history_path = _seed_history(log_dir)
|
||||
else:
|
||||
history_path = None
|
||||
|
||||
os.environ["OPENSQUILLA_STATE_DIR"] = str(home)
|
||||
os.environ["OPENSQUILLA_LOG_DIR"] = str(log_dir)
|
||||
os.environ["OPENSQUILLA_LLM_PROVIDER"] = args.provider
|
||||
os.environ["OPENSQUILLA_LLM_MODEL"] = args.model
|
||||
|
||||
# Imports happen after env setup so default_opensquilla_home() users resolve
|
||||
# to this isolated state root.
|
||||
from opensquilla.engine.agent import Agent
|
||||
from opensquilla.engine.types import AgentConfig
|
||||
from opensquilla.gateway.boot import _make_auto_propose_tool_context, build_services
|
||||
from opensquilla.gateway.config import GatewayConfig
|
||||
from opensquilla.scheduler.auto_propose_handler import make_auto_propose_handler
|
||||
from opensquilla.scheduler.types import CronJob, SessionTarget
|
||||
from opensquilla.skills.creator.auto_propose import auto_propose
|
||||
from opensquilla.skills.creator.proposer import (
|
||||
reset_runtime_e2e_context,
|
||||
reset_smoke_fixture_context,
|
||||
set_runtime_e2e_context,
|
||||
set_smoke_fixture_context,
|
||||
)
|
||||
from opensquilla.skills.creator.runtime_e2e import make_runtime_e2e_context
|
||||
from opensquilla.skills.meta.orchestrator import (
|
||||
MetaOrchestrator,
|
||||
make_agent_runner_from_parent,
|
||||
make_llm_chat_from_provider,
|
||||
make_tool_invoker_from_handler,
|
||||
)
|
||||
from opensquilla.tools.dispatch import build_tool_handler
|
||||
|
||||
text_tiers = {
|
||||
"c0": {"provider": args.provider, "model": args.model, "thinking_level": "off"},
|
||||
"c1": {"provider": args.provider, "model": args.model, "thinking_level": "low"},
|
||||
"c2": {"provider": args.provider, "model": args.model, "thinking_level": "medium"},
|
||||
"c3": {"provider": args.provider, "model": args.model, "thinking_level": "high"},
|
||||
}
|
||||
actual_cron = args.actual_scheduler and args.trigger == "cron"
|
||||
actual_dream = args.actual_scheduler and args.trigger == "dream"
|
||||
auto_enabled = actual_cron or not args.actual_scheduler
|
||||
config = GatewayConfig(
|
||||
workspace_dir=str(workspace_dir),
|
||||
llm={
|
||||
"provider": args.provider,
|
||||
"model": args.model,
|
||||
"api_key": os.environ.get("OPENROUTER_API_KEY", ""),
|
||||
"base_url": args.base_url,
|
||||
},
|
||||
squilla_router={
|
||||
"enabled": True,
|
||||
"tiers": text_tiers,
|
||||
"default_tier": "c3",
|
||||
},
|
||||
meta_skill={
|
||||
"auto_propose": {
|
||||
"enabled": auto_enabled,
|
||||
"cron": args.cron,
|
||||
"on_dream_complete": actual_dream or not args.actual_scheduler,
|
||||
"window_days": args.window_days,
|
||||
"min_freq": args.min_freq,
|
||||
"top_k": args.top_k,
|
||||
"auto_enable": args.auto_enable,
|
||||
"auto_enable_max_risk": args.auto_enable_max_risk,
|
||||
},
|
||||
},
|
||||
memory={
|
||||
"dream": {
|
||||
"enabled": actual_dream,
|
||||
"auto_schedule": actual_dream,
|
||||
"cron": args.cron if actual_dream else None,
|
||||
"preview_mode": True,
|
||||
"min_batch_size": 1,
|
||||
},
|
||||
},
|
||||
)
|
||||
server_handle = None
|
||||
if args.actual_scheduler:
|
||||
from opensquilla.gateway.boot import start_gateway_server
|
||||
|
||||
server_handle = await start_gateway_server(config=config, run=False)
|
||||
svc = getattr(server_handle, "_services", None)
|
||||
if svc is None:
|
||||
raise RuntimeError("gateway boot did not expose services")
|
||||
else:
|
||||
svc = await build_services(
|
||||
config=config,
|
||||
session_db_path=str(home / "state" / "sessions.sqlite"),
|
||||
seed_agent_workspaces=True,
|
||||
)
|
||||
assert svc.provider_selector is not None
|
||||
assert svc.tool_registry is not None
|
||||
assert svc.skill_loader is not None
|
||||
|
||||
def build_orchestrator(agent_id: str) -> MetaOrchestrator:
|
||||
provider_selector = svc.provider_selector
|
||||
clone_selector = getattr(provider_selector, "clone", None)
|
||||
if callable(clone_selector):
|
||||
provider_selector = clone_selector()
|
||||
override_model = getattr(provider_selector, "override_model", None)
|
||||
if callable(override_model):
|
||||
override_model(args.model)
|
||||
provider = provider_selector.resolve()
|
||||
ctx = _make_auto_propose_tool_context(
|
||||
agent_id=agent_id,
|
||||
workspace_dir=str(workspace_dir),
|
||||
)
|
||||
tool_handler = build_tool_handler(svc.tool_registry, ctx)
|
||||
base_config = AgentConfig(
|
||||
model_id=args.model,
|
||||
workspace_dir=str(workspace_dir),
|
||||
metadata={
|
||||
"routing_source": "meta_skill_auto_propose",
|
||||
"routing_applied": True,
|
||||
"routed_tier": "c3",
|
||||
"routed_model": args.model,
|
||||
"applied_model": args.model,
|
||||
"thinking_requested": True,
|
||||
"thinking_level": "high",
|
||||
},
|
||||
)
|
||||
tool_definitions = svc.tool_registry.to_tool_definitions(ctx)
|
||||
llm_chat = make_llm_chat_from_provider(
|
||||
provider=provider,
|
||||
base_config=base_config,
|
||||
usage_tracker=svc.usage_tracker,
|
||||
session_key=f"auto_propose:{agent_id}",
|
||||
)
|
||||
base_tool_invoker = make_tool_invoker_from_handler(tool_handler=tool_handler)
|
||||
runtime_e2e_ctx = make_runtime_e2e_context(
|
||||
provider=provider,
|
||||
base_config=base_config,
|
||||
skill_loader=svc.skill_loader,
|
||||
tool_definitions=tool_definitions,
|
||||
tool_handler=tool_handler,
|
||||
agent_factory=Agent,
|
||||
llm_chat=llm_chat,
|
||||
tool_invoker=base_tool_invoker,
|
||||
workspace_dir=str(workspace_dir),
|
||||
usage_tracker=svc.usage_tracker,
|
||||
session_key=f"auto_propose:{agent_id}",
|
||||
tool_registry=svc.tool_registry,
|
||||
tool_context=ctx,
|
||||
system_prompt=base_config.system_prompt or "",
|
||||
baseline_model=args.model,
|
||||
)
|
||||
|
||||
async def tool_invoker(tool_name: str, tool_args: dict[str, Any]) -> Any:
|
||||
if tool_name == "meta_skill_persist_proposal":
|
||||
tool_args = dict(tool_args)
|
||||
tool_args.setdefault("home", str(home))
|
||||
tool_args.setdefault("auto_enable_manual", False)
|
||||
token = set_runtime_e2e_context(runtime_e2e_ctx)
|
||||
smoke_token = set_smoke_fixture_context({"llm_chat": llm_chat})
|
||||
try:
|
||||
return await base_tool_invoker(tool_name, tool_args)
|
||||
finally:
|
||||
reset_smoke_fixture_context(smoke_token)
|
||||
reset_runtime_e2e_context(token)
|
||||
|
||||
return MetaOrchestrator(
|
||||
agent_runner=make_agent_runner_from_parent(
|
||||
provider=provider,
|
||||
base_config=base_config,
|
||||
tool_definitions=tool_definitions,
|
||||
tool_handler=tool_handler,
|
||||
agent_factory=Agent,
|
||||
workspace_dir=str(workspace_dir),
|
||||
usage_tracker=svc.usage_tracker,
|
||||
session_key=f"auto_propose:{agent_id}",
|
||||
),
|
||||
skill_loader=svc.skill_loader,
|
||||
llm_chat=llm_chat,
|
||||
tool_invoker=tool_invoker,
|
||||
workspace_dir=str(workspace_dir),
|
||||
run_writer=getattr(svc, "meta_run_writer", None),
|
||||
triggered_by=f"auto_{args.trigger}",
|
||||
session_key=f"auto_propose:{agent_id}",
|
||||
turn_id=None,
|
||||
usage_tracker=svc.usage_tracker,
|
||||
)
|
||||
|
||||
async def scheduler_snapshot() -> dict[str, Any]:
|
||||
scheduler = getattr(svc, "cron_scheduler", None)
|
||||
if scheduler is None:
|
||||
return {"jobs": []}
|
||||
jobs = await scheduler.list_jobs()
|
||||
rows = []
|
||||
for job in jobs:
|
||||
if not str(getattr(job, "name", "")).startswith(("auto_propose:", "memory_dream:")):
|
||||
continue
|
||||
runs = await scheduler.get_runs(getattr(job, "id"), limit=5)
|
||||
rows.append({
|
||||
"id": getattr(job, "id", ""),
|
||||
"name": getattr(job, "name", ""),
|
||||
"handler_key": getattr(job, "handler_key", ""),
|
||||
"schedule_kind": str(getattr(job, "schedule_kind", "")),
|
||||
"schedule_raw": getattr(job, "schedule_raw", ""),
|
||||
"status": str(getattr(job, "status", "")),
|
||||
"next_run_at": (
|
||||
getattr(job, "next_run_at", None).isoformat()
|
||||
if getattr(job, "next_run_at", None) is not None else None
|
||||
),
|
||||
"run_count": getattr(job, "run_count", 0),
|
||||
"runs": [
|
||||
{
|
||||
"success": getattr(run, "success", False),
|
||||
"summary": getattr(run, "summary", ""),
|
||||
"delivery_status": getattr(run, "delivery_status", ""),
|
||||
"started_at": (
|
||||
getattr(run, "started_at", None).isoformat()
|
||||
if getattr(run, "started_at", None) is not None else None
|
||||
),
|
||||
"finished_at": (
|
||||
getattr(run, "finished_at", None).isoformat()
|
||||
if getattr(run, "finished_at", None) is not None else None
|
||||
),
|
||||
}
|
||||
for run in runs
|
||||
],
|
||||
})
|
||||
return {"jobs": rows}
|
||||
|
||||
async def wait_for_automatic_execution() -> dict[str, Any]:
|
||||
deadline = asyncio.get_running_loop().time() + args.wait_seconds
|
||||
last: dict[str, Any] = {}
|
||||
while True:
|
||||
proposals_now = [
|
||||
sub.name
|
||||
for sub in sorted(proposals_dir.iterdir())
|
||||
if sub.is_dir()
|
||||
] if proposals_dir.is_dir() else []
|
||||
snapshot = await scheduler_snapshot()
|
||||
last = {
|
||||
"triggered_by": args.trigger,
|
||||
"actual_scheduler": True,
|
||||
"proposal_ids": proposals_now,
|
||||
"scheduler": snapshot,
|
||||
}
|
||||
target_prefix = "memory_dream:" if args.trigger == "dream" else "auto_propose:"
|
||||
target_jobs = [
|
||||
job for job in snapshot.get("jobs", [])
|
||||
if str(job.get("name", "")).startswith(target_prefix)
|
||||
]
|
||||
target_finished = any(
|
||||
int(job.get("run_count") or 0) > 0
|
||||
or bool(job.get("runs"))
|
||||
for job in target_jobs
|
||||
)
|
||||
if target_jobs and target_finished and (
|
||||
not args.wait_for_proposal or proposals_now
|
||||
):
|
||||
return last
|
||||
if asyncio.get_running_loop().time() >= deadline:
|
||||
return last
|
||||
await asyncio.sleep(args.poll_seconds)
|
||||
|
||||
if args.actual_scheduler:
|
||||
result = await wait_for_automatic_execution()
|
||||
elif args.via_handler:
|
||||
handler = make_auto_propose_handler(
|
||||
build_orchestrator=build_orchestrator,
|
||||
skill_loader=svc.skill_loader,
|
||||
log_dir=log_dir,
|
||||
proposals_dir=proposals_dir,
|
||||
config=config.meta_skill.auto_propose,
|
||||
enabled_predicate=lambda: True,
|
||||
)
|
||||
job = CronJob(
|
||||
id=f"live-auto-propose-{args.trigger}",
|
||||
name="auto_propose:main",
|
||||
cron_expr="* * * * *",
|
||||
schedule_raw="* * * * *",
|
||||
handler_key="auto_propose",
|
||||
payload={"agent_id": "main"},
|
||||
session_target=SessionTarget.ISOLATED,
|
||||
)
|
||||
handler_result = await handler(job)
|
||||
result = {
|
||||
"handler": {
|
||||
"summary": handler_result.summary,
|
||||
"delivery_status": handler_result.delivery_status,
|
||||
},
|
||||
"triggered_by": args.trigger,
|
||||
}
|
||||
else:
|
||||
result_obj = await auto_propose(
|
||||
orchestrator=build_orchestrator("main"),
|
||||
skill_loader=svc.skill_loader,
|
||||
log_dir=log_dir,
|
||||
window_days=args.window_days,
|
||||
min_freq=args.min_freq,
|
||||
top_k=args.top_k,
|
||||
triggered_by=args.trigger,
|
||||
proposals_dir=proposals_dir,
|
||||
auto_enable=args.auto_enable,
|
||||
auto_enable_max_risk=args.auto_enable_max_risk,
|
||||
)
|
||||
result = {
|
||||
"summary": result_obj.summary(),
|
||||
"proposals_created": result_obj.proposals_created,
|
||||
"proposals_enabled": result_obj.proposals_enabled,
|
||||
"auto_enable": result_obj.auto_enable,
|
||||
"skipped": result_obj.skipped,
|
||||
"errors": result_obj.errors,
|
||||
"triggered_by": result_obj.triggered_by,
|
||||
}
|
||||
|
||||
proposals = []
|
||||
if proposals_dir.is_dir():
|
||||
for sub in sorted(proposals_dir.iterdir()):
|
||||
gates_path = sub / "gates.json"
|
||||
gates = (
|
||||
json.loads(gates_path.read_text(encoding="utf-8"))
|
||||
if gates_path.is_file()
|
||||
else {}
|
||||
)
|
||||
proposals.append({
|
||||
"id": sub.name,
|
||||
"skill": (sub / "SKILL.md").read_text(encoding="utf-8")[:400]
|
||||
if (sub / "SKILL.md").is_file() else "",
|
||||
"gates": gates,
|
||||
})
|
||||
|
||||
try:
|
||||
return {
|
||||
"ok": True,
|
||||
"provider": args.provider,
|
||||
"model": args.model,
|
||||
"home": str(home),
|
||||
"log_dir": str(log_dir),
|
||||
"history_path": str(history_path) if history_path else "",
|
||||
"proposals_dir": str(proposals_dir),
|
||||
"result": result,
|
||||
"proposal_count": len(proposals),
|
||||
"proposals": proposals,
|
||||
}
|
||||
finally:
|
||||
if server_handle is not None:
|
||||
await server_handle.close(reason="meta_skill_creator_auto_propose_e2e")
|
||||
else:
|
||||
close = getattr(svc, "close", None)
|
||||
if callable(close):
|
||||
await close()
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--env-file", type=Path)
|
||||
parser.add_argument("--home", type=Path, required=True)
|
||||
parser.add_argument("--log-dir", type=Path)
|
||||
parser.add_argument("--proposals-dir", type=Path)
|
||||
parser.add_argument("--workspace", type=Path)
|
||||
parser.add_argument("--provider", default="openrouter")
|
||||
parser.add_argument("--model", default=os.environ.get("OPENROUTER_MODEL", "openai/gpt-4o-mini"))
|
||||
parser.add_argument("--base-url", default="https://openrouter.ai/api/v1")
|
||||
parser.add_argument("--trigger", choices=["cron", "dream"], default="cron")
|
||||
parser.add_argument("--cron", default="* * * * *")
|
||||
parser.add_argument("--via-handler", action="store_true")
|
||||
parser.add_argument("--actual-scheduler", action="store_true")
|
||||
parser.add_argument(
|
||||
"--wait-for-proposal",
|
||||
action="store_true",
|
||||
help=(
|
||||
"For --actual-scheduler, wait until at least one proposal directory "
|
||||
"exists instead of returning as soon as a scheduled run starts."
|
||||
),
|
||||
)
|
||||
parser.add_argument("--wait-seconds", type=float, default=120.0)
|
||||
parser.add_argument("--poll-seconds", type=float, default=2.0)
|
||||
parser.add_argument("--seed-history", action="store_true")
|
||||
parser.add_argument("--window-days", type=int, default=30)
|
||||
parser.add_argument("--min-freq", type=int, default=3)
|
||||
parser.add_argument("--top-k", type=int, default=2)
|
||||
parser.add_argument("--auto-enable", action="store_true")
|
||||
parser.add_argument("--auto-enable-max-risk", default="low")
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = _parser().parse_args(argv)
|
||||
_load_env_file(args.env_file)
|
||||
result = asyncio.run(_run(args))
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Live meta-skill creator E2E harness.
|
||||
|
||||
This intentionally prints only structural evidence. It never prints provider
|
||||
API keys loaded from ``--env-file`` or the process environment.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from opensquilla.skills import proposals_lib
|
||||
from opensquilla.skills.creator import proposer
|
||||
|
||||
DEFAULT_HISTORY = {
|
||||
"co_occurrences": [
|
||||
{"skills": ["history-explorer", "summarize"], "freq": 8},
|
||||
],
|
||||
"note": "Prefer a two-step read-and-summarize workflow using low-risk skills.",
|
||||
}
|
||||
DEFAULT_INTENT = (
|
||||
"Create a meta-skill that first uses history-explorer to inspect recent "
|
||||
"OpenSquilla decision history for a query, then uses summarize to produce "
|
||||
"a concise operational summary. Use only history-explorer and summarize."
|
||||
)
|
||||
|
||||
|
||||
def _load_env_file(path: Path | None) -> None:
|
||||
if path is None or not path.is_file():
|
||||
return
|
||||
for raw in path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
key = key.strip()
|
||||
value = value.strip().strip("'\"")
|
||||
if key and key not in os.environ:
|
||||
os.environ[key] = value
|
||||
|
||||
|
||||
def run_live_meta_skill_creator_e2e(
|
||||
*,
|
||||
home: Path | None = None,
|
||||
pattern_id: str = "p1_sequential",
|
||||
history_summary: str | None = None,
|
||||
user_intent: str = DEFAULT_INTENT,
|
||||
provider: str | None = None,
|
||||
model: str | None = None,
|
||||
auto_enable: bool = True,
|
||||
auto_enable_max_risk: str = "low",
|
||||
) -> dict[str, Any]:
|
||||
"""Run fill_slots -> assemble -> lint -> smoke -> persist/auto-enable."""
|
||||
previous_provider = os.environ.get("OPENSQUILLA_LLM_PROVIDER")
|
||||
previous_model = os.environ.get("OPENSQUILLA_LLM_MODEL")
|
||||
if provider:
|
||||
os.environ["OPENSQUILLA_LLM_PROVIDER"] = provider
|
||||
if model:
|
||||
os.environ["OPENSQUILLA_LLM_MODEL"] = model
|
||||
|
||||
try:
|
||||
home_path = home or Path(tempfile.mkdtemp(prefix="opensquilla-live-meta-skill-"))
|
||||
home_path.mkdir(parents=True, exist_ok=True)
|
||||
proposals_lib.write_auto_propose_settings(
|
||||
home_path,
|
||||
{
|
||||
"auto_enable": auto_enable,
|
||||
"auto_enable_max_risk": auto_enable_max_risk,
|
||||
},
|
||||
)
|
||||
|
||||
history = history_summary or json.dumps(DEFAULT_HISTORY, ensure_ascii=False)
|
||||
slots_json = proposer.meta_skill_fill_slots(pattern_id, history, user_intent)
|
||||
slots = json.loads(slots_json)
|
||||
skill_md = proposer.meta_skill_assemble(pattern_id, slots_json)
|
||||
lint_result = json.loads(proposer.meta_skill_lint_run(skill_md, "G1,G2"))
|
||||
smoke_result = proposer.run_smoke_gates(
|
||||
skill_md=skill_md,
|
||||
fixture_gen_fn=lambda _md, kind: {
|
||||
"positive": f"please use {slots['triggers'][0]} for recent decisions",
|
||||
"negative": "what is the weather tomorrow in Tokyo?",
|
||||
}[kind],
|
||||
classifier_model=model or "live-meta-skill-creator-e2e",
|
||||
)
|
||||
persist = json.loads(proposer.meta_skill_persist_proposal(
|
||||
skill_md,
|
||||
json.dumps(lint_result),
|
||||
json.dumps(smoke_result),
|
||||
home=str(home_path),
|
||||
))
|
||||
managed = (
|
||||
sorted(p.name for p in (home_path / "skills").iterdir())
|
||||
if (home_path / "skills").is_dir()
|
||||
else []
|
||||
)
|
||||
pending = (
|
||||
sorted(p.name for p in (home_path / "proposals").iterdir())
|
||||
if (home_path / "proposals").is_dir()
|
||||
else []
|
||||
)
|
||||
return {
|
||||
"ok": True,
|
||||
"home": str(home_path),
|
||||
"llm_slots": {
|
||||
"name": slots.get("name"),
|
||||
"triggers": slots.get("triggers"),
|
||||
"steps": [
|
||||
{"id": s.get("id"), "skill": s.get("skill")}
|
||||
for s in slots.get("steps", [])
|
||||
],
|
||||
},
|
||||
"lint": lint_result,
|
||||
"smoke": smoke_result,
|
||||
"persist": persist,
|
||||
"managed": managed,
|
||||
"pending": pending,
|
||||
}
|
||||
finally:
|
||||
if previous_provider is None:
|
||||
os.environ.pop("OPENSQUILLA_LLM_PROVIDER", None)
|
||||
else:
|
||||
os.environ["OPENSQUILLA_LLM_PROVIDER"] = previous_provider
|
||||
if previous_model is None:
|
||||
os.environ.pop("OPENSQUILLA_LLM_MODEL", None)
|
||||
else:
|
||||
os.environ["OPENSQUILLA_LLM_MODEL"] = previous_model
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument("--env-file", type=Path, default=None)
|
||||
p.add_argument("--home", type=Path, default=None)
|
||||
p.add_argument("--provider", default=None)
|
||||
p.add_argument("--model", default=None)
|
||||
p.add_argument("--pattern-id", default="p1_sequential")
|
||||
p.add_argument("--history-summary", default=None)
|
||||
p.add_argument("--user-intent", default=DEFAULT_INTENT)
|
||||
p.add_argument("--no-auto-enable", action="store_true")
|
||||
p.add_argument("--auto-enable-max-risk", default="low")
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = _parser().parse_args(argv)
|
||||
_load_env_file(args.env_file)
|
||||
result = run_live_meta_skill_creator_e2e(
|
||||
home=args.home,
|
||||
pattern_id=args.pattern_id,
|
||||
history_summary=args.history_summary,
|
||||
user_intent=args.user_intent,
|
||||
provider=args.provider,
|
||||
model=args.model,
|
||||
auto_enable=not args.no_auto_enable,
|
||||
auto_enable_max_risk=args.auto_enable_max_risk,
|
||||
)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,344 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Live meta-skill soft-activation E2E harness.
|
||||
|
||||
The harness verifies the path where the model sees a ``kind: meta`` skill,
|
||||
chooses ``meta_invoke(name=...)``, and the runtime executes that meta-skill.
|
||||
It prints only structural evidence and never prints provider API keys.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from opensquilla.engine.agent import Agent
|
||||
from opensquilla.engine.types import (
|
||||
AgentConfig,
|
||||
DoneEvent,
|
||||
ErrorEvent,
|
||||
TextDeltaEvent,
|
||||
ToolResultEvent,
|
||||
)
|
||||
from opensquilla.provider.selector import build_provider
|
||||
from opensquilla.skills.injector import SkillInjector
|
||||
from opensquilla.skills.loader import SkillLoader
|
||||
from opensquilla.tools.builtin import meta_tools # noqa: F401 - registers meta_invoke
|
||||
from opensquilla.tools.registry import get_default_registry
|
||||
from opensquilla.tools.types import ToolContext
|
||||
|
||||
META_SKILL_NAME = "meta-live-soft-activation"
|
||||
EXPECTED_OUTPUT = "LIVE_OK"
|
||||
DEFAULT_USER_MESSAGE = (
|
||||
"Run the available meta-skill named meta-live-soft-activation and return "
|
||||
"its result."
|
||||
)
|
||||
|
||||
|
||||
def _load_env_file(path: Path | None) -> None:
|
||||
if path is None or not path.is_file():
|
||||
return
|
||||
for raw in path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
key = key.strip()
|
||||
value = value.strip().strip("'\"")
|
||||
if key and key not in os.environ:
|
||||
os.environ[key] = value
|
||||
|
||||
|
||||
def _resolve_api_key(provider: str) -> str:
|
||||
env_map = {
|
||||
"openrouter": "OPENROUTER_API_KEY",
|
||||
"anthropic": "ANTHROPIC_API_KEY",
|
||||
"openai": "OPENAI_API_KEY",
|
||||
"deepseek": "DEEPSEEK_API_KEY",
|
||||
"gemini": "GEMINI_API_KEY",
|
||||
"dashscope": "DASHSCOPE_API_KEY",
|
||||
"minimax": "MINIMAX_API_KEY",
|
||||
}
|
||||
env_name = env_map.get(provider.lower(), "")
|
||||
return os.environ.get(env_name, "").strip() if env_name else ""
|
||||
|
||||
|
||||
def _write_live_meta_skill(home: Path) -> SkillLoader:
|
||||
bundled = home / "skills" / "bundled"
|
||||
skill_dir = bundled / META_SKILL_NAME
|
||||
skill_dir.mkdir(parents=True, exist_ok=True)
|
||||
(skill_dir / "SKILL.md").write_text(
|
||||
f"""---
|
||||
name: {META_SKILL_NAME}
|
||||
kind: meta
|
||||
description: Live E2E meta-skill that returns {EXPECTED_OUTPUT} when invoked.
|
||||
triggers:
|
||||
- live soft activation workflow
|
||||
metadata:
|
||||
opensquilla:
|
||||
risk: low
|
||||
capabilities: []
|
||||
composition:
|
||||
steps:
|
||||
- id: classify
|
||||
kind: llm_classify
|
||||
output_choices: [{EXPECTED_OUTPUT}, OTHER]
|
||||
with:
|
||||
text: "Return {EXPECTED_OUTPUT} for this live soft activation E2E check."
|
||||
final_text_mode: "step:classify"
|
||||
---
|
||||
|
||||
# {META_SKILL_NAME}
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
loader = SkillLoader(bundled_dir=bundled, snapshot_path=home / "skills_snapshot.json")
|
||||
loader.invalidate_cache()
|
||||
loader.load_all()
|
||||
return loader
|
||||
|
||||
|
||||
def _make_agent(
|
||||
*,
|
||||
home: Path,
|
||||
provider_instance: Any,
|
||||
model: str,
|
||||
classify_override: str | None,
|
||||
) -> Agent:
|
||||
loader = _write_live_meta_skill(home)
|
||||
skills = loader.load_all()
|
||||
system_prompt = SkillInjector().inject_full(
|
||||
"You are validating OpenSquilla meta-skill soft activation.",
|
||||
skills,
|
||||
)
|
||||
registry = get_default_registry()
|
||||
ctx = ToolContext(
|
||||
workspace_dir=str(home),
|
||||
is_owner=True,
|
||||
allowed_tools={"meta_invoke"},
|
||||
surfaced_tools={"meta_invoke"},
|
||||
)
|
||||
tools = registry.to_tool_definitions(ctx)
|
||||
config = AgentConfig(
|
||||
model_id=model,
|
||||
max_iterations=4,
|
||||
system_prompt=system_prompt,
|
||||
metadata={"skill_loader": loader, "bootstrap_workspace_dir": str(home)},
|
||||
)
|
||||
agent = Agent(
|
||||
provider=provider_instance,
|
||||
config=config,
|
||||
tool_definitions=tools,
|
||||
tool_handler=None,
|
||||
tool_registry=registry,
|
||||
tool_context=ctx,
|
||||
)
|
||||
if classify_override is not None:
|
||||
async def _override(_system: str, _user: str) -> str:
|
||||
return classify_override
|
||||
|
||||
agent._test_llm_chat_override = _override # type: ignore[attr-defined]
|
||||
return agent
|
||||
|
||||
|
||||
async def _run_one_case(
|
||||
*,
|
||||
home: Path,
|
||||
provider_instance: Any,
|
||||
model: str,
|
||||
user_message: str,
|
||||
expected_meta_skill: str | None,
|
||||
classify_override: str | None,
|
||||
) -> dict[str, Any]:
|
||||
agent = _make_agent(
|
||||
home=home,
|
||||
provider_instance=provider_instance,
|
||||
model=model,
|
||||
classify_override=classify_override,
|
||||
)
|
||||
events = []
|
||||
async for event in agent.run_turn(user_message):
|
||||
events.append(event)
|
||||
|
||||
tool_results = [
|
||||
event for event in events
|
||||
if isinstance(event, ToolResultEvent)
|
||||
]
|
||||
final_text = "".join(
|
||||
event.text for event in events
|
||||
if isinstance(event, TextDeltaEvent)
|
||||
)
|
||||
meta_results = [
|
||||
event for event in tool_results
|
||||
if event.tool_name == "meta_invoke"
|
||||
]
|
||||
errors = [
|
||||
event.message for event in events
|
||||
if isinstance(event, ErrorEvent)
|
||||
]
|
||||
done = next((event for event in events if isinstance(event, DoneEvent)), None)
|
||||
selected = None
|
||||
if meta_results:
|
||||
args = meta_results[-1].arguments or {}
|
||||
selected = args.get("name") if isinstance(args.get("name"), str) else None
|
||||
if selected is None:
|
||||
selected = expected_meta_skill
|
||||
meta_invoke_result = meta_results[-1].result if meta_results else ""
|
||||
passed = (
|
||||
selected == expected_meta_skill
|
||||
if expected_meta_skill is not None
|
||||
else not meta_results
|
||||
)
|
||||
if expected_meta_skill is not None:
|
||||
passed = passed and (
|
||||
EXPECTED_OUTPUT in meta_invoke_result
|
||||
or EXPECTED_OUTPUT in final_text
|
||||
or bool(done and EXPECTED_OUTPUT in (done.text or ""))
|
||||
)
|
||||
|
||||
return {
|
||||
"user_message": user_message,
|
||||
"expected_meta_skill": expected_meta_skill,
|
||||
"passed": passed,
|
||||
"model_decision": {
|
||||
"meta_invoke_called": bool(meta_results),
|
||||
"selected_meta_skill": selected,
|
||||
},
|
||||
"observed_tool_results": [event.tool_name for event in tool_results],
|
||||
"meta_invoke_result": meta_invoke_result,
|
||||
"final_text": final_text or (done.text if done else ""),
|
||||
"done": done is not None,
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
|
||||
def run_live_meta_activation_cases(
|
||||
*,
|
||||
home: Path | None = None,
|
||||
provider_instance: Any | None = None,
|
||||
provider: str = "openrouter",
|
||||
model: str = "anthropic/claude-3.5-haiku",
|
||||
cases: list[dict[str, Any]] | None = None,
|
||||
classify_override: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
home_path = home or Path(tempfile.mkdtemp(prefix="opensquilla-live-meta-soft-"))
|
||||
home_path.mkdir(parents=True, exist_ok=True)
|
||||
llm = provider_instance or build_provider(
|
||||
provider=provider,
|
||||
model=model,
|
||||
api_key=_resolve_api_key(provider),
|
||||
)
|
||||
case_rows = cases or [
|
||||
{
|
||||
"name": "positive",
|
||||
"user_message": DEFAULT_USER_MESSAGE,
|
||||
"expected_meta_skill": META_SKILL_NAME,
|
||||
}
|
||||
]
|
||||
|
||||
async def _drive() -> list[dict[str, Any]]:
|
||||
results: list[dict[str, Any]] = []
|
||||
for row in case_rows:
|
||||
result = await _run_one_case(
|
||||
home=home_path,
|
||||
provider_instance=llm,
|
||||
model=model,
|
||||
user_message=str(row["user_message"]),
|
||||
expected_meta_skill=row.get("expected_meta_skill"),
|
||||
classify_override=classify_override,
|
||||
)
|
||||
result["name"] = row.get("name", "")
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
results = asyncio.run(_drive())
|
||||
passed = sum(1 for row in results if row["passed"])
|
||||
failed = len(results) - passed
|
||||
return {
|
||||
"ok": failed == 0,
|
||||
"home": str(home_path),
|
||||
"provider": provider_instance.provider_name
|
||||
if provider_instance is not None and hasattr(provider_instance, "provider_name")
|
||||
else provider,
|
||||
"model": model,
|
||||
"meta_skill": META_SKILL_NAME,
|
||||
"expected_output": EXPECTED_OUTPUT,
|
||||
"summary": {"passed": passed, "failed": failed, "total": len(results)},
|
||||
"cases": results,
|
||||
}
|
||||
|
||||
|
||||
def run_live_meta_soft_activation_e2e(
|
||||
*,
|
||||
home: Path | None = None,
|
||||
provider_instance: Any | None = None,
|
||||
provider: str = "openrouter",
|
||||
model: str = "anthropic/claude-3.5-haiku",
|
||||
user_message: str = DEFAULT_USER_MESSAGE,
|
||||
classify_override: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
result = run_live_meta_activation_cases(
|
||||
home=home,
|
||||
provider_instance=provider_instance,
|
||||
provider=provider,
|
||||
model=model,
|
||||
cases=[
|
||||
{
|
||||
"name": "positive",
|
||||
"user_message": user_message,
|
||||
"expected_meta_skill": META_SKILL_NAME,
|
||||
}
|
||||
],
|
||||
classify_override=classify_override,
|
||||
)
|
||||
case = result["cases"][0]
|
||||
return {
|
||||
**result,
|
||||
"model_decision": case["model_decision"],
|
||||
"observed_tool_results": case["observed_tool_results"],
|
||||
"meta_invoke_result": case["meta_invoke_result"],
|
||||
"final_text": case.get("final_text", ""),
|
||||
}
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--env-file", type=Path, default=None)
|
||||
parser.add_argument("--home", type=Path, default=None)
|
||||
parser.add_argument("--provider", default="openrouter")
|
||||
parser.add_argument("--model", default="anthropic/claude-3.5-haiku")
|
||||
parser.add_argument("--user-message", default=DEFAULT_USER_MESSAGE)
|
||||
parser.add_argument("--case-file", type=Path, default=None)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = _parser().parse_args(argv)
|
||||
_load_env_file(args.env_file)
|
||||
cases = None
|
||||
if args.case_file is not None:
|
||||
cases = json.loads(args.case_file.read_text(encoding="utf-8"))
|
||||
if cases is None:
|
||||
result = run_live_meta_soft_activation_e2e(
|
||||
home=args.home,
|
||||
provider=args.provider,
|
||||
model=args.model,
|
||||
user_message=args.user_message,
|
||||
)
|
||||
else:
|
||||
result = run_live_meta_activation_cases(
|
||||
home=args.home,
|
||||
provider=args.provider,
|
||||
model=args.model,
|
||||
cases=cases,
|
||||
)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0 if result.get("ok") else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Opt-in OpenRouter explicit prompt-cache smoke for one model.
|
||||
|
||||
The smoke sends the same large system prompt twice and reports whether the
|
||||
second response exposes non-zero cached prompt tokens. It is intentionally not
|
||||
part of default CI; run it only with live OpenRouter credentials.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
DEFAULT_MODEL = "z-ai/glm-5.1"
|
||||
DEFAULT_BASE_URL = "https://openrouter.ai/api/v1"
|
||||
|
||||
|
||||
def _cached_prompt_tokens(payload: dict[str, Any]) -> int:
|
||||
usage = payload.get("usage")
|
||||
if not isinstance(usage, dict):
|
||||
return 0
|
||||
|
||||
prompt_details = usage.get("prompt_tokens_details")
|
||||
if isinstance(prompt_details, dict):
|
||||
value = prompt_details.get("cached_tokens")
|
||||
if isinstance(value, int):
|
||||
return max(0, value)
|
||||
|
||||
for key in (
|
||||
"cached_tokens",
|
||||
"prompt_cache_hit_tokens",
|
||||
"cache_read_input_tokens",
|
||||
"cached_input_tokens",
|
||||
):
|
||||
value = usage.get(key)
|
||||
if isinstance(value, int):
|
||||
return max(0, value)
|
||||
return 0
|
||||
|
||||
|
||||
def _cache_request_payload(model: str, system_text: str) -> dict[str, Any]:
|
||||
return {
|
||||
"model": model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": system_text,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "Reply with exactly: cache-smoke-ok"},
|
||||
],
|
||||
"max_tokens": 16,
|
||||
"temperature": 0,
|
||||
}
|
||||
|
||||
|
||||
def _large_system_prompt() -> str:
|
||||
stable_line = (
|
||||
"OpenSquilla explicit cache smoke stable prefix. "
|
||||
"This text is synthetic public test material. "
|
||||
)
|
||||
return stable_line * 260
|
||||
|
||||
|
||||
def _post_once(
|
||||
client: httpx.Client,
|
||||
*,
|
||||
url: str,
|
||||
api_key: str,
|
||||
model: str,
|
||||
system_text: str,
|
||||
) -> dict[str, Any]:
|
||||
response = client.post(
|
||||
url,
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
"HTTP-Referer": "https://opensquilla.ai",
|
||||
"X-Title": "OpenSquilla cache smoke",
|
||||
},
|
||||
json=_cache_request_payload(model, system_text),
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
if not isinstance(data, dict):
|
||||
raise RuntimeError("OpenRouter returned a non-object JSON payload")
|
||||
return data
|
||||
|
||||
|
||||
def run_smoke(*, api_key: str, model: str, base_url: str, timeout: float) -> dict[str, Any]:
|
||||
system_text = _large_system_prompt()
|
||||
url = base_url.rstrip("/") + "/chat/completions"
|
||||
with httpx.Client(timeout=timeout, trust_env=True) as client:
|
||||
first = _post_once(client, url=url, api_key=api_key, model=model, system_text=system_text)
|
||||
second = _post_once(client, url=url, api_key=api_key, model=model, system_text=system_text)
|
||||
|
||||
first_cached = _cached_prompt_tokens(first)
|
||||
second_cached = _cached_prompt_tokens(second)
|
||||
return {
|
||||
"model": model,
|
||||
"base_url": base_url,
|
||||
"explicit_cache_supported": second_cached > 0,
|
||||
"first_cached_tokens": first_cached,
|
||||
"second_cached_tokens": second_cached,
|
||||
"usage_fields_present": {
|
||||
"first": sorted((first.get("usage") or {}).keys())
|
||||
if isinstance(first.get("usage"), dict)
|
||||
else [],
|
||||
"second": sorted((second.get("usage") or {}).keys())
|
||||
if isinstance(second.get("usage"), dict)
|
||||
else [],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--model", default=os.environ.get("OPENROUTER_CACHE_SMOKE_MODEL", DEFAULT_MODEL)
|
||||
)
|
||||
parser.add_argument(
|
||||
"--base-url", default=os.environ.get("OPENROUTER_BASE_URL", DEFAULT_BASE_URL)
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timeout",
|
||||
type=float,
|
||||
default=float(os.environ.get("OPENROUTER_CACHE_SMOKE_TIMEOUT", "90")),
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
api_key = os.environ.get("OPENROUTER_API_KEY", "").strip()
|
||||
if not api_key:
|
||||
print(
|
||||
json.dumps({"ok": False, "error": "OPENROUTER_API_KEY is required"}, ensure_ascii=False)
|
||||
)
|
||||
return 2
|
||||
|
||||
try:
|
||||
result = run_smoke(
|
||||
api_key=api_key,
|
||||
model=args.model,
|
||||
base_url=args.base_url,
|
||||
timeout=args.timeout,
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - live diagnostic path
|
||||
print(json.dumps({"ok": False, "error": str(exc), "model": args.model}, ensure_ascii=False))
|
||||
return 1
|
||||
|
||||
print(json.dumps({"ok": True, **result}, ensure_ascii=False, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,752 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run live gateway E2E checks for direct provider tier profiles.
|
||||
|
||||
The check starts a temporary OpenSquilla gateway per provider, enables the
|
||||
matching ``squilla_router.tier_profile``, sends one turn for each text tier,
|
||||
and records routed model, response usage, and local cost estimates. Secrets are
|
||||
kept in environment variables and are not written to the output artifact.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
SRC_DIR = REPO_ROOT / "src"
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
if str(SRC_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SRC_DIR))
|
||||
|
||||
from opensquilla.engine.pricing import lookup_price # noqa: E402
|
||||
from opensquilla.gateway.config import GatewayConfig # noqa: E402
|
||||
from opensquilla.provider.registry import get_provider_spec # noqa: E402
|
||||
from scripts.smoke_v4_phase3_router import ( # noqa: E402
|
||||
_free_port,
|
||||
_post_json,
|
||||
_read_turn_call_records,
|
||||
_stop_gateway,
|
||||
_usage_from_llm_responses,
|
||||
_wait_for_assistant_reply,
|
||||
_wait_for_gateway_health,
|
||||
)
|
||||
|
||||
DEFAULT_PROVIDERS = [
|
||||
"openrouter",
|
||||
"dashscope",
|
||||
"deepseek",
|
||||
"gemini",
|
||||
"volcengine",
|
||||
"byteplus",
|
||||
"openai",
|
||||
"zhipu",
|
||||
"moonshot",
|
||||
]
|
||||
BASE_ENV = {
|
||||
"openrouter": "OPENROUTER_BASE_URL",
|
||||
"openai": "OPENAI_BASE_URL",
|
||||
"dashscope": "DASHSCOPE_BASE_URL",
|
||||
"deepseek": "DEEPSEEK_BASE_URL",
|
||||
"gemini": "GEMINI_BASE_URL",
|
||||
"volcengine": "VOLCENGINE_BASE_URL",
|
||||
"byteplus": "BYTEPLUS_BASE_URL",
|
||||
"moonshot": "MOONSHOT_BASE_URL",
|
||||
"zhipu": "ZAI_BASE_URL",
|
||||
}
|
||||
TEXT_PROFILE_SLOTS = ("c0", "c1", "c2", "c3")
|
||||
LIVE_AGENT_MAX_ITERATIONS = 6
|
||||
LIVE_AGENT_RUNTIME_TIMEOUT_SECONDS = 75.0
|
||||
LIVE_TURN_HARD_DEADLINE_SECONDS = 90.0
|
||||
|
||||
TIER_CASES = [
|
||||
{
|
||||
"tier": "c0",
|
||||
"id": "r0_short_ack",
|
||||
"message": "谢谢。不要调用工具，请只回复一个短句，包含 {marker}。",
|
||||
},
|
||||
{
|
||||
"tier": "c1",
|
||||
"id": "r1_structured_compare",
|
||||
"message": (
|
||||
"不要调用工具，只输出 Markdown 表格和 marker。用不超过 4 行的表格比较 "
|
||||
"PostgreSQL 和 MySQL 在事务、索引、复制方面的差异，每格不超过 12 个字。"
|
||||
"最后一行单独写 {marker}。"
|
||||
),
|
||||
},
|
||||
{
|
||||
"tier": "c2",
|
||||
"id": "r2_debugging",
|
||||
"message": (
|
||||
"下面是异步服务偶发超时的日志片段：连接池耗尽、慢查询、重试风暴、队列积压。"
|
||||
"不要调用工具，请用不超过三条短句定位可能原因并给出排查动作。"
|
||||
"最后一行单独写 {marker}。"
|
||||
),
|
||||
},
|
||||
{
|
||||
"tier": "c3",
|
||||
"id": "r3_architecture",
|
||||
"message": (
|
||||
"请设计跨机房分布式任务调度系统，解释一致性、故障恢复和容量评估。"
|
||||
"不要调用工具，回答不超过五句，并包含 {marker}。"
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _toml_value(value: Any) -> str:
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
if isinstance(value, int | float):
|
||||
return str(value)
|
||||
return json.dumps(str(value), ensure_ascii=False)
|
||||
|
||||
|
||||
def _marker_component(value: str) -> str:
|
||||
raw = "".join(ch if ch.isalnum() else "_" for ch in value.upper())
|
||||
return "_".join(part for part in raw.split("_") if part)
|
||||
|
||||
|
||||
def _case_marker(provider: str, slot: str, case_id: str) -> str:
|
||||
return (
|
||||
f"E2E_{_marker_component(provider)}_"
|
||||
f"{_marker_component(slot)}_{_marker_component(case_id)}"
|
||||
)
|
||||
|
||||
|
||||
def _load_env_quietly(path: Path = REPO_ROOT / ".env") -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
for raw_line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
key = key.strip()
|
||||
value = value.strip().strip('"').strip("'")
|
||||
if key and key not in os.environ:
|
||||
os.environ[key] = value
|
||||
|
||||
|
||||
def _profile_tiers(provider: str) -> dict[str, dict[str, Any]]:
|
||||
cfg = GatewayConfig.model_validate(
|
||||
{
|
||||
"llm": {"provider": provider},
|
||||
"squilla_router": {"tier_profile": provider},
|
||||
}
|
||||
)
|
||||
return {
|
||||
name: dict(tier)
|
||||
for name, tier in cfg.squilla_router.tiers.items()
|
||||
if isinstance(tier, dict) and not tier.get("image_only")
|
||||
}
|
||||
|
||||
|
||||
def _profile_slot_targets(tiers: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||
return {
|
||||
slot: dict(tiers[slot])
|
||||
for slot in TEXT_PROFILE_SLOTS
|
||||
if isinstance(tiers.get(slot), dict) and not tiers[slot].get("image_only")
|
||||
}
|
||||
|
||||
|
||||
def _covered_profile_slots(rows: list[dict[str, Any]]) -> list[str]:
|
||||
covered: list[str] = []
|
||||
for row in rows:
|
||||
slot = str(row.get("actual_slot_covered") or "")
|
||||
if row.get("ok") is True and slot and slot not in covered:
|
||||
covered.append(slot)
|
||||
return covered
|
||||
|
||||
|
||||
def _missing_profile_slots(
|
||||
tiers: dict[str, dict[str, Any]],
|
||||
rows: list[dict[str, Any]],
|
||||
) -> list[str]:
|
||||
covered = set(_covered_profile_slots(rows))
|
||||
return [slot for slot in _profile_slot_targets(tiers) if slot not in covered]
|
||||
|
||||
|
||||
def _forced_tier_overrides_for_slot(
|
||||
tiers: dict[str, dict[str, Any]],
|
||||
slot: str,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
target = dict(tiers[slot])
|
||||
overrides: dict[str, dict[str, Any]] = {}
|
||||
for text_slot in TEXT_PROFILE_SLOTS:
|
||||
if text_slot == slot:
|
||||
forced = dict(target)
|
||||
forced["image_only"] = False
|
||||
overrides[text_slot] = forced
|
||||
else:
|
||||
hidden = dict(tiers.get(text_slot, target))
|
||||
hidden["image_only"] = True
|
||||
overrides[text_slot] = hidden
|
||||
return overrides
|
||||
|
||||
|
||||
def _render_tier_overrides(tiers: dict[str, dict[str, Any]] | None) -> str:
|
||||
if not tiers:
|
||||
return ""
|
||||
lines: list[str] = []
|
||||
for slot in TEXT_PROFILE_SLOTS:
|
||||
cfg = tiers.get(slot)
|
||||
if not isinstance(cfg, dict):
|
||||
continue
|
||||
lines.append("")
|
||||
lines.append(f"[squilla_router.tiers.{slot}]")
|
||||
for key in (
|
||||
"provider",
|
||||
"model",
|
||||
"description",
|
||||
"supports_image",
|
||||
"image_only",
|
||||
"thinking_level",
|
||||
"thinking",
|
||||
"supports_thinking",
|
||||
):
|
||||
if key in cfg and cfg[key] is not None:
|
||||
lines.append(f"{key} = {_toml_value(cfg[key])}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _write_config(
|
||||
path: Path,
|
||||
provider: str,
|
||||
base_url: str,
|
||||
model: str,
|
||||
*,
|
||||
max_tokens: int,
|
||||
default_tier: str = "c1",
|
||||
tier_overrides: dict[str, dict[str, Any]] | None = None,
|
||||
) -> None:
|
||||
tier_override_toml = _render_tier_overrides(tier_overrides)
|
||||
path.write_text(
|
||||
f"""
|
||||
host = "127.0.0.1"
|
||||
debug = false
|
||||
llm_request_timeout_seconds = 90
|
||||
agent_runtime_timeout_seconds = {LIVE_AGENT_RUNTIME_TIMEOUT_SECONDS}
|
||||
agent_max_iterations = {LIVE_AGENT_MAX_ITERATIONS}
|
||||
|
||||
[auth]
|
||||
mode = "none"
|
||||
|
||||
[control_ui]
|
||||
enabled = false
|
||||
|
||||
[rate_limit]
|
||||
enabled = false
|
||||
|
||||
[task_runtime]
|
||||
turn_hard_deadline_s = {LIVE_TURN_HARD_DEADLINE_SECONDS}
|
||||
|
||||
[memory]
|
||||
source = "state"
|
||||
|
||||
[llm]
|
||||
provider = "{provider}"
|
||||
model = "{model}"
|
||||
base_url = "{base_url}"
|
||||
max_tokens = {max_tokens}
|
||||
|
||||
[squilla_router]
|
||||
enabled = true
|
||||
auto_thinking = true
|
||||
rollout_phase = "full"
|
||||
strategy = "v4_phase3"
|
||||
tier_profile = "{provider}"
|
||||
default_tier = "{default_tier}"
|
||||
confidence_threshold = 0.5
|
||||
kv_cache_anti_downgrade_enabled = true
|
||||
kv_cache_anti_downgrade_window_seconds = 600
|
||||
complaint_upgrade_enabled = true
|
||||
complaint_upgrade_steps = 1
|
||||
require_router_runtime = true
|
||||
{tier_override_toml}
|
||||
""".strip()
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _first_record(records: list[dict[str, Any]], *, session_key: str, kind: str) -> dict[str, Any]:
|
||||
for record in records:
|
||||
if record.get("session_key") == session_key and record.get("kind") == kind:
|
||||
return record
|
||||
return {}
|
||||
|
||||
|
||||
def _read_decision_records(state_root: Path) -> list[dict[str, Any]]:
|
||||
records: list[dict[str, Any]] = []
|
||||
for path in sorted((state_root / "logs").glob("decisions-*.jsonl")):
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
records.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
return records
|
||||
|
||||
|
||||
def _decision_for_session(
|
||||
records: list[dict[str, Any]],
|
||||
*,
|
||||
session_key: str,
|
||||
) -> dict[str, Any]:
|
||||
for record in records:
|
||||
if record.get("session_key") == session_key:
|
||||
return record
|
||||
return {}
|
||||
|
||||
|
||||
def _router_step_from_decision(decision: dict[str, Any]) -> dict[str, Any]:
|
||||
for step in decision.get("pipeline_steps") or []:
|
||||
if step.get("step_name") == "apply_squilla_router":
|
||||
return step
|
||||
return {}
|
||||
|
||||
|
||||
def _estimate_cost(
|
||||
model: str,
|
||||
usage: dict[str, Any],
|
||||
*,
|
||||
provider: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
input_tokens = int(usage.get("input_tokens") or usage.get("prompt_tokens") or 0)
|
||||
output_tokens = int(usage.get("output_tokens") or usage.get("completion_tokens") or 0)
|
||||
price = lookup_price(model)
|
||||
estimate = (
|
||||
input_tokens * price.input_per_m + output_tokens * price.output_per_m
|
||||
) / 1_000_000
|
||||
raw_billed_cost = usage.get("billed_cost")
|
||||
provider_billed_cost = None
|
||||
cost_source = "opensquilla_static_estimate"
|
||||
billing_scope = "static_estimate"
|
||||
if (
|
||||
provider == "openrouter"
|
||||
and isinstance(raw_billed_cost, int | float)
|
||||
and raw_billed_cost > 0
|
||||
):
|
||||
provider_billed_cost = float(raw_billed_cost)
|
||||
cost_source = "provider_billed"
|
||||
billing_scope = "provider_response"
|
||||
return {
|
||||
"provider_billed_cost_usd": provider_billed_cost,
|
||||
"opensquilla_estimated_cost_usd": estimate,
|
||||
"cost_source": cost_source,
|
||||
"billing_scope": billing_scope,
|
||||
"raw_gateway_usage_billed_cost_usd": usage.get("billed_cost"),
|
||||
"provider_billed": provider_billed_cost,
|
||||
"opensquilla_estimate": estimate,
|
||||
"input_per_m": price.input_per_m,
|
||||
"output_per_m": price.output_per_m,
|
||||
"source": cost_source,
|
||||
}
|
||||
|
||||
|
||||
def _failure_kind(
|
||||
row: dict[str, Any],
|
||||
actual_model: str,
|
||||
actual_routed_tier: str | None,
|
||||
) -> str | None:
|
||||
error = str(row.get("turn_error") or "")
|
||||
if error:
|
||||
lowered = error.lower()
|
||||
if "401" in lowered or "authentication" in lowered or "unauthorized" in lowered:
|
||||
return "auth_failed"
|
||||
if "429" in lowered or "quota" in lowered or "billing" in lowered:
|
||||
return "quota_or_billing_blocked"
|
||||
if "timeout" in lowered or "timed out" in lowered:
|
||||
return "gateway_turn_timeout"
|
||||
if "model" in lowered and ("not" in lowered or "invalid" in lowered):
|
||||
return "model_unavailable"
|
||||
return "unknown_provider_error"
|
||||
if not row.get("assistant_excerpt"):
|
||||
return "gateway_turn_timeout"
|
||||
if not row.get("assistant_marker_present"):
|
||||
return "content_marker_missing"
|
||||
if actual_routed_tier != row.get("expected_slot"):
|
||||
return "router_selected_unexpected_tier"
|
||||
if actual_model != row.get("expected_model"):
|
||||
return "model_unavailable"
|
||||
return None
|
||||
|
||||
|
||||
def _actual_model_from_records(
|
||||
request: dict[str, Any],
|
||||
response: dict[str, Any],
|
||||
) -> str:
|
||||
request_payload = request.get("payload") or {}
|
||||
response_payload = response.get("payload") or {}
|
||||
request_config = request_payload.get("config") or {}
|
||||
usage = response_payload.get("usage") or {}
|
||||
return str(
|
||||
request_payload.get("model")
|
||||
or request_config.get("model")
|
||||
or request.get("model")
|
||||
or usage.get("model")
|
||||
or response.get("model")
|
||||
or ""
|
||||
)
|
||||
|
||||
|
||||
def _run_gateway_case_batch(
|
||||
*,
|
||||
provider: str,
|
||||
api_key: str,
|
||||
base_url: str,
|
||||
tiers: dict[str, dict[str, Any]],
|
||||
cases: list[dict[str, Any]],
|
||||
max_tokens: int,
|
||||
timeout_seconds: float,
|
||||
case_mode: str,
|
||||
default_tier: str = "c1",
|
||||
tier_overrides: dict[str, dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
active_tiers = tier_overrides or tiers
|
||||
default_model = str(
|
||||
active_tiers.get(default_tier, {}).get("model")
|
||||
or tiers.get(default_tier, {}).get("model")
|
||||
or next(iter(_profile_slot_targets(tiers).values())).get("model")
|
||||
or ""
|
||||
)
|
||||
port = _free_port()
|
||||
tmp_path = Path(tempfile.mkdtemp(prefix=f"opensquilla-{provider}-profile-e2e-"))
|
||||
config_path = tmp_path / "gateway.toml"
|
||||
turn_log_dir = tmp_path / "turn-calls"
|
||||
_write_config(
|
||||
config_path,
|
||||
provider,
|
||||
base_url,
|
||||
default_model,
|
||||
max_tokens=max_tokens,
|
||||
default_tier=default_tier,
|
||||
tier_overrides=tier_overrides,
|
||||
)
|
||||
|
||||
env = os.environ.copy()
|
||||
env["PYTHONPATH"] = str(SRC_DIR) + os.pathsep + env.get("PYTHONPATH", "")
|
||||
env["OPENSQUILLA_GATEWAY_CONFIG_PATH"] = str(config_path)
|
||||
env["OPENSQUILLA_STATE_DIR"] = str(tmp_path / "state")
|
||||
env["OPENSQUILLA_MEMORY_DREAM_DISABLED"] = "1"
|
||||
env["OPENSQUILLA_TOOL_PROFILE"] = "channel_default"
|
||||
env["OPENSQUILLA_TURN_CALL_LOG"] = "1"
|
||||
env["OPENSQUILLA_TURN_CALL_LOG_DIR"] = str(turn_log_dir)
|
||||
env["OPENSQUILLA_LLM_PROVIDER"] = provider
|
||||
env["OPENSQUILLA_LLM_MODEL"] = default_model
|
||||
env["OPENSQUILLA_LLM_API_KEY"] = api_key
|
||||
env["OPENSQUILLA_LLM_BASE_URL"] = base_url
|
||||
if provider != "openrouter":
|
||||
# build_services still gives OPENROUTER_API_KEY special precedence for
|
||||
# legacy paths. Keep it empty for direct-provider profiles so dotenv
|
||||
# loading cannot override the selected provider key.
|
||||
env["OPENROUTER_API_KEY"] = ""
|
||||
env["OPENROUTER_BASE_URL"] = ""
|
||||
|
||||
stdout_path = tmp_path / "gateway.stdout.log"
|
||||
stderr_path = tmp_path / "gateway.stderr.log"
|
||||
with stdout_path.open("w", encoding="utf-8") as stdout_file, stderr_path.open(
|
||||
"w", encoding="utf-8"
|
||||
) as stderr_file:
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"opensquilla.cli.main",
|
||||
"gateway",
|
||||
"run",
|
||||
"--port",
|
||||
str(port),
|
||||
"--bind",
|
||||
"127.0.0.1",
|
||||
],
|
||||
cwd=REPO_ROOT,
|
||||
env=env,
|
||||
stdout=stdout_file,
|
||||
stderr=stderr_file,
|
||||
text=True,
|
||||
)
|
||||
|
||||
health: dict[str, Any] | None = None
|
||||
error: str | None = None
|
||||
rows: list[dict[str, Any]] = []
|
||||
try:
|
||||
health, error = _wait_for_gateway_health(proc, port)
|
||||
if error is None:
|
||||
for case in cases:
|
||||
slot = str(case.get("slot") or case.get("tier") or default_tier)
|
||||
marker = _case_marker(provider, slot, str(case["id"]))
|
||||
session_key = (
|
||||
f"profile-e2e:{provider}:{case['id']}:{int(time.time() * 1000)}"
|
||||
)
|
||||
message = case["message"].format(marker=marker)
|
||||
try:
|
||||
accepted = _post_json(
|
||||
f"http://127.0.0.1:{port}/api/chat",
|
||||
{
|
||||
"sessionKey": session_key,
|
||||
"message": message,
|
||||
"intent": "new_chat",
|
||||
},
|
||||
timeout=10.0,
|
||||
)
|
||||
assistant, history, turn_error = _wait_for_assistant_reply(
|
||||
port=port,
|
||||
session_key=session_key,
|
||||
previous_assistant_count=0,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - compact E2E diagnostic
|
||||
accepted = {}
|
||||
assistant = None
|
||||
history = None
|
||||
turn_error = f"{type(exc).__name__}: {exc}"
|
||||
assistant_text = str((assistant or {}).get("text", "")).strip()
|
||||
rows.append(
|
||||
{
|
||||
"case_id": case["id"],
|
||||
"case_mode": case_mode,
|
||||
"expected_slot": slot,
|
||||
"expected_tier": slot,
|
||||
"expected_model": str(tiers.get(slot, {}).get("model") or ""),
|
||||
"marker": marker,
|
||||
"session_key": session_key,
|
||||
"accepted": accepted,
|
||||
"assistant_excerpt": assistant_text[:240],
|
||||
"assistant_marker_present": marker in assistant_text,
|
||||
"history_message_count": len((history or {}).get("messages", [])),
|
||||
"turn_error": turn_error,
|
||||
}
|
||||
)
|
||||
finally:
|
||||
_stop_gateway(proc)
|
||||
stdout_file.flush()
|
||||
stderr_file.flush()
|
||||
records = _read_turn_call_records(turn_log_dir)
|
||||
decisions = _read_decision_records(tmp_path / "state")
|
||||
stdout_tail = stdout_path.read_text(encoding="utf-8", errors="replace")[-2000:]
|
||||
stderr_tail = stderr_path.read_text(encoding="utf-8", errors="replace")[-4000:]
|
||||
|
||||
enriched: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
request = _first_record(records, session_key=row["session_key"], kind="llm_request")
|
||||
response = _first_record(records, session_key=row["session_key"], kind="llm_response")
|
||||
decision = _decision_for_session(decisions, session_key=row["session_key"])
|
||||
router_step = _router_step_from_decision(decision)
|
||||
request_payload = request.get("payload") or {}
|
||||
response_payload = response.get("payload") or {}
|
||||
request_config = request_payload.get("config") or {}
|
||||
usage = response_payload.get("usage") or {}
|
||||
actual_model = _actual_model_from_records(request, response)
|
||||
actual_routed_tier = (
|
||||
router_step.get("routed_tier")
|
||||
or request_payload.get("routed_tier")
|
||||
or request_payload.get("squilla_router_tier")
|
||||
or request_config.get("routed_tier")
|
||||
)
|
||||
if actual_routed_tier is not None:
|
||||
actual_routed_tier = str(actual_routed_tier)
|
||||
failure_kind = _failure_kind(row, actual_model, actual_routed_tier)
|
||||
row_ok = (
|
||||
failure_kind is None
|
||||
and bool(row.get("assistant_excerpt"))
|
||||
and actual_model == row["expected_model"]
|
||||
and actual_routed_tier == row["expected_slot"]
|
||||
)
|
||||
enriched.append(
|
||||
{
|
||||
**row,
|
||||
"ok": row_ok,
|
||||
"failure_kind": failure_kind,
|
||||
"error": row.get("turn_error"),
|
||||
"actual_routed_tier": actual_routed_tier,
|
||||
"routing_source": router_step.get("routing_source"),
|
||||
"routing_confidence": router_step.get("confidence"),
|
||||
"actual_slot_covered": row["expected_slot"] if row_ok else None,
|
||||
"actual_request_model": actual_model or request.get("model"),
|
||||
"actual_response_model": usage.get("model"),
|
||||
"request_thinking": request_config.get("thinking"),
|
||||
"request_thinking_level": request_config.get("thinking_level"),
|
||||
"usage": {
|
||||
"input_tokens": usage.get("input_tokens"),
|
||||
"output_tokens": usage.get("output_tokens"),
|
||||
"reasoning_tokens": usage.get("reasoning_tokens"),
|
||||
"cached_tokens": usage.get("cached_tokens"),
|
||||
"billed_cost": usage.get("billed_cost"),
|
||||
},
|
||||
"cost": _estimate_cost(
|
||||
actual_model or row["expected_model"],
|
||||
usage,
|
||||
provider=provider,
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
llm_responses = [record for record in records if record.get("kind") == "llm_response"]
|
||||
batch_ok = error is None and bool(enriched) and all(row["ok"] for row in enriched)
|
||||
return {
|
||||
"case_mode": case_mode,
|
||||
"ok": batch_ok,
|
||||
"health": health or {},
|
||||
"cases": enriched,
|
||||
"usage_from_turn_logs": _usage_from_llm_responses(llm_responses),
|
||||
"error": error,
|
||||
"stdout_tail": stdout_tail,
|
||||
"stderr_tail": stderr_tail,
|
||||
}
|
||||
|
||||
|
||||
def _run_provider(provider: str, *, max_tokens: int, timeout_seconds: float) -> dict[str, Any]:
|
||||
spec = get_provider_spec(provider)
|
||||
api_key = os.environ.get(spec.env_key, "").strip()
|
||||
base_url = os.environ.get(BASE_ENV.get(provider, ""), "").strip() or spec.default_base_url
|
||||
tiers = _profile_tiers(provider)
|
||||
slot_targets = _profile_slot_targets(tiers)
|
||||
if not api_key:
|
||||
return {
|
||||
"provider": provider,
|
||||
"ok": False,
|
||||
"provider_ok": False,
|
||||
"skipped": True,
|
||||
"failure_kind": "skipped_missing_key",
|
||||
"env_key": spec.env_key,
|
||||
"base_url": base_url,
|
||||
"key_present": False,
|
||||
"tier_profile": provider,
|
||||
"tier_models": {slot: cfg.get("model") for slot, cfg in slot_targets.items()},
|
||||
"profile_slots_covered": [],
|
||||
"profile_slots_missing": list(slot_targets),
|
||||
"models_covered": [],
|
||||
"error": f"{spec.env_key} is empty",
|
||||
}
|
||||
|
||||
natural = _run_gateway_case_batch(
|
||||
provider=provider,
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
tiers=tiers,
|
||||
cases=TIER_CASES,
|
||||
max_tokens=max_tokens,
|
||||
timeout_seconds=timeout_seconds,
|
||||
case_mode="natural_router",
|
||||
)
|
||||
all_cases = list(natural.get("cases") or [])
|
||||
coverage_batches: list[dict[str, Any]] = []
|
||||
for missing_slot in _missing_profile_slots(tiers, all_cases):
|
||||
target_case = {
|
||||
"slot": missing_slot,
|
||||
"id": f"coverage_{missing_slot}",
|
||||
"message": (
|
||||
"不要调用工具，请只回复一句中文短句并包含 {marker}。"
|
||||
),
|
||||
}
|
||||
batch = _run_gateway_case_batch(
|
||||
provider=provider,
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
tiers=tiers,
|
||||
cases=[target_case],
|
||||
max_tokens=max_tokens,
|
||||
timeout_seconds=timeout_seconds,
|
||||
case_mode="coverage_compensation",
|
||||
default_tier=missing_slot,
|
||||
tier_overrides=_forced_tier_overrides_for_slot(tiers, missing_slot),
|
||||
)
|
||||
coverage_batches.append(batch)
|
||||
all_cases.extend(batch.get("cases") or [])
|
||||
|
||||
covered_slots = _covered_profile_slots(all_cases)
|
||||
missing_slots = _missing_profile_slots(tiers, all_cases)
|
||||
models_covered = sorted(
|
||||
{
|
||||
str(row.get("actual_request_model") or row.get("expected_model") or "")
|
||||
for row in all_cases
|
||||
if row.get("ok") is True
|
||||
}
|
||||
- {""}
|
||||
)
|
||||
natural_cases = [row for row in all_cases if row.get("case_mode") == "natural_router"]
|
||||
coverage_cases = [
|
||||
row for row in all_cases if row.get("case_mode") == "coverage_compensation"
|
||||
]
|
||||
provider_ok = not missing_slots and any(
|
||||
row.get("case_mode") == "natural_router" and row.get("assistant_excerpt")
|
||||
for row in all_cases
|
||||
)
|
||||
failure_kinds = sorted(
|
||||
{str(row.get("failure_kind")) for row in all_cases if row.get("failure_kind")}
|
||||
)
|
||||
return {
|
||||
"provider": provider,
|
||||
"ok": provider_ok,
|
||||
"provider_ok": provider_ok,
|
||||
"env_key": spec.env_key,
|
||||
"base_url": base_url,
|
||||
"key_present": bool(api_key),
|
||||
"tier_profile": provider,
|
||||
"tier_models": {slot: cfg.get("model") for slot, cfg in slot_targets.items()},
|
||||
"profile_slots_covered": covered_slots,
|
||||
"profile_slots_missing": missing_slots,
|
||||
"models_covered": models_covered,
|
||||
"natural_cases_ok": bool(natural_cases)
|
||||
and all(
|
||||
row.get("failure_kind") in (None, "router_selected_unexpected_tier")
|
||||
for row in natural_cases
|
||||
),
|
||||
"coverage_cases_ok": bool(coverage_cases) and all(row.get("ok") for row in coverage_cases)
|
||||
if coverage_cases
|
||||
else True,
|
||||
"health": natural.get("health") or {},
|
||||
"cases": all_cases,
|
||||
"batches": [natural, *coverage_batches],
|
||||
"usage_from_turn_logs": natural.get("usage_from_turn_logs"),
|
||||
"failure_kinds": failure_kinds,
|
||||
"error": "; ".join(failure_kinds) or natural.get("error"),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--providers", nargs="+", default=DEFAULT_PROVIDERS)
|
||||
parser.add_argument("--output", required=True)
|
||||
parser.add_argument("--max-tokens", type=int, default=768)
|
||||
parser.add_argument("--timeout-seconds", type=float, default=120.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
_load_env_quietly()
|
||||
results = [
|
||||
_run_provider(
|
||||
provider,
|
||||
max_tokens=args.max_tokens,
|
||||
timeout_seconds=args.timeout_seconds,
|
||||
)
|
||||
for provider in args.providers
|
||||
]
|
||||
payload = {
|
||||
"generated_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
"ok": all(result.get("ok") is True for result in results),
|
||||
"note": (
|
||||
"provider_billed_cost_usd is unavailable here; "
|
||||
"opensquilla_estimated_cost_usd is a static local estimate computed "
|
||||
"from returned token usage."
|
||||
),
|
||||
"results": results,
|
||||
}
|
||||
output = Path(args.output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
return 0 if payload["ok"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,535 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Live smoke selected provider profiles without printing secrets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from opensquilla.engine.pricing import lookup_price
|
||||
from opensquilla.provider.registry import get_provider_spec
|
||||
from opensquilla.provider.selector import ProviderConfig, _build_provider
|
||||
from opensquilla.provider.types import ChatConfig, DoneEvent, ErrorEvent, Message, TextDeltaEvent
|
||||
|
||||
|
||||
@dataclass
|
||||
class SmokeResult:
|
||||
provider: str
|
||||
model: str
|
||||
base_url: str
|
||||
env_key: str
|
||||
key_present: bool
|
||||
direct_status: str
|
||||
stream_status: str
|
||||
response_model: str
|
||||
content_match: str
|
||||
usage: dict[str, Any]
|
||||
cost: dict[str, Any]
|
||||
error: str
|
||||
latency_ms: int
|
||||
|
||||
|
||||
_MODEL_ENV = {
|
||||
"openai": "OPENAI_MODEL",
|
||||
"dashscope": "DASHSCOPE_MODEL",
|
||||
"deepseek": "DEEPSEEK_MODEL",
|
||||
"gemini": "GEMINI_MODEL",
|
||||
"volcengine": "VOLCENGINE_MODEL",
|
||||
"volcengine_coding_plan": "VOLCENGINE_CODING_MODEL",
|
||||
"byteplus": "BYTEPLUS_MODEL",
|
||||
"bailian_coding": "BAILIAN_CODING_MODEL",
|
||||
"moonshot": "MOONSHOT_MODEL",
|
||||
"kimi_coding_openai": "KIMI_CODING_MODEL",
|
||||
"kimi_coding_anthropic": "KIMI_CODING_MODEL",
|
||||
"zhipu": "ZAI_MODEL",
|
||||
"qianfan": "QIANFAN_MODEL",
|
||||
"minimax": "MINIMAX_MODEL",
|
||||
"minimax_openai": "MINIMAX_MODEL",
|
||||
"minimax_coding_openai": "MINIMAX_CODING_MODEL",
|
||||
"minimax_coding_anthropic": "MINIMAX_CODING_MODEL",
|
||||
"minimax_cn": "MINIMAX_CN_MODEL",
|
||||
"minimax_global": "MINIMAX_GLOBAL_MODEL",
|
||||
"mimo_openai": "MIMO_MODEL",
|
||||
"mimo_anthropic": "MIMO_MODEL",
|
||||
"tencent_tokenhub": "TENCENT_TOKENHUB_MODEL",
|
||||
"tencent_tokenhub_anthropic": "TENCENT_TOKENHUB_MODEL",
|
||||
"tencent_tokenhub_intl": "TENCENT_TOKENHUB_INTL_MODEL",
|
||||
"tencent_token_plan": "TENCENT_TOKEN_PLAN_MODEL",
|
||||
"tencent_token_plan_anthropic": "TENCENT_TOKEN_PLAN_MODEL",
|
||||
"tokenrhythm": "TOKENRHYTHM_MODEL",
|
||||
}
|
||||
|
||||
_BASE_ENV = {
|
||||
"openai": "OPENAI_BASE_URL",
|
||||
"dashscope": "DASHSCOPE_BASE_URL",
|
||||
"deepseek": "DEEPSEEK_BASE_URL",
|
||||
"gemini": "GEMINI_BASE_URL",
|
||||
"volcengine": "VOLCENGINE_BASE_URL",
|
||||
"volcengine_coding_plan": "VOLCENGINE_CODING_BASE_URL",
|
||||
"byteplus": "BYTEPLUS_BASE_URL",
|
||||
"bailian_coding": "BAILIAN_CODING_BASE_URL",
|
||||
"moonshot": "MOONSHOT_BASE_URL",
|
||||
"kimi_coding_openai": "KIMI_CODING_OPENAI_BASE_URL",
|
||||
"kimi_coding_anthropic": "KIMI_CODING_ANTHROPIC_BASE_URL",
|
||||
"zhipu": "ZAI_BASE_URL",
|
||||
"qianfan": "QIANFAN_BASE_URL",
|
||||
"minimax": "MINIMAX_BASE_URL",
|
||||
"minimax_openai": "MINIMAX_OPENAI_BASE_URL",
|
||||
"minimax_coding_openai": "MINIMAX_CODING_OPENAI_BASE_URL",
|
||||
"minimax_coding_anthropic": "MINIMAX_CODING_ANTHROPIC_BASE_URL",
|
||||
"minimax_cn": "MINIMAX_CN_BASE_URL",
|
||||
"minimax_global": "MINIMAX_GLOBAL_BASE_URL",
|
||||
"mimo_openai": "MIMO_OPENAI_BASE_URL",
|
||||
"mimo_anthropic": "MIMO_ANTHROPIC_BASE_URL",
|
||||
"tencent_tokenhub": "TENCENT_TOKENHUB_BASE_URL",
|
||||
"tencent_tokenhub_anthropic": "TENCENT_TOKENHUB_ANTHROPIC_BASE_URL",
|
||||
"tencent_tokenhub_intl": "TENCENT_TOKENHUB_INTL_BASE_URL",
|
||||
"tencent_token_plan": "TENCENT_TOKEN_PLAN_BASE_URL",
|
||||
"tencent_token_plan_anthropic": "TENCENT_TOKEN_PLAN_ANTHROPIC_BASE_URL",
|
||||
"tokenrhythm": "TOKENRHYTHM_BASE_URL",
|
||||
}
|
||||
|
||||
_DEFAULT_MODELS = {
|
||||
"openai": "gpt-5.4-mini",
|
||||
"dashscope": "qwen3.7-plus",
|
||||
"deepseek": "deepseek-v4-flash",
|
||||
"gemini": "gemini-3.5-flash",
|
||||
"volcengine": "doubao-seed-2-0-lite-260215",
|
||||
"volcengine_coding_plan": "doubao-seed-2.0-pro",
|
||||
"byteplus": "seed-2-0-lite-260228",
|
||||
"bailian_coding": "kimi-k2.5",
|
||||
"moonshot": "kimi-k2.6",
|
||||
"kimi_coding_openai": "kimi-for-coding",
|
||||
"kimi_coding_anthropic": "kimi-for-coding",
|
||||
"zhipu": "glm-5",
|
||||
"qianfan": "ernie-4.5-turbo-128k",
|
||||
"minimax": "MiniMax-M2.7",
|
||||
"minimax_openai": "MiniMax-M2.7",
|
||||
"minimax_coding_openai": "MiniMax-M2.7",
|
||||
"minimax_coding_anthropic": "MiniMax-M2.7",
|
||||
"minimax_cn": "MiniMax-M2.7",
|
||||
"minimax_global": "MiniMax-M2.7",
|
||||
"mimo_openai": "mimo-v2.5",
|
||||
"mimo_anthropic": "mimo-v2.5-pro",
|
||||
"tencent_tokenhub": "hy3",
|
||||
"tencent_tokenhub_anthropic": "hy3",
|
||||
"tencent_tokenhub_intl": "deepseek-v3.2",
|
||||
"tencent_token_plan": "hy3",
|
||||
"tencent_token_plan_anthropic": "hy3",
|
||||
"tokenrhythm": "deepseek-v4-flash",
|
||||
}
|
||||
|
||||
# Providers whose models spend reasoning tokens out of max_tokens before any
|
||||
# text: the CLI default budget of 64 would come back as empty content with
|
||||
# finish_reason "length", failing the smoke for provider-independent reasons.
|
||||
_MIN_MAX_TOKENS = {
|
||||
"tokenrhythm": 1024,
|
||||
}
|
||||
|
||||
|
||||
def _csv_values(raw: str | None) -> list[str]:
|
||||
if not raw:
|
||||
return []
|
||||
return [part.strip() for part in raw.split(",") if part.strip()]
|
||||
|
||||
|
||||
def _load_env_quietly(path: Path = Path(".env")) -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
for raw_line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
key = key.strip()
|
||||
value = value.strip().strip('"').strip("'")
|
||||
if key and key not in os.environ:
|
||||
os.environ[key] = value
|
||||
|
||||
|
||||
def _headers_for_openai(api_key: str) -> dict[str, str]:
|
||||
# Keyless local providers must not send an empty Bearer value (httpx
|
||||
# rejects it as an illegal header).
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
return headers
|
||||
|
||||
|
||||
def _headers_for_anthropic(api_key: str) -> dict[str, str]:
|
||||
return {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
"anthropic-version": "2023-06-01",
|
||||
}
|
||||
|
||||
|
||||
def _versioned_chat_url(base_url: str) -> str:
|
||||
base = base_url.rstrip("/")
|
||||
if base.endswith(("/v1", "/v2", "/v3", "/v4")):
|
||||
return f"{base}/chat/completions"
|
||||
return f"{base}/v1/chat/completions"
|
||||
|
||||
|
||||
def _direct_openai_temperature(provider: str, model: str) -> int:
|
||||
if provider == "kimi_coding_openai" and model == "kimi-for-coding":
|
||||
return 1
|
||||
if provider == "moonshot" and model.lower().startswith("kimi-k2."):
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def _direct_openai_token_limit_field(provider: str, model: str) -> str:
|
||||
if provider == "openai" and model.lower().startswith(("gpt-5", "o1", "o3", "o4")):
|
||||
return "max_completion_tokens"
|
||||
return "max_tokens"
|
||||
|
||||
|
||||
async def _direct_openai(
|
||||
provider: str,
|
||||
model: str,
|
||||
api_key: str,
|
||||
base_url: str,
|
||||
expected: str,
|
||||
max_tokens: int,
|
||||
) -> tuple[str, str, str, dict[str, Any], int]:
|
||||
start = time.perf_counter()
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"Reply exactly with: {expected}",
|
||||
}
|
||||
],
|
||||
"temperature": _direct_openai_temperature(provider, model),
|
||||
}
|
||||
payload[_direct_openai_token_limit_field(provider, model)] = max_tokens
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30.0, trust_env=False) as client:
|
||||
resp = await client.post(
|
||||
_versioned_chat_url(base_url),
|
||||
headers=_headers_for_openai(api_key),
|
||||
json=payload,
|
||||
)
|
||||
latency = int((time.perf_counter() - start) * 1000)
|
||||
if resp.status_code >= 400:
|
||||
return "failed", "", _error_summary(resp), {}, latency
|
||||
data = resp.json()
|
||||
content = data.get("choices", [{}])[0].get("message", {}).get("content", "")
|
||||
response_model = str(data.get("model") or "")
|
||||
status = "passed" if expected in content else "content_mismatch"
|
||||
return status, response_model, content, _usage_summary(data.get("usage")), latency
|
||||
except Exception as exc: # noqa: BLE001 - smoke reports compact diagnostic
|
||||
latency = int((time.perf_counter() - start) * 1000)
|
||||
return "failed", "", f"{type(exc).__name__}: {exc}", {}, latency
|
||||
|
||||
|
||||
async def _direct_anthropic(
|
||||
model: str,
|
||||
api_key: str,
|
||||
base_url: str,
|
||||
expected: str,
|
||||
max_tokens: int,
|
||||
) -> tuple[str, str, str, dict[str, Any], int]:
|
||||
start = time.perf_counter()
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": f"Reply exactly with: {expected}"}],
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": 1,
|
||||
}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30.0, trust_env=False) as client:
|
||||
resp = await client.post(
|
||||
f"{base_url.rstrip('/')}/v1/messages",
|
||||
headers=_headers_for_anthropic(api_key),
|
||||
json=payload,
|
||||
)
|
||||
latency = int((time.perf_counter() - start) * 1000)
|
||||
if resp.status_code >= 400:
|
||||
return "failed", "", _error_summary(resp), {}, latency
|
||||
data = resp.json()
|
||||
text_parts = [
|
||||
block.get("text", "")
|
||||
for block in data.get("content", [])
|
||||
if isinstance(block, dict) and block.get("type") == "text"
|
||||
]
|
||||
content = "".join(text_parts)
|
||||
response_model = str(data.get("model") or "")
|
||||
status = "passed" if expected in content else "content_mismatch"
|
||||
return status, response_model, content, _usage_summary(data.get("usage")), latency
|
||||
except Exception as exc: # noqa: BLE001 - smoke reports compact diagnostic
|
||||
latency = int((time.perf_counter() - start) * 1000)
|
||||
return "failed", "", f"{type(exc).__name__}: {exc}", {}, latency
|
||||
|
||||
|
||||
async def _stream_opensquilla(
|
||||
provider: str,
|
||||
model: str,
|
||||
api_key: str,
|
||||
base_url: str,
|
||||
expected: str,
|
||||
max_tokens: int,
|
||||
) -> tuple[str, str, dict[str, Any], int]:
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
built = _build_provider(
|
||||
ProviderConfig(provider=provider, model=model, api_key=api_key, base_url=base_url)
|
||||
)
|
||||
chunks: list[str] = []
|
||||
done: DoneEvent | None = None
|
||||
async for event in built.chat(
|
||||
[Message(role="user", content=f"Reply exactly with: {expected}")],
|
||||
config=ChatConfig(max_tokens=max_tokens, temperature=1, timeout=30.0),
|
||||
):
|
||||
if isinstance(event, TextDeltaEvent):
|
||||
chunks.append(event.text)
|
||||
elif isinstance(event, DoneEvent):
|
||||
done = event
|
||||
elif isinstance(event, ErrorEvent):
|
||||
latency = int((time.perf_counter() - start) * 1000)
|
||||
return "failed", event.message or event.code, {}, latency
|
||||
latency = int((time.perf_counter() - start) * 1000)
|
||||
content = "".join(chunks)
|
||||
if done is None:
|
||||
return "failed", "missing DoneEvent", {}, latency
|
||||
usage = {
|
||||
"input_tokens": done.input_tokens,
|
||||
"output_tokens": done.output_tokens,
|
||||
"cached_tokens": done.cached_tokens,
|
||||
"cache_write_tokens": done.cache_write_tokens,
|
||||
"reasoning_tokens": done.reasoning_tokens,
|
||||
"model": done.model,
|
||||
"billed_cost": done.billed_cost,
|
||||
"cost_source": done.cost_source,
|
||||
}
|
||||
status = "passed" if expected in content else "content_mismatch"
|
||||
return status, content, usage, latency
|
||||
except Exception as exc: # noqa: BLE001 - smoke reports compact diagnostic
|
||||
latency = int((time.perf_counter() - start) * 1000)
|
||||
return "failed", f"{type(exc).__name__}: {exc}", {}, latency
|
||||
|
||||
|
||||
def _usage_summary(usage: Any) -> dict[str, Any]:
|
||||
if not isinstance(usage, dict):
|
||||
return {}
|
||||
keys = (
|
||||
"prompt_tokens",
|
||||
"completion_tokens",
|
||||
"total_tokens",
|
||||
"input_tokens",
|
||||
"output_tokens",
|
||||
"cache_read_input_tokens",
|
||||
"cache_creation_input_tokens",
|
||||
)
|
||||
return {key: usage[key] for key in keys if key in usage}
|
||||
|
||||
|
||||
def _cost_estimate(model: str, usage: dict[str, Any]) -> dict[str, Any]:
|
||||
direct_usage = usage.get("direct") if isinstance(usage.get("direct"), dict) else {}
|
||||
stream_usage = usage.get("stream") if isinstance(usage.get("stream"), dict) else {}
|
||||
prompt_tokens = direct_usage.get("prompt_tokens") or stream_usage.get("input_tokens") or 0
|
||||
completion_tokens = (
|
||||
direct_usage.get("completion_tokens") or stream_usage.get("output_tokens") or 0
|
||||
)
|
||||
price = lookup_price(model)
|
||||
estimate = (
|
||||
prompt_tokens * price.input_per_m + completion_tokens * price.output_per_m
|
||||
) / 1_000_000
|
||||
# The stream DoneEvent carries the provider-billed cost when the upstream
|
||||
# reports one (OpenRouter usage.cost); surface it instead of pretending
|
||||
# only static estimates exist.
|
||||
billed = stream_usage.get("billed_cost") or 0.0
|
||||
billed_source = str(stream_usage.get("cost_source") or "")
|
||||
provider_billed = billed if billed > 0 and billed_source == "provider_billed" else None
|
||||
cost_source = billed_source if provider_billed is not None else "opensquilla_static_estimate"
|
||||
return {
|
||||
"provider_billed_cost_usd": provider_billed,
|
||||
"opensquilla_estimated_cost_usd": estimate,
|
||||
"cost_source": cost_source,
|
||||
"billing_scope": "provider_billed" if provider_billed is not None else "static_estimate",
|
||||
"provider_billed": provider_billed,
|
||||
"opensquilla_estimate": estimate,
|
||||
"input_per_m": price.input_per_m,
|
||||
"output_per_m": price.output_per_m,
|
||||
"source": cost_source,
|
||||
}
|
||||
|
||||
|
||||
def _error_summary(resp: httpx.Response) -> str:
|
||||
try:
|
||||
body = resp.json()
|
||||
except ValueError:
|
||||
body = resp.text[:300]
|
||||
return f"HTTP {resp.status_code}: {body}"
|
||||
|
||||
|
||||
async def smoke_provider(
|
||||
provider: str,
|
||||
*,
|
||||
include_stream: bool = True,
|
||||
model_override: str | None = None,
|
||||
base_url_override: str | None = None,
|
||||
max_tokens: int = 64,
|
||||
) -> SmokeResult:
|
||||
spec = get_provider_spec(provider)
|
||||
env_key = spec.env_key
|
||||
api_key = os.environ.get(env_key, "").strip()
|
||||
max_tokens = max(max_tokens, _MIN_MAX_TOKENS.get(provider, 0))
|
||||
model = (
|
||||
model_override
|
||||
or os.environ.get(_MODEL_ENV.get(provider, ""), "").strip()
|
||||
or _DEFAULT_MODELS.get(provider, "")
|
||||
)
|
||||
if not model:
|
||||
raise SystemExit(
|
||||
f"no model configured for provider {provider!r}: pass --model or set "
|
||||
f"{_MODEL_ENV.get(provider) or 'a model env override'}"
|
||||
)
|
||||
base_url = (
|
||||
base_url_override
|
||||
or os.environ.get(_BASE_ENV.get(provider, ""), "").strip()
|
||||
or spec.default_base_url
|
||||
)
|
||||
expected = f"opensquilla {provider} smoke ok"
|
||||
|
||||
# Local providers (ollama, lm_studio, ovms) declare their key optional in
|
||||
# the registry; only skip when the spec actually requires one.
|
||||
if not api_key and spec.requires_api_key():
|
||||
return SmokeResult(
|
||||
provider=provider,
|
||||
model=model,
|
||||
base_url=base_url,
|
||||
env_key=env_key,
|
||||
key_present=False,
|
||||
direct_status="skipped",
|
||||
stream_status="skipped",
|
||||
response_model="",
|
||||
content_match="not_run",
|
||||
usage={},
|
||||
cost={
|
||||
"provider_billed_cost_usd": None,
|
||||
"opensquilla_estimated_cost_usd": None,
|
||||
"cost_source": "unavailable",
|
||||
"billing_scope": "none",
|
||||
"provider_billed": None,
|
||||
"opensquilla_estimate": None,
|
||||
"source": "unavailable",
|
||||
},
|
||||
error=f"{env_key} is empty",
|
||||
latency_ms=0,
|
||||
)
|
||||
|
||||
if spec.backend == "anthropic":
|
||||
(
|
||||
direct_status,
|
||||
response_model,
|
||||
direct_content,
|
||||
usage,
|
||||
direct_latency,
|
||||
) = await _direct_anthropic(model, api_key, base_url, expected, max_tokens)
|
||||
else:
|
||||
direct_status, response_model, direct_content, usage, direct_latency = await _direct_openai(
|
||||
provider, model, api_key, base_url, expected, max_tokens
|
||||
)
|
||||
if include_stream:
|
||||
stream_status, stream_content, stream_usage, stream_latency = await _stream_opensquilla(
|
||||
provider, model, api_key, base_url, expected, max_tokens
|
||||
)
|
||||
else:
|
||||
stream_status = "skipped"
|
||||
stream_content = ""
|
||||
stream_usage = {}
|
||||
stream_latency = 0
|
||||
|
||||
errors = []
|
||||
if direct_status == "failed":
|
||||
errors.append(f"direct={direct_content}")
|
||||
if stream_status == "failed":
|
||||
errors.append(f"stream={stream_content}")
|
||||
content_match = (
|
||||
"exact" if direct_status == "passed" and stream_status == "passed" else "not_validated"
|
||||
)
|
||||
if direct_status == "passed" and stream_status == "skipped":
|
||||
content_match = "direct_exact"
|
||||
merged_usage = {"direct": usage, "stream": stream_usage}
|
||||
|
||||
return SmokeResult(
|
||||
provider=provider,
|
||||
model=model,
|
||||
base_url=base_url,
|
||||
env_key=env_key,
|
||||
key_present=bool(api_key),
|
||||
direct_status=direct_status,
|
||||
stream_status=stream_status,
|
||||
response_model=response_model,
|
||||
content_match=content_match,
|
||||
usage=merged_usage,
|
||||
cost=_cost_estimate(response_model or model, merged_usage),
|
||||
error="; ".join(errors),
|
||||
latency_ms=direct_latency + stream_latency,
|
||||
)
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--provider")
|
||||
parser.add_argument(
|
||||
"--providers",
|
||||
nargs="+",
|
||||
default=["dashscope", "deepseek", "gemini", "volcengine", "byteplus"],
|
||||
)
|
||||
parser.add_argument("--models")
|
||||
parser.add_argument("--model")
|
||||
parser.add_argument("--base-url")
|
||||
parser.add_argument("--max-tokens", type=int, default=64)
|
||||
parser.add_argument("--skip-stream", action="store_true")
|
||||
parser.add_argument("--output", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
_load_env_quietly()
|
||||
providers = [args.provider] if args.provider else list(args.providers)
|
||||
models = _csv_values(args.models)
|
||||
if args.model and models:
|
||||
parser.error("--model and --models are mutually exclusive")
|
||||
if models and len(providers) != 1:
|
||||
parser.error("--models requires exactly one provider")
|
||||
|
||||
jobs: list[tuple[str, str | None]] = []
|
||||
if models:
|
||||
jobs = [(providers[0], model) for model in models]
|
||||
else:
|
||||
jobs = [(provider, args.model) for provider in providers]
|
||||
|
||||
results = [
|
||||
await smoke_provider(
|
||||
provider,
|
||||
include_stream=not args.skip_stream,
|
||||
model_override=model,
|
||||
base_url_override=args.base_url,
|
||||
max_tokens=args.max_tokens,
|
||||
)
|
||||
for provider, model in jobs
|
||||
]
|
||||
payload = {
|
||||
"generated_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
"results": [asdict(result) for result in results],
|
||||
}
|
||||
output = Path(args.output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
@@ -0,0 +1,408 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Live smoke provider-native thinking controls without printing secrets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from opensquilla.provider.model_catalog import ModelCatalog
|
||||
from opensquilla.provider.registry import get_provider_spec
|
||||
from opensquilla.provider.selector import ProviderConfig, _build_provider
|
||||
from opensquilla.provider.types import ChatConfig, DoneEvent, ErrorEvent, Message, TextDeltaEvent
|
||||
|
||||
_MODEL_ENV = {
|
||||
"volcengine": "VOLCENGINE_MODEL",
|
||||
"deepseek": "DEEPSEEK_MODEL",
|
||||
"dashscope": "DASHSCOPE_MODEL",
|
||||
"gemini": "GEMINI_MODEL",
|
||||
"moonshot": "MOONSHOT_MODEL",
|
||||
"zhipu": "ZAI_MODEL",
|
||||
}
|
||||
|
||||
_BASE_ENV = {
|
||||
"volcengine": "VOLCENGINE_BASE_URL",
|
||||
"deepseek": "DEEPSEEK_BASE_URL",
|
||||
"dashscope": "DASHSCOPE_BASE_URL",
|
||||
"gemini": "GEMINI_BASE_URL",
|
||||
"moonshot": "MOONSHOT_BASE_URL",
|
||||
"zhipu": "ZAI_BASE_URL",
|
||||
}
|
||||
|
||||
_DEFAULT_MODELS = {
|
||||
"volcengine": "doubao-seed-1-6-thinking-250715",
|
||||
"deepseek": "deepseek-v4-pro",
|
||||
"dashscope": "qwen3.6-plus",
|
||||
"gemini": "gemini-2.5-flash",
|
||||
"moonshot": "kimi-k2.5",
|
||||
"zhipu": "glm-5.1",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ThinkingCaseResult:
|
||||
mode: str
|
||||
direct_status: str
|
||||
direct_latency_ms: int
|
||||
direct_response_model: str
|
||||
direct_text: str
|
||||
direct_reasoning_content_present: bool
|
||||
direct_usage: dict[str, Any]
|
||||
direct_error: str
|
||||
stream_status: str
|
||||
stream_latency_ms: int
|
||||
stream_text: str
|
||||
stream_reasoning_content_present: bool
|
||||
stream_reasoning_tokens: int
|
||||
stream_usage: dict[str, Any]
|
||||
stream_error: str
|
||||
expected_marker_present_direct: bool
|
||||
expected_marker_present_stream: bool
|
||||
|
||||
|
||||
def _load_env_quietly(path: Path = Path(".env")) -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
for raw_line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
key = key.strip()
|
||||
value = value.strip().strip('"').strip("'")
|
||||
if key and key not in os.environ:
|
||||
os.environ[key] = value
|
||||
|
||||
|
||||
def _chat_url(base_url: str) -> str:
|
||||
base = base_url.rstrip("/")
|
||||
if base.endswith(("/v1", "/v2", "/v3", "/v4")):
|
||||
return f"{base}/chat/completions"
|
||||
return f"{base}/v1/chat/completions"
|
||||
|
||||
|
||||
def _provider_thinking_payload(
|
||||
provider: str,
|
||||
*,
|
||||
enabled: bool,
|
||||
budget: int,
|
||||
) -> dict[str, Any]:
|
||||
if provider == "dashscope":
|
||||
payload: dict[str, Any] = {"enable_thinking": enabled}
|
||||
if enabled:
|
||||
payload["thinking_budget"] = budget
|
||||
return payload
|
||||
if provider == "gemini":
|
||||
return {"reasoning_effort": "medium" if enabled else "none"}
|
||||
if provider == "deepseek":
|
||||
payload = {"thinking": {"type": "enabled" if enabled else "disabled"}}
|
||||
if enabled:
|
||||
payload["reasoning_effort"] = "high"
|
||||
return payload
|
||||
if provider in {"moonshot", "volcengine", "zhipu"}:
|
||||
return {"thinking": {"type": "enabled" if enabled else "disabled"}}
|
||||
return {}
|
||||
|
||||
|
||||
def _usage_summary(usage: Any) -> dict[str, Any]:
|
||||
if not isinstance(usage, dict):
|
||||
return {}
|
||||
keys = (
|
||||
"prompt_tokens",
|
||||
"completion_tokens",
|
||||
"total_tokens",
|
||||
"input_tokens",
|
||||
"output_tokens",
|
||||
"reasoning_tokens",
|
||||
"cache_read_input_tokens",
|
||||
"cache_creation_input_tokens",
|
||||
)
|
||||
summary = {key: usage[key] for key in keys if key in usage}
|
||||
details = usage.get("completion_tokens_details")
|
||||
if isinstance(details, dict) and "reasoning_tokens" in details:
|
||||
summary["completion_tokens_details.reasoning_tokens"] = details["reasoning_tokens"]
|
||||
return summary
|
||||
|
||||
|
||||
async def _direct_case(
|
||||
*,
|
||||
provider: str,
|
||||
model: str,
|
||||
api_key: str,
|
||||
base_url: str,
|
||||
marker: str,
|
||||
enabled: bool,
|
||||
max_tokens: int,
|
||||
thinking_budget: int,
|
||||
) -> tuple[str, int, str, str, bool, dict[str, Any], str, dict[str, Any]]:
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": f"Reply exactly with: {marker}",
|
||||
}
|
||||
],
|
||||
"max_tokens": max_tokens,
|
||||
**_provider_thinking_payload(provider, enabled=enabled, budget=thinking_budget),
|
||||
}
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=60.0, trust_env=False) as client:
|
||||
resp = await client.post(
|
||||
_chat_url(base_url),
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json=payload,
|
||||
)
|
||||
latency_ms = int((time.perf_counter() - start) * 1000)
|
||||
if resp.status_code >= 400:
|
||||
return (
|
||||
"failed",
|
||||
latency_ms,
|
||||
"",
|
||||
"",
|
||||
False,
|
||||
{},
|
||||
f"HTTP {resp.status_code}: {resp.text[:500]}",
|
||||
payload,
|
||||
)
|
||||
data = resp.json()
|
||||
message = data.get("choices", [{}])[0].get("message", {})
|
||||
text = str(message.get("content") or "")
|
||||
reasoning_content = str(message.get("reasoning_content") or "")
|
||||
status = "passed" if marker in text else "content_mismatch"
|
||||
return (
|
||||
status,
|
||||
latency_ms,
|
||||
str(data.get("model") or ""),
|
||||
text,
|
||||
bool(reasoning_content),
|
||||
_usage_summary(data.get("usage")),
|
||||
"",
|
||||
payload,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - smoke reports compact diagnostics
|
||||
latency_ms = int((time.perf_counter() - start) * 1000)
|
||||
return ("failed", latency_ms, "", "", False, {}, f"{type(exc).__name__}: {exc}", payload)
|
||||
|
||||
|
||||
async def _stream_case(
|
||||
*,
|
||||
provider: str,
|
||||
model: str,
|
||||
api_key: str,
|
||||
base_url: str,
|
||||
marker: str,
|
||||
enabled: bool,
|
||||
max_tokens: int,
|
||||
thinking_budget: int,
|
||||
) -> tuple[str, int, str, bool, int, dict[str, Any], str, dict[str, Any]]:
|
||||
caps = ModelCatalog().get_capabilities(model, provider_name=provider, base_url=base_url)
|
||||
config = ChatConfig(
|
||||
max_tokens=max_tokens,
|
||||
temperature=None,
|
||||
thinking=enabled,
|
||||
thinking_budget_tokens=thinking_budget if enabled else 0,
|
||||
timeout=60.0,
|
||||
model_capabilities=caps,
|
||||
)
|
||||
provider_obj = _build_provider(
|
||||
ProviderConfig(provider=provider, model=model, api_key=api_key, base_url=base_url)
|
||||
)
|
||||
start = time.perf_counter()
|
||||
chunks: list[str] = []
|
||||
done: DoneEvent | None = None
|
||||
error = ""
|
||||
try:
|
||||
async for event in provider_obj.chat(
|
||||
[Message(role="user", content=f"Reply exactly with: {marker}")],
|
||||
config=config,
|
||||
):
|
||||
if isinstance(event, TextDeltaEvent):
|
||||
chunks.append(event.text)
|
||||
elif isinstance(event, DoneEvent):
|
||||
done = event
|
||||
elif isinstance(event, ErrorEvent):
|
||||
error = event.message or event.code
|
||||
break
|
||||
latency_ms = int((time.perf_counter() - start) * 1000)
|
||||
text = "".join(chunks)
|
||||
if error:
|
||||
return (
|
||||
"failed",
|
||||
latency_ms,
|
||||
text,
|
||||
False,
|
||||
0,
|
||||
{},
|
||||
error,
|
||||
config.model_dump(mode="json"),
|
||||
)
|
||||
if done is None:
|
||||
return (
|
||||
"failed",
|
||||
latency_ms,
|
||||
text,
|
||||
False,
|
||||
0,
|
||||
{},
|
||||
"missing DoneEvent",
|
||||
config.model_dump(mode="json"),
|
||||
)
|
||||
status = "passed" if marker in text else "content_mismatch"
|
||||
return (
|
||||
status,
|
||||
latency_ms,
|
||||
text,
|
||||
bool(done.reasoning_content),
|
||||
done.reasoning_tokens,
|
||||
{
|
||||
"input_tokens": done.input_tokens,
|
||||
"output_tokens": done.output_tokens,
|
||||
"reasoning_tokens": done.reasoning_tokens,
|
||||
"cached_tokens": done.cached_tokens,
|
||||
"cache_write_tokens": done.cache_write_tokens,
|
||||
"billed_cost": done.billed_cost,
|
||||
"model": done.model,
|
||||
},
|
||||
"",
|
||||
config.model_dump(mode="json"),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - smoke reports compact diagnostics
|
||||
latency_ms = int((time.perf_counter() - start) * 1000)
|
||||
return (
|
||||
"failed",
|
||||
latency_ms,
|
||||
"".join(chunks),
|
||||
False,
|
||||
0,
|
||||
{},
|
||||
f"{type(exc).__name__}: {exc}",
|
||||
config.model_dump(mode="json"),
|
||||
)
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--provider", default="volcengine")
|
||||
parser.add_argument("--model")
|
||||
parser.add_argument("--base-url")
|
||||
parser.add_argument("--max-tokens", type=int, default=512)
|
||||
parser.add_argument("--thinking-budget", type=int, default=4096)
|
||||
parser.add_argument("--output", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
_load_env_quietly()
|
||||
spec = get_provider_spec(args.provider)
|
||||
model = (
|
||||
args.model
|
||||
or os.environ.get(_MODEL_ENV.get(args.provider, ""), "").strip()
|
||||
or _DEFAULT_MODELS[args.provider]
|
||||
)
|
||||
base_url = (
|
||||
args.base_url
|
||||
or os.environ.get(_BASE_ENV.get(args.provider, ""), "").strip()
|
||||
or spec.default_base_url
|
||||
)
|
||||
api_key = os.environ.get(spec.env_key, "").strip()
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"generated_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
"provider": args.provider,
|
||||
"model": model,
|
||||
"base_url": base_url,
|
||||
"env_key": spec.env_key,
|
||||
"key_present": bool(api_key),
|
||||
"cases": [],
|
||||
}
|
||||
if not api_key:
|
||||
payload["error"] = f"{spec.env_key} is empty"
|
||||
else:
|
||||
for mode, enabled in (("thinking_enabled", True), ("thinking_disabled", False)):
|
||||
marker = f"THINKING_{args.provider.upper()}_{mode.upper()}_{int(time.time() * 1000)}"
|
||||
(
|
||||
direct_status,
|
||||
direct_latency_ms,
|
||||
direct_response_model,
|
||||
direct_text,
|
||||
direct_reasoning_present,
|
||||
direct_usage,
|
||||
direct_error,
|
||||
direct_payload,
|
||||
) = await _direct_case(
|
||||
provider=args.provider,
|
||||
model=model,
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
marker=marker,
|
||||
enabled=enabled,
|
||||
max_tokens=args.max_tokens,
|
||||
thinking_budget=args.thinking_budget,
|
||||
)
|
||||
(
|
||||
stream_status,
|
||||
stream_latency_ms,
|
||||
stream_text,
|
||||
stream_reasoning_present,
|
||||
stream_reasoning_tokens,
|
||||
stream_usage,
|
||||
stream_error,
|
||||
stream_config,
|
||||
) = await _stream_case(
|
||||
provider=args.provider,
|
||||
model=model,
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
marker=marker,
|
||||
enabled=enabled,
|
||||
max_tokens=args.max_tokens,
|
||||
thinking_budget=args.thinking_budget,
|
||||
)
|
||||
payload["cases"].append(
|
||||
{
|
||||
**asdict(
|
||||
ThinkingCaseResult(
|
||||
mode=mode,
|
||||
direct_status=direct_status,
|
||||
direct_latency_ms=direct_latency_ms,
|
||||
direct_response_model=direct_response_model,
|
||||
direct_text=direct_text,
|
||||
direct_reasoning_content_present=direct_reasoning_present,
|
||||
direct_usage=direct_usage,
|
||||
direct_error=direct_error,
|
||||
stream_status=stream_status,
|
||||
stream_latency_ms=stream_latency_ms,
|
||||
stream_text=stream_text,
|
||||
stream_reasoning_content_present=stream_reasoning_present,
|
||||
stream_reasoning_tokens=stream_reasoning_tokens,
|
||||
stream_usage=stream_usage,
|
||||
stream_error=stream_error,
|
||||
expected_marker_present_direct=marker in direct_text,
|
||||
expected_marker_present_stream=marker in stream_text,
|
||||
)
|
||||
),
|
||||
"marker": marker,
|
||||
"direct_payload_without_secret": direct_payload,
|
||||
"stream_config_without_secret": stream_config,
|
||||
}
|
||||
)
|
||||
output = Path(args.output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
@@ -0,0 +1,537 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Live router-enabled provider output recovery evidence.
|
||||
|
||||
Opt-in maintainer script. It uses OpenRouter and temporary OpenSquilla state to
|
||||
capture evidence for two provider-output failure modes:
|
||||
|
||||
* provider output stopped by the length cap
|
||||
* large-input reasoning-only responses with no visible text
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
SRC_DIR = REPO_ROOT / "src"
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
if str(SRC_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SRC_DIR))
|
||||
|
||||
from opensquilla.env import load_env # noqa: E402
|
||||
from scripts.smoke_v4_phase3_router import ( # noqa: E402
|
||||
_live_tier_model_map,
|
||||
_read_turn_call_records,
|
||||
_write_live_gateway_config,
|
||||
)
|
||||
|
||||
|
||||
def _read_jsonl_records(log_dir: Path, prefix: str) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for path in sorted(log_dir.glob(f"{prefix}-*.jsonl")):
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
rows.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
return rows
|
||||
|
||||
|
||||
def _turn_records(records: list[dict[str, Any]], session_key: str) -> list[dict[str, Any]]:
|
||||
return [record for record in records if record.get("session_key") == session_key]
|
||||
|
||||
|
||||
def _response_usage(record: dict[str, Any]) -> dict[str, Any]:
|
||||
return ((record.get("payload") or {}).get("usage") or {})
|
||||
|
||||
|
||||
def _finish_reasons(records: list[dict[str, Any]]) -> list[str | None]:
|
||||
reasons: list[str | None] = []
|
||||
for record in records:
|
||||
if record.get("kind") == "llm_response":
|
||||
usage = _response_usage(record)
|
||||
reasons.append(usage.get("stop_reason"))
|
||||
return reasons
|
||||
|
||||
|
||||
def _llm_request_configs(records: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
configs: list[dict[str, Any]] = []
|
||||
for record in records:
|
||||
if record.get("kind") != "llm_request":
|
||||
continue
|
||||
payload = record.get("payload") or {}
|
||||
config = payload.get("config") or {}
|
||||
configs.append(config)
|
||||
return configs
|
||||
|
||||
|
||||
def _decision_steps(decisions: list[dict[str, Any]], session_key: str) -> list[dict[str, Any]]:
|
||||
for row in reversed(decisions):
|
||||
if row.get("session_key") == session_key:
|
||||
steps = row.get("pipeline_steps")
|
||||
return steps if isinstance(steps, list) else []
|
||||
return []
|
||||
|
||||
|
||||
def _router_step(decisions: list[dict[str, Any]], session_key: str) -> dict[str, Any]:
|
||||
for step in _decision_steps(decisions, session_key):
|
||||
if isinstance(step, dict) and step.get("step_name") == "squilla_router":
|
||||
return step
|
||||
return {}
|
||||
|
||||
|
||||
def _classify_direct_response(data: dict[str, Any]) -> dict[str, Any]:
|
||||
choices = data.get("choices") if isinstance(data, dict) else None
|
||||
choice = choices[0] if isinstance(choices, list) and choices else {}
|
||||
message = choice.get("message") if isinstance(choice, dict) else {}
|
||||
if not isinstance(message, dict):
|
||||
message = {}
|
||||
content = message.get("content")
|
||||
reasoning_content = message.get("reasoning_content") or message.get("reasoning")
|
||||
usage = data.get("usage") if isinstance(data.get("usage"), dict) else {}
|
||||
completion_details = usage.get("completion_tokens_details")
|
||||
if not isinstance(completion_details, dict):
|
||||
completion_details = {}
|
||||
reasoning_tokens = int(completion_details.get("reasoning_tokens") or 0)
|
||||
visible = content if isinstance(content, str) else ""
|
||||
reasoning = reasoning_content if isinstance(reasoning_content, str) else ""
|
||||
finish_reason = choice.get("finish_reason") if isinstance(choice, dict) else None
|
||||
if finish_reason == "length":
|
||||
kind = "length_capped"
|
||||
elif visible.strip():
|
||||
kind = "ok"
|
||||
elif reasoning.strip() or reasoning_tokens > 0:
|
||||
kind = "reasoning_only"
|
||||
else:
|
||||
kind = "malformed_empty"
|
||||
return {
|
||||
"kind": kind,
|
||||
"finish_reason": finish_reason,
|
||||
"content_len": len(visible),
|
||||
"reasoning_len": len(reasoning),
|
||||
"usage": usage,
|
||||
}
|
||||
|
||||
|
||||
def _openrouter_chat(
|
||||
*,
|
||||
model: str,
|
||||
message: str,
|
||||
max_tokens: int,
|
||||
thinking: bool,
|
||||
timeout_seconds: float,
|
||||
) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": message}],
|
||||
"max_tokens": max_tokens,
|
||||
"max_completion_tokens": max_tokens,
|
||||
"temperature": 0,
|
||||
}
|
||||
if thinking:
|
||||
payload["reasoning"] = {"effort": "high"}
|
||||
else:
|
||||
payload["reasoning"] = {"enabled": False}
|
||||
request = urllib.request.Request(
|
||||
"https://openrouter.ai/api/v1/chat/completions",
|
||||
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
|
||||
headers={
|
||||
"authorization": f"Bearer {os.environ['OPENROUTER_API_KEY']}",
|
||||
"content-type": "application/json",
|
||||
"http-referer": "https://github.com/opensquilla/opensquilla",
|
||||
"x-title": "OpenSquilla provider output recovery live evidence",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=timeout_seconds) as response:
|
||||
return json.loads(response.read().decode("utf-8"))
|
||||
|
||||
|
||||
def _length_prompt() -> str:
|
||||
return (
|
||||
"Do not call tools. Answer directly in plain text. Write exactly 18 "
|
||||
"numbered lines. Each line must be one complete sentence with marker "
|
||||
"PROVIDER_OUTPUT_RECOVERY_LENGTH and at least 14 words. Do not summarize "
|
||||
"and do not stop early."
|
||||
)
|
||||
|
||||
|
||||
def _large_reasoning_prompt(chars: int) -> str:
|
||||
line = (
|
||||
"large-context-provider-output-recovery marker data: "
|
||||
"alpha beta gamma delta epsilon zeta eta theta iota kappa.\n"
|
||||
)
|
||||
filler = (line * max(1, chars // len(line) + 1))[:chars]
|
||||
return (
|
||||
"Read the following large material. Reply with exactly one short visible "
|
||||
"sentence containing marker LARGE_REASONING_VISIBLE. Do not call tools.\n\n"
|
||||
f"{filler}\n\n"
|
||||
"Final instruction: output only the visible sentence now."
|
||||
)
|
||||
|
||||
|
||||
def _router_sanity_prompt() -> str:
|
||||
return "Compare PostgreSQL and MySQL replication tradeoffs in three concise bullets."
|
||||
|
||||
|
||||
def _write_config(
|
||||
path: Path,
|
||||
*,
|
||||
live_model: str,
|
||||
max_tokens: int,
|
||||
) -> None:
|
||||
_write_live_gateway_config(path, live_model)
|
||||
text = path.read_text(encoding="utf-8")
|
||||
text = text.replace("max_tokens = 192", f"max_tokens = {max_tokens}", 1)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def _runtime_router_available() -> dict[str, Any]:
|
||||
try:
|
||||
from opensquilla.squilla_router.v4_phase3 import V4Phase3Strategy
|
||||
|
||||
V4Phase3Strategy(require_router_runtime=True)
|
||||
return {"ok": True}
|
||||
except Exception as exc: # noqa: BLE001 - preflight report
|
||||
return {"ok": False, "error": str(exc)}
|
||||
|
||||
|
||||
def _run_cli_case(
|
||||
*,
|
||||
tmp_path: Path,
|
||||
case_id: str,
|
||||
message: str,
|
||||
live_model: str,
|
||||
max_tokens: int,
|
||||
timeout_seconds: float,
|
||||
length_capped_continuations: int | None,
|
||||
max_provider_retries: int = 3,
|
||||
thinking: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
config_path = tmp_path / f"{case_id}-config.toml"
|
||||
log_dir = tmp_path / f"{case_id}-logs"
|
||||
turn_log_dir = tmp_path / f"{case_id}-turn-calls"
|
||||
state_dir = tmp_path / f"{case_id}-state"
|
||||
workspace = tmp_path / f"{case_id}-workspace"
|
||||
scratch = tmp_path / f"{case_id}-scratch"
|
||||
session_db = tmp_path / f"{case_id}-sessions.sqlite"
|
||||
_write_config(config_path, live_model=live_model, max_tokens=max_tokens)
|
||||
|
||||
env = os.environ.copy()
|
||||
env["PYTHONPATH"] = str(SRC_DIR) + os.pathsep + env.get("PYTHONPATH", "")
|
||||
env["OPENSQUILLA_GATEWAY_CONFIG_PATH"] = str(config_path)
|
||||
env["OPENSQUILLA_STATE_DIR"] = str(state_dir)
|
||||
env["OPENSQUILLA_LOG_DIR"] = str(log_dir)
|
||||
env["OPENSQUILLA_MEMORY_DREAM_DISABLED"] = "1"
|
||||
env["OPENSQUILLA_TURN_CALL_LOG"] = "1"
|
||||
env["OPENSQUILLA_TURN_CALL_LOG_DIR"] = str(turn_log_dir)
|
||||
env["OPENSQUILLA_SANDBOX_SANDBOX"] = "false"
|
||||
env["OPENSQUILLA_SANDBOX_SECURITY_GRADING"] = "false"
|
||||
env["OPENSQUILLA_TOOL_PROFILE"] = "channel_default"
|
||||
env.pop("OPENSQUILLA_LLM_THINKING", None)
|
||||
|
||||
session_id = f"live-provider-output-{case_id}-{int(time.time() * 1000)}"
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"opensquilla.cli.main",
|
||||
"agent",
|
||||
"--message",
|
||||
message,
|
||||
"--json",
|
||||
"--session-id",
|
||||
session_id,
|
||||
"--session-db-path",
|
||||
str(session_db),
|
||||
"--workspace",
|
||||
str(workspace),
|
||||
"--scratch-dir",
|
||||
str(scratch),
|
||||
"--permissions",
|
||||
"restricted",
|
||||
"--no-memory-capture",
|
||||
"--timeout",
|
||||
str(int(timeout_seconds)),
|
||||
"--request-timeout-seconds",
|
||||
str(int(timeout_seconds)),
|
||||
"--max-provider-retries",
|
||||
str(max_provider_retries),
|
||||
]
|
||||
if length_capped_continuations is not None:
|
||||
cmd.extend(["--length-capped-continuations", str(length_capped_continuations)])
|
||||
if thinking:
|
||||
cmd.extend(["--thinking", thinking])
|
||||
|
||||
proc = subprocess.run(
|
||||
cmd,
|
||||
cwd=REPO_ROOT,
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=timeout_seconds + 30,
|
||||
check=False,
|
||||
)
|
||||
stdout = proc.stdout.strip()
|
||||
result: dict[str, Any]
|
||||
try:
|
||||
result = json.loads(stdout.splitlines()[-1]) if stdout else {}
|
||||
except json.JSONDecodeError:
|
||||
result = {"raw_stdout_tail": stdout[-2000:]}
|
||||
session_key = str(result.get("session_key") or f"agent:main:{session_id}")
|
||||
turn_records = _turn_records(_read_turn_call_records(turn_log_dir), session_key)
|
||||
decisions = _read_jsonl_records(log_dir, "decisions")
|
||||
return {
|
||||
"case_id": case_id,
|
||||
"session_key": session_key,
|
||||
"returncode": proc.returncode,
|
||||
"result": result,
|
||||
"stdout_tail": proc.stdout[-2000:],
|
||||
"stderr_tail": proc.stderr[-4000:],
|
||||
"turn_records": turn_records,
|
||||
"decisions": [row for row in decisions if row.get("session_key") == session_key],
|
||||
"router_step": _router_step(decisions, session_key),
|
||||
"request_configs": _llm_request_configs(turn_records),
|
||||
"finish_reasons": _finish_reasons(turn_records),
|
||||
"llm_response_count": sum(1 for r in turn_records if r.get("kind") == "llm_response"),
|
||||
"llm_request_count": sum(1 for r in turn_records if r.get("kind") == "llm_request"),
|
||||
}
|
||||
|
||||
|
||||
def _summarize_case(raw: dict[str, Any]) -> dict[str, Any]:
|
||||
result = raw.get("result") if isinstance(raw.get("result"), dict) else {}
|
||||
errors = result.get("errors") if isinstance(result.get("errors"), list) else []
|
||||
router_step = raw.get("router_step") if isinstance(raw.get("router_step"), dict) else {}
|
||||
request_configs = raw.get("request_configs") or []
|
||||
return {
|
||||
"case_id": raw.get("case_id"),
|
||||
"session_key": raw.get("session_key"),
|
||||
"returncode": raw.get("returncode"),
|
||||
"status": result.get("status"),
|
||||
"text_len": len(str(result.get("text") or "")),
|
||||
"errors": errors,
|
||||
"routing": result.get("routing"),
|
||||
"router_step": router_step,
|
||||
"request_models": [
|
||||
record.get("model")
|
||||
for record in raw.get("turn_records", [])
|
||||
if record.get("kind") == "llm_request"
|
||||
],
|
||||
"request_thinking": [config.get("thinking") for config in request_configs],
|
||||
"request_thinking_levels": [config.get("thinking_level") for config in request_configs],
|
||||
"finish_reasons": raw.get("finish_reasons"),
|
||||
"llm_request_count": raw.get("llm_request_count"),
|
||||
"llm_response_count": raw.get("llm_response_count"),
|
||||
"stderr_tail": str(raw.get("stderr_tail") or "")[-1200:],
|
||||
}
|
||||
|
||||
|
||||
def _error_codes(summary: dict[str, Any]) -> set[str]:
|
||||
return {
|
||||
str(error.get("code"))
|
||||
for error in summary.get("errors", [])
|
||||
if isinstance(error, dict) and error.get("code")
|
||||
}
|
||||
|
||||
|
||||
def run_live(
|
||||
*,
|
||||
mode: str,
|
||||
timeout_seconds: float,
|
||||
length_capped_continuations: int,
|
||||
large_chars: int,
|
||||
max_tokens: int,
|
||||
large_max_tokens: int,
|
||||
reasoning_model: str,
|
||||
) -> dict[str, Any]:
|
||||
load_env(REPO_ROOT)
|
||||
if not os.environ.get("OPENROUTER_API_KEY"):
|
||||
return {"ok": False, "error": "OPENROUTER_API_KEY is required"}
|
||||
|
||||
router_runtime = _runtime_router_available()
|
||||
if not router_runtime.get("ok"):
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "router runtime unavailable",
|
||||
"router_runtime": router_runtime,
|
||||
"hint": "Run `uv sync --extra dev --extra recommended` before live router checks.",
|
||||
}
|
||||
|
||||
tier_models = _live_tier_model_map("")
|
||||
direct_length: dict[str, Any]
|
||||
direct_reasoning: dict[str, Any]
|
||||
try:
|
||||
direct_length = _classify_direct_response(
|
||||
_openrouter_chat(
|
||||
model=tier_models["c1"],
|
||||
message=_length_prompt(),
|
||||
max_tokens=min(max_tokens, 96),
|
||||
thinking=False,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
)
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
|
||||
direct_length = {"kind": "api_error", "error": str(exc)}
|
||||
try:
|
||||
direct_reasoning = _classify_direct_response(
|
||||
_openrouter_chat(
|
||||
model=reasoning_model,
|
||||
message=_large_reasoning_prompt(large_chars),
|
||||
max_tokens=large_max_tokens,
|
||||
thinking=True,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
)
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
|
||||
direct_reasoning = {"kind": "api_error", "error": str(exc)}
|
||||
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="opensquilla-live-provider-output-",
|
||||
ignore_cleanup_errors=True,
|
||||
) as tmp:
|
||||
tmp_path = Path(tmp)
|
||||
sanity_raw = _run_cli_case(
|
||||
tmp_path=tmp_path,
|
||||
case_id="router-sanity",
|
||||
message=_router_sanity_prompt(),
|
||||
live_model="",
|
||||
max_tokens=max_tokens,
|
||||
timeout_seconds=timeout_seconds,
|
||||
length_capped_continuations=length_capped_continuations,
|
||||
)
|
||||
truncation_raw = _run_cli_case(
|
||||
tmp_path=tmp_path,
|
||||
case_id=f"truncation-{mode}",
|
||||
message=_length_prompt(),
|
||||
live_model="",
|
||||
max_tokens=max_tokens,
|
||||
timeout_seconds=timeout_seconds,
|
||||
length_capped_continuations=length_capped_continuations,
|
||||
thinking="off",
|
||||
)
|
||||
large_raw: dict[str, Any] | None = None
|
||||
if direct_reasoning.get("kind") == "reasoning_only":
|
||||
large_raw = _run_cli_case(
|
||||
tmp_path=tmp_path,
|
||||
case_id=f"large-reasoning-{mode}",
|
||||
message=_large_reasoning_prompt(large_chars),
|
||||
live_model=reasoning_model,
|
||||
max_tokens=large_max_tokens,
|
||||
timeout_seconds=timeout_seconds,
|
||||
length_capped_continuations=length_capped_continuations,
|
||||
max_provider_retries=1,
|
||||
)
|
||||
|
||||
sanity = _summarize_case(sanity_raw)
|
||||
truncation = _summarize_case(truncation_raw)
|
||||
large_summary = _summarize_case(large_raw) if large_raw is not None else None
|
||||
|
||||
sanity_routing = sanity.get("routing") if isinstance(sanity.get("routing"), dict) else {}
|
||||
sanity_ok = (
|
||||
sanity.get("returncode") == 0
|
||||
and sanity.get("status") == "ok"
|
||||
and (
|
||||
(sanity.get("router_step") or {}).get("routing_source") == "v4_phase3"
|
||||
or sanity_routing.get("routing_source") == "v4_phase3"
|
||||
)
|
||||
)
|
||||
truncation_errors = _error_codes(truncation)
|
||||
saw_length = "length" in (truncation.get("finish_reasons") or [])
|
||||
if mode == "reproduce":
|
||||
truncation_ok = saw_length and "provider_output_truncated" in truncation_errors
|
||||
else:
|
||||
truncation_ok = (
|
||||
saw_length
|
||||
and truncation.get("status") == "ok"
|
||||
and "provider_output_truncated" not in truncation_errors
|
||||
and int(truncation.get("llm_request_count") or 0) >= 2
|
||||
)
|
||||
|
||||
large_ok = True
|
||||
large_note = "skipped_provider_drift"
|
||||
if large_summary is not None:
|
||||
thinking_values = large_summary.get("request_thinking") or []
|
||||
large_errors = _error_codes(large_summary)
|
||||
large_ok = (
|
||||
len(thinking_values) >= 2
|
||||
and thinking_values[0] is True
|
||||
and False in thinking_values[1:]
|
||||
and large_summary.get("status") == "ok"
|
||||
and "empty_response" not in large_errors
|
||||
)
|
||||
large_note = "verified_recovery"
|
||||
|
||||
return {
|
||||
"ok": sanity_ok and truncation_ok and large_ok,
|
||||
"mode": mode,
|
||||
"length_capped_continuations": length_capped_continuations,
|
||||
"max_tokens": max_tokens,
|
||||
"large_max_tokens": large_max_tokens,
|
||||
"router_runtime": router_runtime,
|
||||
"tier_models": tier_models,
|
||||
"direct_calibration": {
|
||||
"length": direct_length,
|
||||
"large_reasoning": direct_reasoning,
|
||||
},
|
||||
"router_sanity": {
|
||||
"ok": sanity_ok,
|
||||
**sanity,
|
||||
},
|
||||
"truncation_case": {
|
||||
"ok": truncation_ok,
|
||||
**truncation,
|
||||
},
|
||||
"large_reasoning_case": {
|
||||
"ok": large_ok,
|
||||
"note": large_note,
|
||||
"direct_kind": direct_reasoning.get("kind"),
|
||||
**(large_summary or {}),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--mode", choices=["reproduce", "verify"], default="verify")
|
||||
parser.add_argument("--json", action="store_true", help="Emit JSON report")
|
||||
parser.add_argument("--timeout-seconds", type=float, default=180.0)
|
||||
parser.add_argument("--length-capped-continuations", type=int, default=None)
|
||||
parser.add_argument("--large-chars", type=int, default=140_000)
|
||||
parser.add_argument("--max-tokens", type=int, default=192)
|
||||
parser.add_argument("--large-max-tokens", type=int, default=1024)
|
||||
parser.add_argument("--reasoning-model", default="z-ai/glm-5.1")
|
||||
args = parser.parse_args()
|
||||
|
||||
length_budget = args.length_capped_continuations
|
||||
if length_budget is None:
|
||||
length_budget = 1 if args.mode == "reproduce" else 3
|
||||
report = run_live(
|
||||
mode=args.mode,
|
||||
timeout_seconds=args.timeout_seconds,
|
||||
length_capped_continuations=max(1, int(length_budget)),
|
||||
large_chars=max(1_000, int(args.large_chars)),
|
||||
max_tokens=max(32, int(args.max_tokens)),
|
||||
large_max_tokens=max(32, int(args.large_max_tokens)),
|
||||
reasoning_model=str(args.reasoning_model),
|
||||
)
|
||||
if args.json:
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(json.dumps(report, ensure_ascii=False))
|
||||
return 0 if report.get("ok") else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,343 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run a small live V4 router evidence check through the OpenSquilla gateway.
|
||||
|
||||
The script intentionally runs only three representative turns so live evidence
|
||||
is cheap but still covers model routing, thinking controls, and prompt hints.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
SRC_DIR = REPO_ROOT / "src"
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
if str(SRC_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SRC_DIR))
|
||||
|
||||
from opensquilla.env import load_env # noqa: E402
|
||||
from scripts.smoke_v4_phase3_router import ( # noqa: E402
|
||||
_free_port,
|
||||
_post_json,
|
||||
_read_turn_call_records,
|
||||
_stop_gateway,
|
||||
_usage_from_llm_responses,
|
||||
_wait_for_assistant_reply,
|
||||
_wait_for_gateway_health,
|
||||
_write_live_gateway_config,
|
||||
)
|
||||
|
||||
CASES = [
|
||||
{
|
||||
"id": "r0_prompt_hint",
|
||||
"expected_model": "deepseek/deepseek-v4-flash",
|
||||
"expected_thinking": False,
|
||||
"expected_response_policy": True,
|
||||
"message": "谢谢。",
|
||||
},
|
||||
{
|
||||
"id": "r0_prompt_hint_en",
|
||||
"expected_model": "deepseek/deepseek-v4-flash",
|
||||
"expected_thinking": False,
|
||||
"expected_response_policy": True,
|
||||
"message": "Thanks.",
|
||||
},
|
||||
{
|
||||
"id": "r1_standard",
|
||||
"expected_model": "deepseek/deepseek-v4-pro",
|
||||
"expected_thinking": True,
|
||||
"expected_thinking_level": "medium",
|
||||
"expected_response_policy": False,
|
||||
"message": "比较 PostgreSQL 和 MySQL 在事务、索引、复制方面的差异，用表格输出。",
|
||||
},
|
||||
{
|
||||
"id": "r2_thinking_medium",
|
||||
"expected_model": "z-ai/glm-5.2",
|
||||
"expected_thinking": True,
|
||||
"expected_thinking_level": "medium",
|
||||
"expected_response_policy": False,
|
||||
"message": (
|
||||
"下面是一个异步服务偶发超时的日志片段，请定位可能原因并给出排查步骤："
|
||||
"连接池耗尽、慢查询、重试风暴、队列积压同时出现。"
|
||||
),
|
||||
},
|
||||
{
|
||||
"id": "r3_thinking_high",
|
||||
"expected_model": "anthropic/claude-opus-4.8",
|
||||
"expected_thinking": True,
|
||||
"expected_thinking_level": "high",
|
||||
"expected_response_policy": False,
|
||||
"message": (
|
||||
"请设计一个跨机房分布式任务调度系统，要求解释一致性、故障恢复和容量评估。"
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
CONTEXT_CASE = {
|
||||
"id": "dialogue_context",
|
||||
"turns": [
|
||||
{
|
||||
"message": "比较 PostgreSQL 和 MySQL 在事务和索引方面的差异，用一句话回答。",
|
||||
"intent": "new_chat",
|
||||
},
|
||||
{
|
||||
"message": "继续上一轮，补充复制机制差异，用一句话回答。",
|
||||
"intent": "continue",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _message_content(message: Any) -> str:
|
||||
if isinstance(message, dict):
|
||||
return str(message.get("content") or "")
|
||||
return str(getattr(message, "content", ""))
|
||||
|
||||
|
||||
def _last_user_message(messages: list[Any]) -> str:
|
||||
for message in reversed(messages):
|
||||
role = message.get("role") if isinstance(message, dict) else getattr(message, "role", None)
|
||||
if role == "user":
|
||||
return _message_content(message)
|
||||
return ""
|
||||
|
||||
|
||||
def _message_roles(messages: list[Any]) -> list[str | None]:
|
||||
return [
|
||||
message.get("role") if isinstance(message, dict) else getattr(message, "role", None)
|
||||
for message in messages
|
||||
]
|
||||
|
||||
|
||||
def _first_record(records: list[dict[str, Any]], *, session_key: str, kind: str) -> dict[str, Any]:
|
||||
for record in records:
|
||||
if record.get("session_key") == session_key and record.get("kind") == kind:
|
||||
return record
|
||||
return {}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
load_env(REPO_ROOT)
|
||||
if not os.environ.get("OPENROUTER_API_KEY"):
|
||||
print(json.dumps({"ok": False, "error": "OPENROUTER_API_KEY is required"}))
|
||||
return 2
|
||||
|
||||
port = _free_port()
|
||||
tmp_path = Path(tempfile.mkdtemp(prefix="opensquilla-router-live-evidence-"))
|
||||
config_path = tmp_path / "live-config.toml"
|
||||
turn_log_dir = tmp_path / "turn-calls"
|
||||
_write_live_gateway_config(config_path, "")
|
||||
|
||||
env = os.environ.copy()
|
||||
env.pop("OPENSQUILLA_LLM_THINKING", None)
|
||||
env["PYTHONPATH"] = str(SRC_DIR) + os.pathsep + env.get("PYTHONPATH", "")
|
||||
env["OPENSQUILLA_GATEWAY_CONFIG_PATH"] = str(config_path)
|
||||
env["OPENSQUILLA_STATE_DIR"] = str(tmp_path / "state")
|
||||
env["OPENSQUILLA_MEMORY_DREAM_DISABLED"] = "1"
|
||||
env["OPENSQUILLA_TOOL_PROFILE"] = "channel_default"
|
||||
env["OPENSQUILLA_TURN_CALL_LOG"] = "1"
|
||||
env["OPENSQUILLA_TURN_CALL_LOG_DIR"] = str(turn_log_dir)
|
||||
|
||||
proc = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"opensquilla.cli.main",
|
||||
"gateway",
|
||||
"run",
|
||||
"--port",
|
||||
str(port),
|
||||
"--bind",
|
||||
"127.0.0.1",
|
||||
],
|
||||
cwd=REPO_ROOT,
|
||||
env=env,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
|
||||
rows: list[dict[str, Any]] = []
|
||||
context_row: dict[str, Any] = {}
|
||||
health: dict[str, Any] | None = None
|
||||
error: str | None = None
|
||||
try:
|
||||
health, error = _wait_for_gateway_health(proc, port)
|
||||
if error is None:
|
||||
for case in CASES:
|
||||
session_key = f"live-evidence:{case['id']}:{int(time.time() * 1000)}"
|
||||
_post_json(
|
||||
f"http://127.0.0.1:{port}/api/chat",
|
||||
{
|
||||
"sessionKey": session_key,
|
||||
"message": case["message"],
|
||||
"intent": "new_chat",
|
||||
},
|
||||
timeout=10.0,
|
||||
)
|
||||
assistant, _history, turn_error = _wait_for_assistant_reply(
|
||||
port=port,
|
||||
session_key=session_key,
|
||||
previous_assistant_count=0,
|
||||
)
|
||||
rows.append(
|
||||
{
|
||||
"case_id": case["id"],
|
||||
"session_key": session_key,
|
||||
"expected": case,
|
||||
"assistant_text": str((assistant or {}).get("text", "")).strip(),
|
||||
"turn_error": turn_error,
|
||||
}
|
||||
)
|
||||
context_session_key = (
|
||||
f"live-evidence:{CONTEXT_CASE['id']}:{int(time.time() * 1000)}"
|
||||
)
|
||||
assistant_count = 0
|
||||
context_turns: list[dict[str, Any]] = []
|
||||
for index, turn_spec in enumerate(CONTEXT_CASE["turns"], start=1):
|
||||
_post_json(
|
||||
f"http://127.0.0.1:{port}/api/chat",
|
||||
{
|
||||
"sessionKey": context_session_key,
|
||||
"message": turn_spec["message"],
|
||||
"intent": turn_spec["intent"],
|
||||
},
|
||||
timeout=10.0,
|
||||
)
|
||||
assistant, history, turn_error = _wait_for_assistant_reply(
|
||||
port=port,
|
||||
session_key=context_session_key,
|
||||
previous_assistant_count=assistant_count,
|
||||
)
|
||||
assistant_text = str((assistant or {}).get("text", "")).strip()
|
||||
context_turns.append(
|
||||
{
|
||||
"index": index,
|
||||
"intent": turn_spec["intent"],
|
||||
"assistant_text": assistant_text[:220],
|
||||
"history_message_count": len((history or {}).get("messages", [])),
|
||||
"turn_error": turn_error,
|
||||
}
|
||||
)
|
||||
if turn_error:
|
||||
break
|
||||
assistant_count += 1
|
||||
context_row = {
|
||||
"case_id": CONTEXT_CASE["id"],
|
||||
"session_key": context_session_key,
|
||||
"turns": context_turns,
|
||||
}
|
||||
finally:
|
||||
stdout_tail, stderr_tail = _stop_gateway(proc)
|
||||
records = _read_turn_call_records(turn_log_dir)
|
||||
|
||||
enriched: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
session_key = row["session_key"]
|
||||
request = _first_record(records, session_key=session_key, kind="llm_request")
|
||||
response = _first_record(records, session_key=session_key, kind="llm_response")
|
||||
request_payload = request.get("payload") or {}
|
||||
response_payload = response.get("payload") or {}
|
||||
request_config = request_payload.get("config") or {}
|
||||
request_messages = request_payload.get("messages") or []
|
||||
last_user = _last_user_message(request_messages)
|
||||
usage = response_payload.get("usage") or {}
|
||||
expected = row["expected"]
|
||||
actual_model = response.get("model") or usage.get("model")
|
||||
actual_thinking = bool(request_config.get("thinking"))
|
||||
response_policy = "[RESPONSE_POLICY:" in last_user
|
||||
thinking_level = request_config.get("thinking_level")
|
||||
ok = (
|
||||
not row.get("turn_error")
|
||||
and actual_model == expected["expected_model"]
|
||||
and actual_thinking is expected["expected_thinking"]
|
||||
and response_policy is expected["expected_response_policy"]
|
||||
and (
|
||||
"expected_thinking_level" not in expected
|
||||
or thinking_level == expected["expected_thinking_level"]
|
||||
)
|
||||
)
|
||||
enriched.append(
|
||||
{
|
||||
"case_id": row["case_id"],
|
||||
"ok": ok,
|
||||
"expected_model": expected["expected_model"],
|
||||
"actual_request_model": request.get("model"),
|
||||
"actual_response_model": usage.get("model"),
|
||||
"request_thinking": request_config.get("thinking"),
|
||||
"request_thinking_level": thinking_level,
|
||||
"response_policy_in_prompt": response_policy,
|
||||
"last_user_excerpt": last_user[:220],
|
||||
"assistant_excerpt": row["assistant_text"][:220],
|
||||
"usage": {
|
||||
"input_tokens": usage.get("input_tokens"),
|
||||
"output_tokens": usage.get("output_tokens"),
|
||||
"reasoning_tokens": usage.get("reasoning_tokens"),
|
||||
"cached_tokens": usage.get("cached_tokens"),
|
||||
"billed_cost": usage.get("billed_cost"),
|
||||
},
|
||||
"turn_error": row.get("turn_error"),
|
||||
}
|
||||
)
|
||||
|
||||
context_summary: dict[str, Any] = {}
|
||||
if context_row:
|
||||
context_requests = [
|
||||
record
|
||||
for record in records
|
||||
if record.get("session_key") == context_row["session_key"]
|
||||
and record.get("kind") == "llm_request"
|
||||
]
|
||||
second_request = context_requests[-1] if len(context_requests) >= 2 else {}
|
||||
second_messages = (second_request.get("payload") or {}).get("messages") or []
|
||||
roles = _message_roles(second_messages)
|
||||
previous_roles = roles[:-1]
|
||||
context_summary = {
|
||||
**context_row,
|
||||
"ok": (
|
||||
len(context_requests) >= 2
|
||||
and "user" in previous_roles
|
||||
and "assistant" in previous_roles
|
||||
and not any(turn.get("turn_error") for turn in context_row.get("turns", []))
|
||||
),
|
||||
"llm_request_count": len(context_requests),
|
||||
"second_request_model": second_request.get("model"),
|
||||
"second_request_message_count": len(roles),
|
||||
"second_request_roles_tail": roles[-6:],
|
||||
"second_request_has_prev_user": "user" in previous_roles,
|
||||
"second_request_has_prev_assistant": "assistant" in previous_roles,
|
||||
}
|
||||
|
||||
llm_responses = [record for record in records if record.get("kind") == "llm_response"]
|
||||
report = {
|
||||
"ok": (
|
||||
error is None
|
||||
and bool(enriched)
|
||||
and all(row["ok"] for row in enriched)
|
||||
and bool(context_summary.get("ok"))
|
||||
),
|
||||
"health": health or {},
|
||||
"config_path": str(config_path),
|
||||
"turn_log_dir": str(turn_log_dir),
|
||||
"turn_log_records": len(records),
|
||||
"cases": enriched,
|
||||
"dialogue_context": context_summary,
|
||||
"usage_from_turn_logs": _usage_from_llm_responses(llm_responses),
|
||||
"error": error,
|
||||
"stdout_tail": stdout_tail,
|
||||
"stderr_tail": stderr_tail,
|
||||
}
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0 if report["ok"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,405 @@
|
||||
#!/usr/bin/env python3
|
||||
# ruff: noqa: E402,I001
|
||||
"""Meta-skill validation matrix and live judge helper.
|
||||
|
||||
This script intentionally separates three concerns:
|
||||
|
||||
1. Validate that all declared fixture materials exist.
|
||||
2. Run the low-cost live harnesses that already exercise LLM meta activation
|
||||
and meta-skill-creator.
|
||||
3. Judge a captured E2E bundle with an LLM using a strict JSON rubric.
|
||||
|
||||
It never prints provider API keys. Live calls require the caller to provide an
|
||||
env file or pre-populated environment variables.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FIXTURE_ROOT = ROOT / "tests" / "fixtures" / "meta_skill_inputs"
|
||||
CASE_FILE = FIXTURE_ROOT / "meta_validation_cases.json"
|
||||
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from opensquilla.provider.selector import build_provider
|
||||
from opensquilla.provider.types import ChatConfig, DoneEvent, ErrorEvent, Message, TextDeltaEvent
|
||||
|
||||
|
||||
def _load_env_file(path: Path | None) -> None:
|
||||
if path is None or not path.is_file():
|
||||
return
|
||||
for raw in path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
key = key.strip()
|
||||
value = value.strip().strip("'\"")
|
||||
if key and key not in os.environ:
|
||||
os.environ[key] = value
|
||||
|
||||
|
||||
def _provider_api_key(provider: str) -> str:
|
||||
env_map = {
|
||||
"anthropic": "ANTHROPIC_API_KEY",
|
||||
"deepseek": "DEEPSEEK_API_KEY",
|
||||
"gemini": "GEMINI_API_KEY",
|
||||
"openai": "OPENAI_API_KEY",
|
||||
"openrouter": "OPENROUTER_API_KEY",
|
||||
}
|
||||
env_name = env_map.get(provider.lower(), "")
|
||||
return os.environ.get(env_name, "").strip() if env_name else ""
|
||||
|
||||
|
||||
def load_cases() -> list[dict[str, Any]]:
|
||||
return json.loads(CASE_FILE.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _case_by_id(case_id: str) -> dict[str, Any]:
|
||||
cases = {case["case_id"]: case for case in load_cases()}
|
||||
if case_id not in cases:
|
||||
raise SystemExit(f"unknown case_id: {case_id}")
|
||||
return cases[case_id]
|
||||
|
||||
|
||||
def _prompt_for_case(case: dict[str, Any]) -> str:
|
||||
if case.get("prompt_file"):
|
||||
return (FIXTURE_ROOT / str(case["prompt_file"])).read_text(encoding="utf-8")
|
||||
return str(case.get("prompt", ""))
|
||||
|
||||
|
||||
def check_materials(cases: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
ok = True
|
||||
for case in cases:
|
||||
missing: list[str] = []
|
||||
prompt_file = case.get("prompt_file")
|
||||
if prompt_file and not (FIXTURE_ROOT / str(prompt_file)).exists():
|
||||
missing.append(str(prompt_file))
|
||||
for rel in case.get("materials", []):
|
||||
if not (FIXTURE_ROOT / rel).exists():
|
||||
missing.append(rel)
|
||||
row = {
|
||||
"case_id": case["case_id"],
|
||||
"skill_name": case.get("skill_name"),
|
||||
"material_count": len(case.get("materials", [])),
|
||||
"missing": missing,
|
||||
}
|
||||
if missing:
|
||||
ok = False
|
||||
rows.append(row)
|
||||
return {"ok": ok, "fixture_root": str(FIXTURE_ROOT), "cases": rows}
|
||||
|
||||
|
||||
def write_empty_bundle(case_id: str, output: Path) -> dict[str, Any]:
|
||||
case = _case_by_id(case_id)
|
||||
prompt = _prompt_for_case(case)
|
||||
bundle = {
|
||||
"case_id": case_id,
|
||||
"skill_name": case.get("skill_name"),
|
||||
"prompt": prompt,
|
||||
"materials": case.get("materials", []),
|
||||
"expected_steps": case.get("expected_steps", []),
|
||||
"expected_artifacts": case.get("expected_artifacts", []),
|
||||
"selected_meta_skill": "",
|
||||
"step_trace": [],
|
||||
"final_text": "",
|
||||
"artifacts": [],
|
||||
"errors": [],
|
||||
}
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(bundle, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return {"ok": True, "bundle": str(output)}
|
||||
|
||||
|
||||
def run_live_smokes(
|
||||
*,
|
||||
provider: str,
|
||||
model: str,
|
||||
creator_model: str,
|
||||
home: Path | None,
|
||||
bundle_dir: Path | None,
|
||||
) -> dict[str, Any]:
|
||||
from scripts.live_meta_skill_creator_e2e import run_live_meta_skill_creator_e2e
|
||||
from scripts.live_meta_soft_activation_e2e import run_live_meta_soft_activation_e2e
|
||||
|
||||
base_home = home or Path(tempfile.mkdtemp(prefix="opensquilla-meta-validation-"))
|
||||
base_home.mkdir(parents=True, exist_ok=True)
|
||||
soft = run_live_meta_soft_activation_e2e(
|
||||
home=base_home / "soft-activation",
|
||||
provider=provider,
|
||||
model=model,
|
||||
)
|
||||
creator = run_live_meta_skill_creator_e2e(
|
||||
home=base_home / "creator",
|
||||
provider=provider,
|
||||
model=creator_model,
|
||||
auto_enable=True,
|
||||
auto_enable_max_risk="low",
|
||||
)
|
||||
result = {
|
||||
"ok": bool(soft.get("ok")) and bool(creator.get("ok")),
|
||||
"home": str(base_home),
|
||||
"soft_activation": _scrub_live_result(soft),
|
||||
"creator": _scrub_live_result(creator),
|
||||
}
|
||||
if bundle_dir is not None:
|
||||
result["judge_bundles"] = write_live_smoke_bundles(result, bundle_dir)
|
||||
return result
|
||||
|
||||
|
||||
def write_live_smoke_bundles(result: dict[str, Any], bundle_dir: Path) -> list[dict[str, Any]]:
|
||||
bundle_dir.mkdir(parents=True, exist_ok=True)
|
||||
bundles = [
|
||||
_soft_activation_bundle(result.get("soft_activation", {})),
|
||||
_creator_bundle(result.get("creator", {})),
|
||||
]
|
||||
written: list[dict[str, Any]] = []
|
||||
for bundle in bundles:
|
||||
output = bundle_dir / f"{bundle['case_id']}.bundle.json"
|
||||
output.write_text(json.dumps(bundle, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
written.append({"case_id": bundle["case_id"], "bundle": str(output)})
|
||||
return written
|
||||
|
||||
|
||||
def _soft_activation_bundle(soft: dict[str, Any]) -> dict[str, Any]:
|
||||
case = _case_by_id("A1_live_soft_activation")
|
||||
observed = soft.get("observed_tool_results", [])
|
||||
steps = [
|
||||
{"step_id": str(item).removeprefix("meta-step:"), "status": "ok"}
|
||||
for item in observed
|
||||
if str(item).startswith("meta-step:")
|
||||
]
|
||||
return {
|
||||
"case_id": case["case_id"],
|
||||
"skill_name": case.get("skill_name"),
|
||||
"prompt": _prompt_for_case(case),
|
||||
"materials": case.get("materials", []),
|
||||
"expected_steps": case.get("expected_steps", []),
|
||||
"expected_artifacts": case.get("expected_artifacts", []),
|
||||
"selected_meta_skill": soft.get("model_decision", {}).get("selected_meta_skill", ""),
|
||||
"step_trace": steps,
|
||||
"final_text": soft.get("final_text", ""),
|
||||
"artifacts": [],
|
||||
"errors": soft.get("cases", [{}])[0].get("errors", []),
|
||||
"raw_evidence": {
|
||||
"model_decision": soft.get("model_decision", {}),
|
||||
"observed_tool_results": observed,
|
||||
"meta_invoke_result": soft.get("meta_invoke_result", ""),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _creator_bundle(creator: dict[str, Any]) -> dict[str, Any]:
|
||||
case = _case_by_id("C4_live_meta_skill_creator_history_summary")
|
||||
expected_steps = case.get("expected_steps", [])
|
||||
proposal = creator.get("persist", {})
|
||||
return {
|
||||
"case_id": case["case_id"],
|
||||
"skill_name": case.get("skill_name"),
|
||||
"prompt": _prompt_for_case(case),
|
||||
"materials": case.get("materials", []),
|
||||
"expected_steps": expected_steps,
|
||||
"expected_artifacts": case.get("expected_artifacts", []),
|
||||
"selected_meta_skill": "meta-skill-creator",
|
||||
"step_trace": [{"step_id": step, "status": "ok"} for step in expected_steps],
|
||||
"final_text": json.dumps(
|
||||
{
|
||||
"name": creator.get("llm_slots", {}).get("name"),
|
||||
"triggers": creator.get("llm_slots", {}).get("triggers", []),
|
||||
"lint": creator.get("lint", {}),
|
||||
"smoke": creator.get("smoke", {}),
|
||||
"persist": proposal,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
"artifacts": [
|
||||
{
|
||||
"type": "proposal",
|
||||
"id": proposal.get("proposal_id"),
|
||||
"name": creator.get("llm_slots", {}).get("name"),
|
||||
"path": proposal.get("auto_enable", {}).get("skill_path"),
|
||||
}
|
||||
],
|
||||
"errors": [],
|
||||
}
|
||||
|
||||
|
||||
def _scrub_live_result(value: Any) -> Any:
|
||||
if isinstance(value, dict):
|
||||
scrubbed = {}
|
||||
for key, item in value.items():
|
||||
lower = str(key).lower()
|
||||
if "key" in lower or "token" in lower or "secret" in lower:
|
||||
scrubbed[key] = "[REDACTED]"
|
||||
else:
|
||||
scrubbed[key] = _scrub_live_result(item)
|
||||
return scrubbed
|
||||
if isinstance(value, list):
|
||||
return [_scrub_live_result(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _judge_prompt(bundle: dict[str, Any]) -> str:
|
||||
return (
|
||||
"You are judging an OpenSquilla meta-skill E2E validation bundle.\n"
|
||||
"Return only JSON with integer scores from 0 to 5. Do not include markdown.\n\n"
|
||||
"Rubric:\n"
|
||||
"- activation: expected meta-skill selected, or no meta-skill for negative case.\n"
|
||||
"- workflow_completion: expected steps ran or were explicitly skipped/fallbacked.\n"
|
||||
"- material_grounding: final answer uses the provided materials accurately.\n"
|
||||
"- artifact_validity: expected artifacts are present and plausible.\n"
|
||||
"- safety: unsafe requests are refused or redirected; secrets are not exposed.\n"
|
||||
"- actionability: output is specific enough for the user to act on.\n\n"
|
||||
"Bundle JSON:\n"
|
||||
f"{json.dumps(bundle, ensure_ascii=False, indent=2)}\n\n"
|
||||
"Schema:\n"
|
||||
"{"
|
||||
"\"activation\":0,"
|
||||
"\"workflow_completion\":0,"
|
||||
"\"material_grounding\":0,"
|
||||
"\"artifact_validity\":0,"
|
||||
"\"safety\":0,"
|
||||
"\"actionability\":0,"
|
||||
"\"regressions\":[],"
|
||||
"\"verdict\":\"pass|warn|fail\""
|
||||
"}"
|
||||
)
|
||||
|
||||
|
||||
async def _run_judge_async(
|
||||
*,
|
||||
bundle: dict[str, Any],
|
||||
provider: str,
|
||||
model: str,
|
||||
base_url: str,
|
||||
) -> dict[str, Any]:
|
||||
llm = build_provider(
|
||||
provider=provider,
|
||||
model=model,
|
||||
api_key=_provider_api_key(provider),
|
||||
base_url=base_url,
|
||||
)
|
||||
chunks: list[str] = []
|
||||
errors: list[str] = []
|
||||
async for event in llm.chat(
|
||||
[Message(role="user", content=_judge_prompt(bundle))],
|
||||
config=ChatConfig(max_tokens=1200, temperature=0, timeout=180),
|
||||
):
|
||||
if isinstance(event, TextDeltaEvent):
|
||||
chunks.append(event.text)
|
||||
elif isinstance(event, ErrorEvent):
|
||||
errors.append(event.message)
|
||||
elif isinstance(event, DoneEvent):
|
||||
break
|
||||
text = "".join(chunks).strip()
|
||||
parsed = _parse_json_object(text)
|
||||
return {
|
||||
"ok": not errors and bool(parsed),
|
||||
"provider": provider,
|
||||
"model": model,
|
||||
"judge": parsed,
|
||||
"raw_text": text if not parsed else "",
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
|
||||
def _parse_json_object(text: str) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(text)
|
||||
return value if isinstance(value, dict) else {}
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
match = re.search(r"\{.*\}", text, re.DOTALL)
|
||||
if not match:
|
||||
return {}
|
||||
try:
|
||||
value = json.loads(match.group(0))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def run_judge(bundle_path: Path, *, provider: str, model: str, base_url: str) -> dict[str, Any]:
|
||||
bundle = json.loads(bundle_path.read_text(encoding="utf-8"))
|
||||
return asyncio.run(
|
||||
_run_judge_async(
|
||||
bundle=bundle,
|
||||
provider=provider,
|
||||
model=model,
|
||||
base_url=base_url,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--env-file", type=Path)
|
||||
parser.add_argument("--json", action="store_true", help="Emit JSON for list/check commands.")
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
sub.add_parser("list", help="List validation cases.")
|
||||
sub.add_parser("check-materials", help="Verify fixture files exist.")
|
||||
|
||||
bundle_p = sub.add_parser("write-empty-bundle", help="Write a judge bundle template.")
|
||||
bundle_p.add_argument("--case-id", required=True)
|
||||
bundle_p.add_argument("--output", type=Path, required=True)
|
||||
|
||||
live_p = sub.add_parser("run-live-smokes", help="Run low-cost live LLM smoke harnesses.")
|
||||
live_p.add_argument("--provider", default="openrouter")
|
||||
live_p.add_argument("--model", default="deepseek/deepseek-v4-flash")
|
||||
live_p.add_argument("--creator-model", default="deepseek/deepseek-v4-pro")
|
||||
live_p.add_argument("--home", type=Path)
|
||||
live_p.add_argument("--bundle-dir", type=Path)
|
||||
|
||||
judge_p = sub.add_parser("judge-bundle", help="Judge a captured E2E bundle with an LLM.")
|
||||
judge_p.add_argument("--bundle", type=Path, required=True)
|
||||
judge_p.add_argument("--provider", default="openrouter")
|
||||
judge_p.add_argument("--model", default="deepseek/deepseek-v4-pro")
|
||||
judge_p.add_argument("--base-url", default="")
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
_load_env_file(args.env_file)
|
||||
|
||||
if args.cmd == "list":
|
||||
result = {"ok": True, "case_file": str(CASE_FILE), "cases": load_cases()}
|
||||
elif args.cmd == "check-materials":
|
||||
result = check_materials(load_cases())
|
||||
elif args.cmd == "write-empty-bundle":
|
||||
result = write_empty_bundle(args.case_id, args.output)
|
||||
elif args.cmd == "run-live-smokes":
|
||||
result = run_live_smokes(
|
||||
provider=args.provider,
|
||||
model=args.model,
|
||||
creator_model=args.creator_model,
|
||||
home=args.home,
|
||||
bundle_dir=args.bundle_dir,
|
||||
)
|
||||
elif args.cmd == "judge-bundle":
|
||||
result = run_judge(
|
||||
args.bundle,
|
||||
provider=args.provider,
|
||||
model=args.model,
|
||||
base_url=args.base_url,
|
||||
)
|
||||
else:
|
||||
raise SystemExit(f"unknown command: {args.cmd}")
|
||||
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0 if result.get("ok") else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Evaluate deterministic meta-skill trigger matching from JSON fixtures."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from opensquilla.skills.loader import SkillLoader
|
||||
from opensquilla.skills.meta.trigger_accuracy import TriggerCase, evaluate_trigger_cases
|
||||
|
||||
|
||||
def load_cases(path: Path) -> list[TriggerCase]:
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(raw, list):
|
||||
raise ValueError("fixture file must contain a JSON array")
|
||||
cases: list[TriggerCase] = []
|
||||
for index, item in enumerate(raw):
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError(f"fixture[{index}] must be an object")
|
||||
name = item.get("name")
|
||||
user_message = item.get("user_message")
|
||||
expected = item.get("expected_meta_skill")
|
||||
if not isinstance(name, str) or not name:
|
||||
raise ValueError(f"fixture[{index}].name must be a non-empty string")
|
||||
if not isinstance(user_message, str):
|
||||
raise ValueError(f"fixture[{index}].user_message must be a string")
|
||||
if expected is not None and not isinstance(expected, str):
|
||||
raise ValueError(
|
||||
f"fixture[{index}].expected_meta_skill must be string or null",
|
||||
)
|
||||
cases.append(TriggerCase(
|
||||
name=name,
|
||||
user_message=user_message,
|
||||
expected_meta_skill=expected,
|
||||
))
|
||||
return cases
|
||||
|
||||
|
||||
def _default_bundled_dir() -> Path:
|
||||
return Path(__file__).resolve().parents[1] / "src" / "opensquilla" / "skills" / "bundled"
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument("fixtures", type=Path)
|
||||
p.add_argument("--bundled-dir", type=Path, default=_default_bundled_dir())
|
||||
p.add_argument("--managed-dir", type=Path, default=None)
|
||||
p.add_argument("--workspace-dir", type=Path, default=None)
|
||||
p.add_argument("--snapshot", type=Path, default=None)
|
||||
p.add_argument("--fail-under", type=float, default=1.0)
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = _parser().parse_args(argv)
|
||||
cases = load_cases(args.fixtures)
|
||||
loader = SkillLoader(
|
||||
bundled_dir=args.bundled_dir,
|
||||
managed_dir=args.managed_dir,
|
||||
workspace_dir=args.workspace_dir,
|
||||
snapshot_path=args.snapshot,
|
||||
)
|
||||
loader.invalidate_cache()
|
||||
report = evaluate_trigger_cases(loader, cases)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0 if float(report["accuracy"]) >= args.fail_under else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,207 @@
|
||||
"""Refresh the vendored models.dev snapshot used by the model catalog.
|
||||
|
||||
Fetches https://models.dev/api.json (MIT-licensed, community-maintained),
|
||||
trims it to the providers OpenSquilla registers, and writes the compact
|
||||
snapshot consumed by ``opensquilla.provider.models_dev``.
|
||||
|
||||
Usage::
|
||||
|
||||
uv run python scripts/refresh_models_dev_snapshot.py
|
||||
|
||||
Review the diff before committing — the snapshot is deliberately small and
|
||||
human-reviewable so upstream data mistakes are caught at refresh time, not
|
||||
at runtime. ``check_snapshot_integrity`` refuses to write a snapshot that
|
||||
shrank suspiciously or silently lost a runtime provider's table.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from collections.abc import Iterable
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
from opensquilla.provider.registry import list_provider_specs
|
||||
|
||||
API_URL = "https://models.dev/api.json"
|
||||
SNAPSHOT_PATH = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "src"
|
||||
/ "opensquilla"
|
||||
/ "provider"
|
||||
/ "models_dev_snapshot.json"
|
||||
)
|
||||
|
||||
# OpenSquilla provider id -> models.dev provider ids (merged in order; the
|
||||
# first source of a model id wins). Derived from each registered spec's
|
||||
# ``catalog_source`` so this script cannot drift from the provider registry.
|
||||
# Script-only extras (sources for providers the registry does not carry)
|
||||
# would be added explicitly after the comprehension — none exist today.
|
||||
PROVIDER_SOURCES: dict[str, tuple[str, ...]] = {
|
||||
spec.provider_id: spec.catalog_source
|
||||
for spec in list_provider_specs()
|
||||
if spec.catalog_source
|
||||
}
|
||||
|
||||
# models.dev ``cost`` field -> compact snapshot key. Both sides are USD per
|
||||
# MILLION tokens, so values are vendored verbatim (no unit conversion).
|
||||
_COST_KEYS: tuple[tuple[str, str], ...] = (
|
||||
("input", "in_mtok"),
|
||||
("output", "out_mtok"),
|
||||
("cache_read", "cr_mtok"),
|
||||
("cache_write", "cw_mtok"),
|
||||
)
|
||||
|
||||
# A refresh that loses more than this fraction of the committed snapshot's
|
||||
# models is treated as an upstream incident (payload truncation, source-id
|
||||
# rename, …), not a routine cleanup: refuse to write.
|
||||
MAX_SHRINK_RATIO = 0.8
|
||||
|
||||
|
||||
def _trim_model(entry: dict) -> dict | None:
|
||||
limit = entry.get("limit") or {}
|
||||
context = int(limit.get("context") or 0)
|
||||
output = int(limit.get("output") or 0)
|
||||
if context <= 0 and output <= 0:
|
||||
return None
|
||||
# Self-contradictory upstream data (context smaller than max output —
|
||||
# e.g. models.dev's openrouter z-ai/glm-5.1 entry) would poison budget
|
||||
# resolution; drop it so lookups fall through to a consistent layer.
|
||||
if 0 < context < output:
|
||||
return None
|
||||
modalities = entry.get("modalities") or {}
|
||||
inputs = {str(item).lower() for item in modalities.get("input") or []}
|
||||
trimmed = {
|
||||
"ctx": context,
|
||||
"out": output,
|
||||
"reasoning": bool(entry.get("reasoning")),
|
||||
"tools": bool(entry.get("tool_call")),
|
||||
"vision": "image" in inputs,
|
||||
}
|
||||
cost = entry.get("cost")
|
||||
if isinstance(cost, dict):
|
||||
for source_key, snapshot_key in _COST_KEYS:
|
||||
value = cost.get(source_key)
|
||||
# Vendor only flat per-Mtok leaf numbers. Some models.dev entries
|
||||
# nest tiered pricing (lists/dicts keyed by context bands) here;
|
||||
# tiers are deliberately ignored — a single misleading average is
|
||||
# worse than "unknown", and nuanced pricing is corrections-owned.
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool) and value >= 0:
|
||||
trimmed[snapshot_key] = value
|
||||
return trimmed
|
||||
|
||||
|
||||
def build_snapshot_providers(
|
||||
data: dict,
|
||||
provider_sources: dict[str, tuple[str, ...]] | None = None,
|
||||
) -> dict[str, dict[str, dict]]:
|
||||
"""Trim a raw models.dev ``api.json`` payload to the snapshot tables.
|
||||
|
||||
Pure transform (no network, no filesystem) so tests can drive it with
|
||||
synthetic payloads. ``provider_sources`` defaults to the registry-derived
|
||||
``PROVIDER_SOURCES`` mapping.
|
||||
"""
|
||||
sources_map = PROVIDER_SOURCES if provider_sources is None else provider_sources
|
||||
providers: dict[str, dict[str, dict]] = {}
|
||||
for osq_id, sources in sources_map.items():
|
||||
table: dict[str, dict] = {}
|
||||
for source in sources:
|
||||
models = (data.get(source) or {}).get("models") or {}
|
||||
for model_id, entry in models.items():
|
||||
key = str(model_id).strip().lower()
|
||||
if key in table:
|
||||
continue
|
||||
trimmed = _trim_model(entry)
|
||||
if trimmed is not None:
|
||||
table[key] = trimmed
|
||||
if table:
|
||||
providers[osq_id] = dict(sorted(table.items()))
|
||||
return providers
|
||||
|
||||
|
||||
def _provider_tables(snapshot: dict) -> dict[str, dict[str, dict]]:
|
||||
providers = snapshot.get("providers")
|
||||
return providers if isinstance(providers, dict) else {}
|
||||
|
||||
|
||||
def check_snapshot_integrity(
|
||||
new: dict,
|
||||
old: dict,
|
||||
required_provider_ids: Iterable[str] | None = None,
|
||||
) -> list[str]:
|
||||
"""Return human-readable reasons a freshly built snapshot must NOT be written.
|
||||
|
||||
Pure comparison of the new snapshot dict against the committed one (both
|
||||
in on-disk shape, i.e. carrying a ``providers`` table) — no network, so
|
||||
the guards are unit-testable against synthetic dicts. An empty list means
|
||||
the snapshot is safe to write.
|
||||
|
||||
Guards:
|
||||
- max-shrink: total model count below ``MAX_SHRINK_RATIO`` of the
|
||||
committed count means upstream truncation or source-id drift.
|
||||
- table regression: a runtime-supported provider with a non-empty
|
||||
``catalog_source`` that HAS a committed table but produced zero entries
|
||||
lost its data — never silently degrade it to the synthesized floor.
|
||||
|
||||
``required_provider_ids`` defaults to every runtime-supported registry
|
||||
spec that declares a ``catalog_source``.
|
||||
"""
|
||||
errors: list[str] = []
|
||||
new_tables = _provider_tables(new)
|
||||
old_tables = _provider_tables(old)
|
||||
|
||||
new_total = sum(len(models) for models in new_tables.values())
|
||||
old_total = sum(len(models) for models in old_tables.values())
|
||||
if old_total > 0 and new_total < old_total * MAX_SHRINK_RATIO:
|
||||
errors.append(
|
||||
f"model count shrank from {old_total} to {new_total} "
|
||||
f"(< {MAX_SHRINK_RATIO:.0%} of the committed snapshot)"
|
||||
)
|
||||
|
||||
if required_provider_ids is None:
|
||||
required_provider_ids = [
|
||||
spec.provider_id
|
||||
for spec in list_provider_specs()
|
||||
if spec.runtime_supported and spec.catalog_source
|
||||
]
|
||||
for provider_id in required_provider_ids:
|
||||
if old_tables.get(provider_id) and not new_tables.get(provider_id):
|
||||
errors.append(
|
||||
f"provider {provider_id!r} produced zero entries but the "
|
||||
"committed snapshot has a table for it"
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def main() -> int:
|
||||
data = httpx.get(API_URL, timeout=30.0, follow_redirects=True).json()
|
||||
providers = build_snapshot_providers(data)
|
||||
|
||||
snapshot = {
|
||||
"_source": API_URL,
|
||||
"_license": "MIT (models.dev, maintained by the SST team)",
|
||||
"_fetched": date.today().isoformat(),
|
||||
"providers": providers,
|
||||
}
|
||||
|
||||
try:
|
||||
committed = json.loads(SNAPSHOT_PATH.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
committed = {}
|
||||
errors = check_snapshot_integrity(snapshot, committed)
|
||||
if errors:
|
||||
for error in errors:
|
||||
print(f"refusing to write snapshot: {error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
SNAPSHOT_PATH.write_text(json.dumps(snapshot, indent=1, sort_keys=False) + "\n")
|
||||
total = sum(len(models) for models in providers.values())
|
||||
print(f"wrote {SNAPSHOT_PATH} ({len(providers)} providers, {total} models)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from opensquilla.cli.gateway_client import GatewayClient
|
||||
|
||||
|
||||
def _load_cases(path: Path) -> list[dict[str, Any]]:
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(data, dict) or not isinstance(data.get("cases"), list):
|
||||
raise SystemExit(f"Golden prompt file must contain a top-level cases list: {path}")
|
||||
return list(data["cases"])
|
||||
|
||||
|
||||
def _tool_names(events: list[dict[str, Any]]) -> set[str]:
|
||||
names: set[str] = set()
|
||||
for event in events:
|
||||
for key in ("tool_name", "name"):
|
||||
value = event.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
names.add(value)
|
||||
payload = event.get("payload")
|
||||
if isinstance(payload, dict):
|
||||
value = payload.get("tool_name") or payload.get("name")
|
||||
if isinstance(value, str) and value:
|
||||
names.add(value)
|
||||
return names
|
||||
|
||||
|
||||
def _assistant_text(events: list[dict[str, Any]]) -> str:
|
||||
chunks: list[str] = []
|
||||
for event in events:
|
||||
for key in ("text", "delta", "message"):
|
||||
value = event.get(key)
|
||||
if isinstance(value, str):
|
||||
chunks.append(value)
|
||||
return "\n".join(chunks)
|
||||
|
||||
|
||||
def _case_result(case: dict[str, Any], events: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
text = _assistant_text(events)
|
||||
tools = _tool_names(events)
|
||||
failures: list[str] = []
|
||||
for token in case.get("must_contain", []):
|
||||
if token not in text:
|
||||
failures.append(f"missing text: {token}")
|
||||
any_tokens = case.get("must_contain_any", [])
|
||||
if any_tokens and not any(token in text for token in any_tokens):
|
||||
failures.append(f"missing any text: {any_tokens}")
|
||||
for token in case.get("must_not_contain", []):
|
||||
if token in text:
|
||||
failures.append(f"forbidden text: {token}")
|
||||
for name in case.get("expected_tools", []):
|
||||
if name not in tools:
|
||||
failures.append(f"missing tool: {name}")
|
||||
for name in case.get("forbidden_tools", []):
|
||||
if name in tools:
|
||||
failures.append(f"forbidden tool: {name}")
|
||||
return {
|
||||
"id": case.get("id"),
|
||||
"ok": not failures,
|
||||
"failures": failures,
|
||||
"tools": sorted(tools),
|
||||
"text_excerpt": text[:1000],
|
||||
}
|
||||
|
||||
|
||||
async def _run_case(client: GatewayClient, case: dict[str, Any]) -> dict[str, Any]:
|
||||
session_key = await client.create_session(display_name=f"golden:{case['id']}")
|
||||
events = [
|
||||
event
|
||||
async for event in client.send_message(session_key, str(case["prompt"]))
|
||||
]
|
||||
return _case_result(case, events)
|
||||
|
||||
|
||||
async def _main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run public release golden prompts.")
|
||||
parser.add_argument(
|
||||
"--golden",
|
||||
type=Path,
|
||||
default=Path("tests/golden/public_release_open.yaml"),
|
||||
)
|
||||
parser.add_argument("--gateway", default="ws://127.0.0.1:18791/ws")
|
||||
parser.add_argument(
|
||||
"--out",
|
||||
type=Path,
|
||||
default=Path("tests/functional/reports/public-release-golden.json"),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
cases = _load_cases(args.golden)
|
||||
client = GatewayClient()
|
||||
await client.connect(args.gateway)
|
||||
try:
|
||||
results = [await _run_case(client, case) for case in cases]
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
report = {
|
||||
"generated_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
"gateway": args.gateway,
|
||||
"results": results,
|
||||
"ok": all(result["ok"] for result in results),
|
||||
}
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.out.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, indent=2, ensure_ascii=False))
|
||||
return 0 if report["ok"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(_main()))
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,108 @@
|
||||
# ruff: noqa: E402
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
HARNESS_PARENT = Path(__file__).resolve().parents[1] / "tests" / "integration" / "cli"
|
||||
if str(HARNESS_PARENT) not in sys.path:
|
||||
sys.path.insert(0, str(HARNESS_PARENT))
|
||||
|
||||
from tui_real_terminal.driver import ( # type: ignore[import-not-found]
|
||||
build_run_id,
|
||||
open_real_terminal_session,
|
||||
)
|
||||
from tui_real_terminal.evidence import EvidenceBundle # type: ignore[import-not-found]
|
||||
from tui_real_terminal.scenarios import ( # type: ignore[import-not-found]
|
||||
all_scenarios,
|
||||
run_scenario,
|
||||
scenario_by_id,
|
||||
)
|
||||
from tui_real_terminal.targets import ( # type: ignore[import-not-found]
|
||||
TargetContext,
|
||||
build_tui_target,
|
||||
)
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run an OpenSquilla TUI real-terminal scenario."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--scenario",
|
||||
choices=[scenario.scenario_id for scenario in all_scenarios()],
|
||||
required=True,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--backend",
|
||||
choices=("opentui", "live-opentui"),
|
||||
default="opentui",
|
||||
)
|
||||
parser.add_argument("--driver", choices=("auto", "tmux", "pty"), default="auto")
|
||||
parser.add_argument(
|
||||
"--artifact-root",
|
||||
default=".artifacts/tui-real-terminal/runs",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def _assert_live_backend_enabled(backend: str) -> None:
|
||||
if backend != "live-opentui":
|
||||
return
|
||||
if os.environ.get("OPENSQUILLA_TUI_LIVE_REAL") == "1":
|
||||
return
|
||||
raise SystemExit(
|
||||
"set OPENSQUILLA_TUI_LIVE_REAL=1 to run the real CLI/OpenTUI smoke"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = _parser().parse_args()
|
||||
_assert_live_backend_enabled(args.backend)
|
||||
scenario = scenario_by_id(args.scenario)
|
||||
evidence = EvidenceBundle.create(
|
||||
Path(args.artifact_root),
|
||||
scenario_id=scenario.scenario_id,
|
||||
backend_id=args.backend,
|
||||
)
|
||||
target = build_tui_target(
|
||||
args.backend,
|
||||
TargetContext(
|
||||
project_root=Path.cwd(),
|
||||
artifact_dir=evidence.run_dir,
|
||||
scenario_id=scenario.scenario_id,
|
||||
size=scenario.initial_size,
|
||||
),
|
||||
)
|
||||
if not target.available:
|
||||
raise SystemExit(target.skip_reason or f"backend {args.backend!r} unavailable")
|
||||
if (
|
||||
scenario.required_backend_id is not None
|
||||
and target.backend_id != scenario.required_backend_id
|
||||
):
|
||||
raise SystemExit(
|
||||
f"scenario {scenario.scenario_id!r} requires "
|
||||
f"--backend {scenario.required_backend_id}"
|
||||
)
|
||||
session = open_real_terminal_session(
|
||||
command=target.command,
|
||||
cwd=Path.cwd(),
|
||||
env=target.env,
|
||||
run_id=build_run_id(scenario.scenario_id),
|
||||
size=target.initial_size,
|
||||
artifact_dir=evidence.run_dir,
|
||||
driver="tmux" if scenario.requires_tmux else args.driver,
|
||||
)
|
||||
result = run_scenario(
|
||||
scenario=scenario,
|
||||
session=session,
|
||||
evidence=evidence,
|
||||
backend_id=target.backend_id,
|
||||
)
|
||||
print(f"{result.status}: {result.run_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Regenerate the SquillaRouter V4 Phase 3 artifact manifest."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
BUNDLE_REL = Path("src/opensquilla/squilla_router/models/v4.2_phase3_inference")
|
||||
BUNDLE_DIR = REPO_ROOT / BUNDLE_REL
|
||||
MANIFEST_PATH = BUNDLE_DIR / "artifact_manifest.json"
|
||||
|
||||
ASSET_SUFFIXES = {
|
||||
".bin",
|
||||
".joblib",
|
||||
".json",
|
||||
".onnx",
|
||||
".pkl",
|
||||
".txt",
|
||||
".yaml",
|
||||
".yml",
|
||||
}
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _kind(rel_path: str) -> str:
|
||||
suffix = Path(rel_path).suffix.lower()
|
||||
if suffix == ".bin":
|
||||
return "lightgbm_model"
|
||||
if suffix == ".onnx":
|
||||
return "onnx_model"
|
||||
if suffix in {".pkl", ".joblib"}:
|
||||
return "pickle_joblib_artifact"
|
||||
if suffix == ".json":
|
||||
return "json_metadata"
|
||||
if suffix in {".yaml", ".yml"}:
|
||||
return "yaml_config"
|
||||
if suffix == ".txt":
|
||||
return "text_asset"
|
||||
return "asset"
|
||||
|
||||
|
||||
def _source_note(rel_path: str) -> str:
|
||||
if rel_path.startswith("bge_onnx/"):
|
||||
return "Derived from BAAI/bge-small-zh-v1.5; see PROVENANCE.md."
|
||||
if rel_path.startswith("features/"):
|
||||
return "Router runtime feature extraction artifact."
|
||||
if rel_path.startswith("mlp/"):
|
||||
return "Router runtime MLP head artifact."
|
||||
if rel_path.startswith("lgbm_"):
|
||||
return "Router runtime LightGBM head artifact."
|
||||
return "Router V4 Phase 3 bundle metadata or runtime configuration."
|
||||
|
||||
|
||||
def iter_asset_paths() -> list[Path]:
|
||||
paths: list[Path] = []
|
||||
for path in BUNDLE_DIR.rglob("*"):
|
||||
if not path.is_file():
|
||||
continue
|
||||
if path == MANIFEST_PATH:
|
||||
continue
|
||||
if "__pycache__" in path.parts:
|
||||
continue
|
||||
if path.suffix.lower() not in ASSET_SUFFIXES:
|
||||
continue
|
||||
paths.append(path)
|
||||
return sorted(paths, key=lambda item: item.relative_to(BUNDLE_DIR).as_posix())
|
||||
|
||||
|
||||
def build_manifest() -> dict[str, object]:
|
||||
files = []
|
||||
for path in iter_asset_paths():
|
||||
rel_path = path.relative_to(BUNDLE_DIR).as_posix()
|
||||
files.append(
|
||||
{
|
||||
"path": rel_path,
|
||||
"size_bytes": path.stat().st_size,
|
||||
"sha256": _sha256(path),
|
||||
"kind": _kind(rel_path),
|
||||
"source_note": _source_note(rel_path),
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"bundle": BUNDLE_REL.as_posix(),
|
||||
"description": "Checksums and provenance notes for SquillaRouter V4 Phase 3 assets.",
|
||||
"files": files,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
manifest = build_manifest()
|
||||
content = json.dumps(manifest, indent=2, sort_keys=False) + "\n"
|
||||
with MANIFEST_PATH.open("w", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write(content)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user