chore: import upstream snapshot with attribution

session_data/replay_prompt.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+"""Replay the prompt that LocalAgent would have built at a given step.
+
+Usage:
+    python replay_prompt.py <session_dir> <step>
+    python replay_prompt.py ~/mano-trajectory/result/sess-20260415-153428-1fe23a68 3
+
+Output: JSON to stdout with keys:
+    task, step, prompt, images (base64 list), system_prompt
+
+Step is 0-indexed. Step 0 = first prediction (1 image: current screenshot).
+"""
+
+import argparse
+import base64
+import io
+import json
+import os
+import re
+import sys
+
+from PIL import Image
+
+SCREENSHOT_WIDTH = 1280
+HISTORY_IMAGE_COUNT = 2
+
+SYSTEM_PROMPT = "You are a helpful assistant."
+
+ACTION_SPACE = """\
+click(start_box='<|box_start|>(x1,y1)<|box_end|>')
+doubleclick(start_box='<|box_start|>(x1,y1)<|box_end|>')
+select(start_box='<|box_start|>(x1,y1)<|box_end|>')
+drag(start_box='<|box_start|>(x1,y1)<|box_end|>', end_box='<|box_start|>(x3,y3)<|box_end|>')
+hotkey(key='')
+wait()
+call_user()
+type(content='', start_box='<|box_start|>(x1,y1)<|box_end|>')
+stop(reason='')
+scroll(direction='down or up or right or left')
+scrollmenu(start_box='<|box_start|>(x1,y1),(x2,y2)<|box_end|>', direction='down or up or right or left')
+finish()"""
+
+INSTRUCTION_TEMPLATE = """\
+You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. When outputting, output the thought process for the next action between the <think> and </think> tags, its description between the <action_desp> and </action_desp> tags, and the action itself between the <action> and </action> tags.
+
+## Output Format
+<think>think process</think>
+<action_desp>next action description</action_desp>
+<action>next action</action>
+
+## Action Space
+{action_space}
+
+## User Instruction
+### task: {task}
+### action history: {history}
+{current_screenshot}"""
+
+
+def load_session(session_dir: str):
+    """Load result.json and return task + history_resps."""
+    result_path = os.path.join(session_dir, "result.json")
+    with open(result_path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    return data["task"], data["history_resps"]
+
+
+def parse_action_desc(resp_str: str) -> str:
+    """Extract action description from history_resps entry.
+    Prefer <action_desp> (matches runtime behavior), fallback to parsing <action>.
+    """
+    # Prefer action_desp (what LocalAgent actually stores in prompt_history)
+    m = re.search(r"<action_desp>(.*?)</action_desp>", resp_str, re.DOTALL)
+    if m and m.group(1).strip():
+        return m.group(1).strip()
+
+    # Fallback: parse from <action> tag
+    m = re.search(r"<action>(.*?)</action>", resp_str, re.DOTALL)
+    if not m:
+        return "unknown"
+    action_str = m.group(1).strip()
+
+    coord_match = re.search(r"'action':\s*'(\w+)'", action_str)
+    action_name = coord_match.group(1) if coord_match else "unknown"
+
+    coord_match2 = re.search(r"'coordinate':\s*\[(\d+),\s*(\d+)\]", action_str)
+    if coord_match2:
+        return f"{action_name} ({coord_match2.group(1)}, {coord_match2.group(2)})"
+
+    text_match = re.search(r"'text':\s*'([^']*)'", action_str)
+    if text_match:
+        return f"{action_name}: {text_match.group(1)[:30]}"
+
+    dir_match = re.search(r"'scroll_direction':\s*'(\w+)'", action_str)
+    if dir_match:
+        return f"{action_name} {dir_match.group(1)}"
+
+    if "DONE" in action_str:
+        return "DONE"
+    if "FAIL" in action_str:
+        return "FAIL"
+
+    return action_name
+
+
+def img_to_b64(img_path: str) -> str:
+    """Load image, resize to SCREENSHOT_WIDTH, return base64."""
+    img = Image.open(img_path)
+    if img.width != SCREENSHOT_WIDTH:
+        ratio = SCREENSHOT_WIDTH / img.width
+        new_h = int(img.height * ratio)
+        img = img.resize((SCREENSHOT_WIDTH, new_h), Image.LANCZOS)
+    buf = io.BytesIO()
+    img.save(buf, format="PNG")
+    return base64.b64encode(buf.getvalue()).decode("utf-8")
+
+
+def build_prompt_at_step(session_dir: str, step: int):
+    """Reconstruct the prompt that LocalAgent would build at the given step."""
+    task, history_resps = load_session(session_dir)
+    trajectory_dir = os.path.join(session_dir, "trajectory")
+
+    if step < 0 or step >= len(history_resps):
+        sys.exit(json.dumps({"error": f"step {step} out of range (0-{len(history_resps)-1})"}))
+
+    # Build prompt_history for steps 0..step-1 (previous steps)
+    prompt_history = []
+    for i in range(step):
+        desc = parse_action_desc(history_resps[i])
+        screenshot_path = os.path.join(trajectory_dir, f"{i}.png")
+        screenshot_b64 = img_to_b64(screenshot_path) if os.path.exists(screenshot_path) else ""
+        prompt_history.append({"desc": desc, "screenshot_b64": screenshot_b64})
+
+    # Current screenshot = step.png
+    current_path = os.path.join(trajectory_dir, f"{step}.png")
+    if not os.path.exists(current_path):
+        sys.exit(json.dumps({"error": f"{current_path} not found"}))
+    current_b64 = img_to_b64(current_path)
+
+    # Build prompt text + images list (same logic as LocalAgent._build_prompt)
+    images = []
+    recent = prompt_history[-(HISTORY_IMAGE_COUNT + 1):]
+
+    history_parts = []
+    for i, h in enumerate(prompt_history):
+        step_num = i + 1
+        desc = h["desc"]
+        if h in recent and h.get("screenshot_b64"):
+            images.append(h["screenshot_b64"])
+            history_parts.append(f"第{step_num}步：{desc}，对应的截图为<image>")
+        else:
+            history_parts.append(f"第{step_num}步：{desc}")
+
+    history_text = "\n".join(history_parts) if history_parts else "无"
+    images.append(current_b64)
+
+    prompt_text = INSTRUCTION_TEMPLATE.format(
+        action_space=ACTION_SPACE,
+        task=task,
+        history=history_text,
+        current_screenshot="当前截图为<image>",
+    )
+
+    return {
+        "task": task,
+        "step": step,
+        "system_prompt": SYSTEM_PROMPT,
+        "prompt": prompt_text,
+        "images": images,
+        "image_count": len(images),
+        "history": [h["desc"] for h in prompt_history],
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Replay LocalAgent prompt at a given step")
+    parser.add_argument("session_dir", help="Path to session directory")
+    parser.add_argument("step", type=int, help="Step number (0-indexed)")
+    args = parser.parse_args()
+
+    session_dir = os.path.expanduser(args.session_dir)
+    result = build_prompt_at_step(session_dir, args.step)
+    with open("prompt", "w") as fo:
+        json.dump(result, fo, ensure_ascii=False)
+
+
+if __name__ == "__main__":
+    main()

session_data/result.json

@@ -0,0 +1,15 @@
+{
+    "task": "Compare available plans for the AeroAPI on Flightaware.",
+    "agent_type": "local",
+    "history_resps": [
+        "<think>当前页面为Flightaware的首页，主要展示了搜索航班的目的地和一些导航栏选项。由于需要比较AeroAPI在Flightaware上的可用套餐，通常这类信息会在“Products”或相关菜单下展示。点击“Products”可以进入产品列表页面，进一步查找和比较AeroAPI的套餐选项。因此，点击“Products”是合理的下一步操作。</think>\n<action>[{'name': 'computer', 'input': {'action': 'left_click', 'coordinate': [196, 150]}, 'id': '65b7dd0d-06eb-4825-b122-11d861f2a586', 'action_type': 'tool_use'}]</action>",
+        "<think>当前页面已经进入了Flightaware的产品（Products）板块，左侧栏展示了多个产品选项，包括AeroAPI。为了比较AeroAPI在不同计划下的可用性，需要先进入AeroAPI的详细页面，查看其具体信息和可选计划。因此，接下来应点击“AeroAPI”选项，以获取相关计划详情。</think>\n<action>[{'name': 'computer', 'input': {'action': 'left_click', 'coordinate': [515, 349]}, 'id': 'ae2ac4c6-e686-492a-8e32-270231588479', 'action_type': 'tool_use'}]</action>",
+        "<think>当前页面显示了AeroAPI的详细信息，包括数据产品、应用和接收器等选项。为了比较不同计划，需要进一步查看页面上关于价格和订阅的详细信息，因此需要向下滚动页面以找到相关计划详情。</think>\n<action>[{'name': 'computer', 'input': {'action': 'scroll', 'scroll_direction': 'down', 'coordinate': [640, 360], 'scroll_amount': 30}, 'id': '1699e398-5cc4-4363-a126-09b34871e984', 'action_type': 'tool_use'}]</action>",
+        "<think>当前页面主要展示了Flightaware的产品和服务介绍，但没有看到具体的计划对比内容。为了找到AeroAPI的可用计划对比信息，需要继续向下滑动页面，查看是否有相关的计划列表或对比内容。页面应向下滑动，以便浏览更多内容，寻找目标信息。</think>\n<action>[{'name': 'computer', 'input': {'action': 'scroll', 'scroll_direction': 'down', 'coordinate': [640, 360], 'scroll_amount': 30}, 'id': '69f7ecd2-f82f-4702-80ba-b15fda7722cc', 'action_type': 'tool_use'}]</action>",
+        "<think>当前页面已经显示了AeroAPI的介绍和相关链接，但没有看到关于可用计划的详细信息。为了查找AeroAPI的可用计划，需要继续向下滚动页面，查看是否有相关的计划对比或详细介绍内容。页面应该向下滑动，以便查看更多信息。</think>\n<action>[{'name': 'computer', 'input': {'action': 'scroll', 'scroll_direction': 'down', 'coordinate': [640, 360], 'scroll_amount': 30}, 'id': 'a020e194-9bcf-441e-a2f4-a8393b85e9ec', 'action_type': 'tool_use'}]</action>",
+        "<think>当前页面主要展示了Flightaware的商业产品和解决方案，但没有直接显示AeroAPI的具体价格和套餐选项。页面上有“Request pricing”按钮，这通常是用于获取免费报价或套餐信息的入口。点击该按钮可以进入下一步，查看AeroAPI的不同套餐和价格详情。因此，接下来应该点击“Request pricing”按钮以获取更多关于AeroAPI套餐的信息。</think>\n<action>[{'name': 'computer', 'input': {'action': 'left_click', 'coordinate': [505, 590]}, 'id': 'b3048205-f135-479d-bd97-ec9236a78efb', 'action_type': 'tool_use'}]</action>",
+        "<think>已经按照操作步骤进入了相关的页面，并且所有必要的点击和滚动操作都已完成。页面上显示了所需的信息和按钮，没有进一步需要操作的地方。因此，任务已经执行完毕，可以结束操作。</think>\n<action>[{'action_type': 'DONE'}]</action>"
+    ],
+    "elapsed_time_sec": 67.93,
+    "step_count": 7
+}