chore: import upstream snapshot with attribution

2026-07-13 13:12:33 +08:00
commit aacb60a4af
3387 changed files with 981307 additions and 0 deletions
@@ -0,0 +1,237 @@
+"""Tests for scripts/experiments/replay_finalize_gate.py live-parity reporting.
+
+The replay harness must mirror the live gate's firing condition: the gate
+only evaluates on a zero-tool-call assistant message (a finalize attempt),
+so transcripts that end mid-loop are reported quiet with an explicit
+suppression reason while keeping the raw triggers for diagnostics.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+
+def _load_script():
+    path = Path(__file__).parents[2] / "scripts" / "experiments" / "replay_finalize_gate.py"
+    spec = importlib.util.spec_from_file_location("replay_finalize_gate", path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def _assistant_tool_call(call_id: str, name: str, arguments: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "message": {
+            "role": "assistant",
+            "content": [
+                {"type": "toolCall", "id": call_id, "name": name, "arguments": arguments}
+            ],
+        }
+    }
+
+
+def _tool_result(
+    call_id: str,
+    name: str,
+    text: str,
+    *,
+    is_error: bool = False,
+    execution_status: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    message: dict[str, Any] = {
+        "role": "toolResult",
+        "toolName": name,
+        "toolCallId": call_id,
+        "isError": is_error,
+        "content": text,
+    }
+    if execution_status is not None:
+        message["executionStatus"] = execution_status
+    return {"message": message}
+
+
+_RED_EXEC_ENTRIES = [
+    _assistant_tool_call("w1", "edit_file", {"path": "src/main.py"}),
+    _tool_result("w1", "edit_file", "edited"),
+    _assistant_tool_call("x1", "exec_command", {"command": "python repro.py"}),
+    _tool_result(
+        "x1",
+        "exec_command",
+        "exit_code=1\nFAILED: assertion did not hold",
+        is_error=True,
+        execution_status={
+            "version": 1,
+            "status": "error",
+            "exit_code": 1,
+            "timed_out": False,
+            "reason": "nonzero_exit",
+        },
+    ),
+]
+
+_FINAL_TEXT_ENTRY = {
+    "message": {
+        "role": "assistant",
+        "content": [{"type": "text", "text": "The fix is complete."}],
+    }
+}
+
+
+def _write_run_dir(
+    tmp_path: Path,
+    entries: list[dict[str, Any]],
+    *,
+    git_patch: str | None = "diff --git a/src/main.py b/src/main.py\n+new\n",
+) -> Path:
+    run_dir = tmp_path / "run"
+    run_dir.mkdir()
+    (run_dir / "transcript.jsonl").write_text(
+        "".join(json.dumps(entry, ensure_ascii=False) + "\n" for entry in entries),
+        encoding="utf-8",
+    )
+    if git_patch is not None:
+        (run_dir / "git.patch").write_text(git_patch, encoding="utf-8")
+    return run_dir
+
+
+def test_replay_fires_on_finalize_attempt_with_red_evidence(tmp_path) -> None:
+    module = _load_script()
+    run_dir = _write_run_dir(tmp_path, [*_RED_EXEC_ENTRIES, _FINAL_TEXT_ENTRY])
+
+    report = module.replay_run(run_dir)
+
+    assert report["finalize_attempt_at_end"] is True
+    assert report["should_challenge"] is True
+    assert report["triggers"] == ["red_execution_after_final_edit"]
+    assert report["has_workspace_diff_source"] == "git.patch"
+
+
+def test_replay_suppresses_transcript_ending_mid_loop(tmp_path) -> None:
+    # Same evidence, but the transcript ends on a tool-call assistant message
+    # (iteration cap / abort): the live gate would never have evaluated.
+    module = _load_script()
+    run_dir = _write_run_dir(tmp_path, list(_RED_EXEC_ENTRIES))
+
+    report = module.replay_run(run_dir)
+
+    assert report["finalize_attempt_at_end"] is False
+    assert report["should_challenge"] is False
+    assert report["suppressed_reason"] == "no_finalize_attempt_at_transcript_end"
+    # Raw triggers stay visible for diagnostics.
+    assert report["triggers"] == ["red_execution_after_final_edit"]
+
+
+def test_replay_quiet_without_workspace_diff(tmp_path) -> None:
+    module = _load_script()
+    run_dir = _write_run_dir(
+        tmp_path, [*_RED_EXEC_ENTRIES, _FINAL_TEXT_ENTRY], git_patch=""
+    )
+
+    report = module.replay_run(run_dir)
+
+    assert report["has_workspace_diff"] is False
+    assert report["should_challenge"] is False
+    assert report["triggers"] == []
+
+
+def test_replay_denied_execution_is_not_red_evidence(tmp_path) -> None:
+    # Live/replay parity for the non-verification skip: a policy-denied
+    # command after a green verification must not challenge (live
+    # defect: sandbox denials counted as trailing reds).
+    module = _load_script()
+    entries = [
+        _assistant_tool_call("w1", "edit_file", {"path": "src/main.py"}),
+        _tool_result("w1", "edit_file", "edited"),
+        _assistant_tool_call("x1", "exec_command", {"command": "pytest tests/test_main.py"}),
+        _tool_result(
+            "x1",
+            "exec_command",
+            "exit_code=0\n4 passed",
+            execution_status={
+                "version": 1,
+                "status": "success",
+                "exit_code": 0,
+                "timed_out": False,
+                "reason": None,
+            },
+        ),
+        _assistant_tool_call(
+            "x2",
+            "exec_command",
+            {"command": "javac -cp /opt/gradle/caches Probe.java"},
+        ),
+        _tool_result(
+            "x2",
+            "exec_command",
+            '{"status": "policy_denied", "message": "blocked by sandbox policy"}',
+            is_error=True,
+            execution_status={
+                "version": 1,
+                "status": "error",
+                "exit_code": None,
+                "timed_out": False,
+                "reason": "denied",
+            },
+        ),
+        _FINAL_TEXT_ENTRY,
+    ]
+    run_dir = _write_run_dir(tmp_path, entries)
+
+    report = module.replay_run(run_dir)
+
+    assert report["finalize_attempt_at_end"] is True
+    assert report["should_challenge"] is False
+    assert report["triggers"] == []
+
+
+def test_replay_approval_retry_keeps_arguments_for_second_result(tmp_path) -> None:
+    # The approval flow emits TWO toolResult messages for the same call id:
+    # first ``approval_pending`` (no outcome), then the real retried result.
+    # The retried result must still see the original arguments — losing them
+    # would drop the command and silently erase genuine red evidence.
+    module = _load_script()
+    entries = [
+        _assistant_tool_call("w1", "edit_file", {"path": "src/main.py"}),
+        _tool_result("w1", "edit_file", "edited"),
+        _assistant_tool_call("x1", "exec_command", {"command": "python repro.py"}),
+        _tool_result(
+            "x1",
+            "exec_command",
+            "Approval requested; awaiting decision",
+            is_error=True,
+            execution_status={
+                "version": 1,
+                "status": "error",
+                "exit_code": None,
+                "timed_out": False,
+                "reason": "approval_pending",
+            },
+        ),
+        _tool_result(
+            "x1",
+            "exec_command",
+            "exit_code=1\nFAILED: assertion did not hold",
+            is_error=True,
+            execution_status={
+                "version": 1,
+                "status": "error",
+                "exit_code": 1,
+                "timed_out": False,
+                "reason": "nonzero_exit",
+            },
+        ),
+        _FINAL_TEXT_ENTRY,
+    ]
+    run_dir = _write_run_dir(tmp_path, entries)
+
+    report = module.replay_run(run_dir)
+
+    assert report["finalize_attempt_at_end"] is True
+    assert report["should_challenge"] is True
+    assert report["triggers"] == ["red_execution_after_final_edit"]