chore: import upstream snapshot with attribution

tools/interactive_shell/shell/__init__.py

@@ -0,0 +1,17 @@
+"""Shell command machinery for the interactive REPL.
+
+Groups the shell command-line concern next to the agent-facing
+``tools.interactive_shell.actions.shell``:
+
+* ``parsing`` turns command text into an executable shape,
+* ``policy`` resolves the (alpha-mode, allow-everything) shell execution plan,
+* ``execution`` runs the subprocess and returns a structured result,
+* ``runner`` wires parsing, policy, builtins (``cd`` / ``pwd``), and execution
+  together and records the turn.
+
+Import submodules explicitly (for example ``tools.interactive_shell.shell.runner``)
+rather than relying on this package initializer, to keep interactive-shell
+startup import-light.
+"""
+
+from __future__ import annotations

tools/interactive_shell/shell/display.py

@@ -0,0 +1,68 @@
+"""Terminal-friendly shell command display helpers."""
+
+from __future__ import annotations
+
+import re
+
+_HEREDOC_DELIMITER_RE = re.compile(r"<<(-)?\s*(?:'([^'\n]+)'|\"([^\"\n]+)\"|([^\s\\|;&<>]+))")
+
+
+def _delimiter_from_match(match: re.Match[str]) -> tuple[str, bool]:
+    strip_tabs = match.group(1) == "-"
+    delimiter = match.group(2) or match.group(3) or match.group(4) or ""
+    return delimiter, strip_tabs
+
+
+def _closing_delimiter_line(line: str, *, delimiter: str, strip_tabs: bool) -> bool:
+    normalized = line.rstrip("\r\n")
+    if strip_tabs:
+        normalized = normalized.lstrip("\t")
+    return normalized == delimiter
+
+
+def format_shell_command_for_display(command: str) -> str:
+    """Return a compact, user-facing command string for the REPL prompt area.
+
+    Heredoc bodies (for example the Python script in ``python3 - <<'PY'``) are
+    collapsed to a single summary line so incidental agent-generated scripts do
+    not flood the terminal. The full ``command`` is still executed unchanged.
+    """
+    lines = command.splitlines()
+    if len(lines) <= 1:
+        return command
+
+    display_lines: list[str] = []
+    index = 0
+    while index < len(lines):
+        line = lines[index]
+        match = _HEREDOC_DELIMITER_RE.search(line)
+        if match is None:
+            display_lines.append(line)
+            index += 1
+            continue
+
+        delimiter, strip_tabs = _delimiter_from_match(match)
+        body_start = index + 1
+        close_index: int | None = None
+        for candidate_index in range(body_start, len(lines)):
+            if _closing_delimiter_line(
+                lines[candidate_index],
+                delimiter=delimiter,
+                strip_tabs=strip_tabs,
+            ):
+                close_index = candidate_index
+                break
+
+        body_line_count = 0 if close_index is None else close_index - body_start
+        if body_line_count <= 0:
+            summary = f"{line} …"
+        else:
+            noun = "line" if body_line_count == 1 else "lines"
+            summary = f"{line} … ({body_line_count} {noun})"
+        display_lines.append(summary)
+        index = len(lines) if close_index is None else close_index + 1
+
+    return "\n".join(display_lines)
+
+
+__all__ = ["format_shell_command_for_display"]

tools/interactive_shell/shell/execution.py

@@ -0,0 +1,125 @@
+"""Structured shell command execution helpers for the interactive REPL."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ShellExecutionResult:
+    """Normalized command execution output."""
+
+    command: str
+    argv: list[str] | None
+    stdout: str
+    stderr: str
+    exit_code: int | None
+    timed_out: bool
+    truncated: bool
+    executed_with_shell: bool
+
+
+def _truncate_output(text: str, *, max_chars: int) -> tuple[str, bool]:
+    if len(text) <= max_chars:
+        return text, False
+    return f"{text[:max_chars].rstrip()}\n... output truncated ...", True
+
+
+def _text_from_timeout_stream(raw: str | bytes | None) -> str:
+    if raw is None:
+        return ""
+    if isinstance(raw, str):
+        return raw
+    return raw.decode("utf-8", errors="replace")
+
+
+def _shell_argv(command: str) -> list[str]:
+    if os.name == "nt":
+        shell = os.environ.get("COMSPEC") or "cmd.exe"
+        return [shell, "/d", "/s", "/c", command]
+    shell = os.environ.get("SHELL") or "/bin/sh"
+    return [shell, "-lc", command]
+
+
+def execute_shell_command(
+    *,
+    command: str,
+    argv: list[str] | None,
+    use_shell: bool,
+    timeout_seconds: int,
+    max_output_chars: int,
+) -> ShellExecutionResult:
+    """Execute a command and return a structured result object."""
+    try:
+        if use_shell:
+            # Intentional REPL shell passthrough for local terminal commands.
+            # The caller runs through interactive confirmation/policy first and
+            # records that the command used a shell in ShellExecutionResult.
+            completed = subprocess.run(
+                _shell_argv(command),
+                shell=False,
+                capture_output=True,
+                text=True,
+                encoding="utf-8",
+                errors="replace",
+                timeout=timeout_seconds,
+                check=False,
+            )
+        else:
+            if argv is None:
+                raise ValueError("argv is required for shell=False execution.")
+            completed = subprocess.run(
+                argv,
+                shell=False,
+                capture_output=True,
+                text=True,
+                encoding="utf-8",
+                errors="replace",
+                timeout=timeout_seconds,
+                check=False,
+            )
+    except subprocess.TimeoutExpired as exc:
+        stdout = _text_from_timeout_stream(exc.stdout)
+        stderr = _text_from_timeout_stream(exc.stderr)
+        stdout, truncated_stdout = _truncate_output(
+            stdout,
+            max_chars=max_output_chars,
+        )
+        stderr, truncated_stderr = _truncate_output(
+            stderr,
+            max_chars=max_output_chars,
+        )
+        return ShellExecutionResult(
+            command=command,
+            argv=argv,
+            stdout=stdout,
+            stderr=stderr,
+            exit_code=None,
+            timed_out=True,
+            truncated=truncated_stdout or truncated_stderr,
+            executed_with_shell=use_shell,
+        )
+
+    stdout, truncated_stdout = _truncate_output(
+        completed.stdout or "",
+        max_chars=max_output_chars,
+    )
+    stderr, truncated_stderr = _truncate_output(
+        completed.stderr or "",
+        max_chars=max_output_chars,
+    )
+    return ShellExecutionResult(
+        command=command,
+        argv=argv,
+        stdout=stdout,
+        stderr=stderr,
+        exit_code=completed.returncode,
+        timed_out=False,
+        truncated=truncated_stdout or truncated_stderr,
+        executed_with_shell=use_shell,
+    )
+
+
+__all__ = ["ShellExecutionResult", "execute_shell_command"]

tools/interactive_shell/shell/parsing.py

@@ -0,0 +1,159 @@
+"""Shell command parsing for the interactive REPL.
+
+Alpha mode: no command-safety policy
+------------------------------------
+While OpenSRE is in alpha we run **every** command the user or action agent
+asks for. There is intentionally no allowlist, no read-only / mutating /
+restricted classification, and no deny floor — guardrails are deliberately
+omitted to keep developer velocity high. See
+``docs/interactive-shell-action-policy.md`` for the rationale.
+
+This module's only job is to turn command text into a shape the runner can
+execute:
+
+* explicit ``!`` passthrough → run the remainder through a shell,
+* commands using shell operators / substitution / heredocs → run through a shell,
+* anything that fails to tokenize → hand the raw string to a shell,
+* everything else → split into ``argv`` and run without a shell (which also lets
+  the runner detect the ``cd`` / ``pwd`` REPL builtins so the working directory
+  persists across turns).
+
+The only non-execution outcome is a ``parse_error`` for genuinely empty input
+(e.g. a bare ``!``). That is input validation, not a safety guardrail.
+"""
+
+from __future__ import annotations
+
+import re
+import shlex
+from dataclasses import dataclass
+
+_EXPLICIT_SHELL_PREFIX = "!"
+_SHELL_OPERATOR_RE = re.compile(r"(^|\s)(\|\||&&|[|;<>]|>>|<<|2>)(\s|$)")
+_INLINE_SUBSHELL_RE = re.compile(r"`|\$\(")
+# Heredoc starts such as ``<<'PY'`` or ``<<EOF`` — ``<<`` alone is already covered
+# by ``_SHELL_OPERATOR_RE`` only when followed by whitespace; quoted/unquoted
+# delimiters need an explicit match so ``python3 - <<'PY'`` is not tokenized.
+_HEREDOC_START_RE = re.compile(r"(^|\s)<<-?\s*(?:'[^'\n]+'|\"[^\"\n]+\"|[^\s\\|;&<>]+)")
+
+
+@dataclass(frozen=True)
+class ParsedShellCommand:
+    """Structured command parsing result.
+
+    ``use_shell`` is True when the command must run through a real shell (explicit
+    ``!`` passthrough, shell operators / substitution, or input that could not be
+    tokenized). ``passthrough`` records only the explicit ``!`` prefix so the
+    runner can surface the "shell passthrough" hint for it.
+    """
+
+    command: str
+    argv: list[str] | None
+    passthrough: bool
+    use_shell: bool
+    parse_error: str | None = None
+
+
+def _split_argv(command: str, *, is_windows: bool) -> list[str] | None:
+    try:
+        return shlex.split(command, posix=not is_windows)
+    except ValueError:
+        try:
+            return shlex.split(command, posix=False)
+        except ValueError:
+            return None
+
+
+def parse_shell_command(command: str, *, is_windows: bool) -> ParsedShellCommand:
+    """Parse command text into an executable shape (no safety policy applied)."""
+    stripped = command.strip()
+
+    if stripped.startswith(_EXPLICIT_SHELL_PREFIX):
+        passthrough_command = stripped[len(_EXPLICIT_SHELL_PREFIX) :].strip()
+        if not passthrough_command:
+            return ParsedShellCommand(
+                command="",
+                argv=None,
+                passthrough=True,
+                use_shell=True,
+                parse_error="missing command after passthrough prefix (!).",
+            )
+        return ParsedShellCommand(
+            command=passthrough_command,
+            argv=None,
+            passthrough=True,
+            use_shell=True,
+        )
+
+    if (
+        _SHELL_OPERATOR_RE.search(stripped) is not None
+        or _INLINE_SUBSHELL_RE.search(stripped) is not None
+        or _HEREDOC_START_RE.search(stripped) is not None
+    ):
+        # Operators / substitution need a real shell; alpha mode runs them.
+        return ParsedShellCommand(
+            command=stripped,
+            argv=None,
+            passthrough=False,
+            use_shell=True,
+        )
+
+    argv = _split_argv(stripped, is_windows=is_windows)
+    if argv is None:
+        # Could not tokenize (e.g. unbalanced quotes). Hand the raw string to the
+        # shell instead of blocking it.
+        return ParsedShellCommand(
+            command=stripped,
+            argv=None,
+            passthrough=False,
+            use_shell=True,
+        )
+
+    if not argv:
+        return ParsedShellCommand(
+            command=stripped,
+            argv=None,
+            passthrough=False,
+            use_shell=False,
+            parse_error="empty command.",
+        )
+
+    if is_windows:
+
+        def _strip_outer_quotes(value: str) -> str:
+            if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
+                return value[1:-1]
+            return value
+
+        argv = [_strip_outer_quotes(token) for token in argv]
+
+    return ParsedShellCommand(
+        command=stripped,
+        argv=argv,
+        passthrough=False,
+        use_shell=False,
+    )
+
+
+def argv_for_repl_builtin_detection(
+    *, parsed: ParsedShellCommand, is_windows: bool
+) -> list[str] | None:
+    """Argv tokens for detecting ``cd`` / ``pwd`` REPL builtins.
+
+    Only the plain ``argv`` path and explicit ``!`` passthrough opt into builtin
+    detection. Operator / substitution commands run wholesale through the shell,
+    so they intentionally return ``None`` here (a leading ``cd`` in
+    ``cd /tmp && ls`` must not be hijacked by the builtin handler).
+    """
+    if parsed.argv is not None:
+        return parsed.argv
+    if not parsed.passthrough or not parsed.command.strip():
+        return None
+    return _split_argv(parsed.command.strip(), is_windows=is_windows)
+
+
+__all__ = [
+    "ParsedShellCommand",
+    "argv_for_repl_builtin_detection",
+    "parse_shell_command",
+]

tools/interactive_shell/shell/policy.py

@@ -0,0 +1,71 @@
+"""Shell-specific execution policy for the interactive REPL.
+
+Alpha mode allows every shell command; the only rejected case is genuinely
+empty input. These helpers live next to the rest of the shell machinery so the shared
+execution-policy module is not imported for shell-only concerns by other tools.
+They reuse the shared policy contracts
+(``ExecutionPolicyResult`` / ``ToolExecutionPlan``) from
+``tools.interactive_shell.shared``.
+"""
+
+from __future__ import annotations
+
+import config.constants.platform as _platform
+from tools.interactive_shell.shared import (
+    ExecutionPolicyResult,
+    ToolExecutionMode,
+    ToolExecutionPlan,
+)
+from tools.interactive_shell.shell.parsing import (
+    ParsedShellCommand,
+    parse_shell_command,
+)
+
+
+def evaluate_shell_from_parsed(parsed: ParsedShellCommand) -> ExecutionPolicyResult:
+    """Alpha mode: allow every shell command; only reject empty input.
+
+    There is no command classification or deny floor — any command (mutating,
+    ``restricted``, operators, substitution, passthrough) is allowed. A
+    ``parse_error`` only occurs for empty input (e.g. a bare ``!``), which is
+    rejected because there is nothing to run.
+    """
+    if parsed.parse_error is not None:
+        return ExecutionPolicyResult(
+            verdict="deny",
+            tool_type="shell",
+            reason=parsed.parse_error,
+            hint="Enter a command to run.",
+            shell_classification="unrestricted",
+        )
+
+    return ExecutionPolicyResult(
+        verdict="allow",
+        tool_type="shell",
+        reason=None,
+        shell_classification="unrestricted",
+    )
+
+
+def plan_shell_execution(parsed: ParsedShellCommand) -> ToolExecutionPlan:
+    policy = evaluate_shell_from_parsed(parsed)
+    classification = policy.shell_classification or "unrestricted"
+    return ToolExecutionPlan(
+        tool_type="shell",
+        classification=classification,
+        execution_mode=ToolExecutionMode.FOREGROUND,
+        policy=policy,
+    )
+
+
+def evaluate_shell_command(command: str) -> ExecutionPolicyResult:
+    """Map shell policy + passthrough rules into allow/ask/deny."""
+    parsed = parse_shell_command(command, is_windows=_platform.IS_WINDOWS)
+    return evaluate_shell_from_parsed(parsed)
+
+
+__all__ = [
+    "evaluate_shell_command",
+    "evaluate_shell_from_parsed",
+    "plan_shell_execution",
+]

tools/interactive_shell/shell/runner.py

@@ -0,0 +1,246 @@
+"""Shell command runner: execute builtins and record results."""
+
+from __future__ import annotations
+
+import os
+import shlex
+from pathlib import Path
+from typing import Any
+
+import config.constants.platform as _platform
+from tools.interactive_shell.shell import execution as shell_execution
+from tools.interactive_shell.shell.display import format_shell_command_for_display
+from tools.interactive_shell.shell.parsing import (
+    argv_for_repl_builtin_detection,
+    parse_shell_command,
+)
+from tools.interactive_shell.shell.policy import plan_shell_execution
+from tools.interactive_shell.subprocess import (
+    MAX_COMMAND_OUTPUT_CHARS,
+    SHELL_COMMAND_TIMEOUT_SECONDS,
+    SubprocessPresenter,
+)
+
+_ERROR_STYLE = "error"
+_HIGHLIGHT_STYLE = "highlight"
+
+
+def _shell_payload(
+    *,
+    command: str,
+    ok: bool,
+    response_text: str | None = None,
+    stdout: str = "",
+    stderr: str = "",
+    exit_code: int | None = None,
+    timed_out: bool = False,
+    truncated: bool = False,
+    executed_with_shell: bool | None = None,
+    cancelled: bool = False,
+) -> dict[str, Any]:
+    payload: dict[str, Any] = {
+        "ok": ok,
+        "command": command,
+        "stdout": stdout.strip(),
+        "stderr": stderr.strip(),
+        "exit_code": exit_code,
+        "timed_out": timed_out,
+        "truncated": truncated,
+        "cancelled": cancelled,
+    }
+    if executed_with_shell is not None:
+        payload["executed_with_shell"] = executed_with_shell
+    if response_text:
+        payload["response_text"] = response_text.strip()
+    return payload
+
+
+def run_shell_command(
+    command: str,
+    presenter: SubprocessPresenter,
+    *,
+    argv: list[str] | None = None,
+) -> dict[str, Any]:
+    session = presenter.session
+    parsed = parse_shell_command(command, is_windows=_platform.IS_WINDOWS)
+    plan = plan_shell_execution(parsed)
+    display_command = format_shell_command_for_display(command)
+    if not presenter.execution_allowed(
+        plan.policy,
+        action_summary=f"$ {display_command}",
+    ):
+        session.record("shell", command, ok=False)
+        return _shell_payload(
+            command=command,
+            ok=False,
+            response_text=plan.policy.reason or "shell command blocked",
+            cancelled=plan.policy.verdict != "deny",
+        )
+
+    presenter.print_bold_command(display_command)
+
+    argv_builtin = argv_for_repl_builtin_detection(parsed=parsed, is_windows=_platform.IS_WINDOWS)
+
+    if argv_builtin is not None and argv_builtin[0].lower() == "cd":
+        return run_cd_command(parsed.command, presenter)
+    if argv_builtin is not None and argv_builtin[0].lower() == "pwd":
+        return run_pwd_command(parsed.command, presenter)
+
+    use_shell = parsed.use_shell
+    if parsed.passthrough:
+        presenter.print("[dim]explicit shell passthrough enabled[/]")
+
+    exec_argv = argv if argv is not None else parsed.argv
+
+    response_text: str | None = None
+
+    try:
+        result = shell_execution.execute_shell_command(
+            command=parsed.command,
+            argv=exec_argv,
+            use_shell=use_shell,
+            timeout_seconds=SHELL_COMMAND_TIMEOUT_SECONDS,
+            max_output_chars=MAX_COMMAND_OUTPUT_CHARS,
+        )
+    except Exception as exc:
+        presenter.report_exception(exc, context="surfaces.interactive_shell.shell_command.start")
+
+        response_text = f"command failed to start: {str(exc)}"
+
+        presenter.print_error(f"command failed to start: {exc}")
+        session.record("shell", command, ok=False, response_text=response_text)
+        return _shell_payload(
+            command=command,
+            ok=False,
+            response_text=response_text,
+            stderr=str(exc),
+            executed_with_shell=use_shell,
+        )
+
+    presenter.print_command_output(result.stdout)
+    presenter.print_command_output(result.stderr, style=_ERROR_STYLE)
+    if result.timed_out:
+        response_text = f"command timed out after {SHELL_COMMAND_TIMEOUT_SECONDS} seconds"
+
+        presenter.print(
+            f"[error]command timed out after {SHELL_COMMAND_TIMEOUT_SECONDS} seconds[/]"
+        )
+        session.record("shell", command, ok=False, response_text=response_text)
+        return _shell_payload(
+            command=command,
+            ok=False,
+            response_text=response_text,
+            stdout=result.stdout,
+            stderr=result.stderr,
+            exit_code=result.exit_code,
+            timed_out=True,
+            truncated=result.truncated,
+            executed_with_shell=result.executed_with_shell,
+        )
+    ok = result.exit_code == 0
+    had_stdout = bool((result.stdout or "").strip())
+    had_stderr = bool((result.stderr or "").strip())
+    if ok:
+        if had_stdout:
+            response_text = (result.stdout or "").strip()
+        elif had_stderr:
+            response_text = (result.stderr or "").strip()
+        else:
+            presenter.print(f"[{_HIGHLIGHT_STYLE}]✓[/]")
+    else:
+        code = result.exit_code if result.exit_code is not None else "?"
+        exit_text = f"✗ exit {code}"
+        presenter.print_error(f"✗ exit {code}")
+
+        response_parts = []
+        if had_stdout:
+            response_parts.append((result.stdout or "").strip())
+        if had_stderr:
+            response_parts.append((result.stderr or "").strip())
+        response_parts.append(exit_text)
+        response_text = "\n".join(response_parts)
+
+    session.record("shell", command, ok=ok, response_text=response_text)
+    stderr_for_result = "" if ok and had_stdout else result.stderr
+    return _shell_payload(
+        command=command,
+        ok=ok,
+        response_text=response_text,
+        stdout=result.stdout,
+        stderr=stderr_for_result,
+        exit_code=result.exit_code,
+        timed_out=False,
+        truncated=result.truncated,
+        executed_with_shell=result.executed_with_shell,
+    )
+
+
+def run_cd_command(command: str, presenter: SubprocessPresenter) -> dict[str, Any]:
+    session = presenter.session
+
+    def _strip_outer_quotes(value: str) -> str:
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
+            return value[1:-1]
+        return value
+
+    try:
+        tokens = shlex.split(command, posix=not _platform.IS_WINDOWS)
+        if _platform.IS_WINDOWS and len(tokens) > 1:
+            tokens = [tokens[0], *(_strip_outer_quotes(token) for token in tokens[1:])]
+    except ValueError as exc:
+        response_text = f"cd failed: {str(exc)}"
+
+        presenter.print_error(f"cd failed: {exc}")
+        session.record("shell", command, ok=False, response_text=response_text)
+        return _shell_payload(command=command, ok=False, response_text=response_text)
+
+    if len(tokens) > 2:
+        response_text = "cd failed: too many arguments"
+
+        presenter.print("[error]cd failed:[/] too many arguments")
+        session.record("shell", command, ok=False, response_text=response_text)
+        return _shell_payload(command=command, ok=False, response_text=response_text)
+
+    target = Path(tokens[1]).expanduser() if len(tokens) == 2 else Path.home()
+    try:
+        os.chdir(target)
+    except Exception as exc:
+        presenter.report_exception(exc, context="surfaces.interactive_shell.shell_cd")
+
+        response_text = f"cd failed: {str(exc)}"
+
+        presenter.print_error(f"cd failed: {exc}")
+        session.record("shell", command, ok=False, response_text=response_text)
+        return _shell_payload(command=command, ok=False, response_text=response_text)
+
+    cwd = str(Path.cwd())
+    presenter.print_plain(cwd)
+    session.record("shell", command)
+    return _shell_payload(command=command, ok=True, response_text=cwd)
+
+
+def run_pwd_command(command: str, presenter: SubprocessPresenter) -> dict[str, Any]:
+    session = presenter.session
+    try:
+        tokens = shlex.split(command, posix=not _platform.IS_WINDOWS)
+    except ValueError as exc:
+        response_text = f"pwd failed: {str(exc)}"
+
+        presenter.print_error(f"pwd failed: {exc}")
+        session.record("shell", command, ok=False, response_text=response_text)
+        return _shell_payload(command=command, ok=False, response_text=response_text)
+
+    if len(tokens) != 1:
+        response_text = "pwd failed: too many arguments"
+
+        presenter.print("[error]pwd failed:[/] too many arguments")
+        session.record("shell", command, ok=False, response_text=response_text)
+        return _shell_payload(command=command, ok=False, response_text=response_text)
+
+    cwd = str(Path.cwd())
+    presenter.print_plain(cwd)
+    session.record("shell", command)
+    return _shell_payload(command=command, ok=True, response_text=cwd, stdout=cwd)
+
+
+__all__ = ["run_cd_command", "run_pwd_command", "run_shell_command"]