feat(tui): support clipboard image paste

Fixes #265
This commit is contained in:
tjb-tech
2026-05-17 06:13:19 +00:00
parent 6747d12385
commit 330ece7a62
9 changed files with 515 additions and 22 deletions
+63 -7
View File
@@ -1,6 +1,7 @@
import React, {useDeferredValue, useEffect, useMemo, useState} from 'react';
import React, {useDeferredValue, useEffect, useMemo, useRef, useState} from 'react';
import {Box, Text, useApp, useInput} from 'ink';
import {readClipboardImage, type ImageAttachment} from './clipboardImage.js';
import {CommandPicker} from './components/CommandPicker.js';
import {ConversationView} from './components/ConversationView.js';
import {ModalHost} from './components/ModalHost.js';
@@ -11,7 +12,7 @@ import {SwarmPanel} from './components/SwarmPanel.js';
import {TodoPanel} from './components/TodoPanel.js';
import {useBackendSession} from './hooks/useBackendSession.js';
import {ThemeProvider, useTheme} from './theme/ThemeContext.js';
import type {FrontendConfig} from './types.js';
import type {FrontendConfig, ImageAttachmentPayload} from './types.js';
const rawReturnSubmit = process.env.OPENHARNESS_FRONTEND_RAW_RETURN === '1';
const scriptedSteps = (() => {
@@ -64,6 +65,8 @@ function AppInner({config}: {config: FrontendConfig}): React.JSX.Element {
const [modalInput, setModalInput] = useState('');
const [history, setHistory] = useState<string[]>([]);
const [historyIndex, setHistoryIndex] = useState(-1);
const [imageAttachments, setImageAttachments] = useState<ImageAttachment[]>([]);
const [clipboardStatus, setClipboardStatus] = useState<string | null>(null);
const [lastEscapeAt, setLastEscapeAt] = useState(0);
const [scriptIndex, setScriptIndex] = useState(0);
const [pickerIndex, setPickerIndex] = useState(0);
@@ -77,6 +80,7 @@ function AppInner({config}: {config: FrontendConfig}): React.JSX.Element {
const deferredTodoMarkdown = useDeferredValue(session.todoMarkdown);
const deferredSwarmTeammates = useDeferredValue(session.swarmTeammates);
const deferredSwarmNotifications = useDeferredValue(session.swarmNotifications);
const clipboardStatusTimerRef = useRef<NodeJS.Timeout | null>(null);
useEffect(() => {
const nextTheme = session.status.theme;
@@ -85,6 +89,47 @@ function AppInner({config}: {config: FrontendConfig}): React.JSX.Element {
}
}, [session.status.theme, setThemeName]);
useEffect(() => {
return () => {
if (clipboardStatusTimerRef.current) {
clearTimeout(clipboardStatusTimerRef.current);
}
};
}, []);
const setTemporaryClipboardStatus = (message: string): void => {
setClipboardStatus(message);
if (clipboardStatusTimerRef.current) {
clearTimeout(clipboardStatusTimerRef.current);
}
clipboardStatusTimerRef.current = setTimeout(() => {
setClipboardStatus(null);
clipboardStatusTimerRef.current = null;
}, 2500);
};
const attachClipboardImage = (): void => {
void (async () => {
const image = await readClipboardImage();
if (!image) {
setTemporaryClipboardStatus('No image found in clipboard');
return;
}
setImageAttachments((items) => [...items, image]);
setTemporaryClipboardStatus(`Attached ${image.label}`);
})().catch((error: unknown) => {
const message = error instanceof Error ? error.message : String(error);
setTemporaryClipboardStatus(`Clipboard image unavailable: ${message}`);
});
};
const imagePayloads = (): ImageAttachmentPayload[] =>
imageAttachments.map((image) => ({
media_type: image.media_type,
data: image.data,
source_path: image.source_path,
}));
// Current tool name for spinner
const currentToolName = useMemo(() => {
for (let i = deferredTranscript.length - 1; i >= 0; i--) {
@@ -191,6 +236,11 @@ function AppInner({config}: {config: FrontendConfig}): React.JSX.Element {
return;
}
if (!session.busy && key.ctrl && chunk === 'v') {
attachClipboardImage();
return;
}
// Let ink-text-input handle pasted text directly.
if (isPaste) {
return;
@@ -367,8 +417,9 @@ function AppInner({config}: {config: FrontendConfig}): React.JSX.Element {
if (isEscape) {
const now = Date.now();
if (input && now - lastEscapeAt < 500) {
if ((input || imageAttachments.length > 0) && now - lastEscapeAt < 500) {
setInput('');
setImageAttachments([]);
setHistoryIndex(-1);
setLastEscapeAt(0);
return;
@@ -408,7 +459,7 @@ function AppInner({config}: {config: FrontendConfig}): React.JSX.Element {
setModalInput('');
return;
}
if (!value.trim() || session.busy || !session.ready) {
if ((!value.trim() && imageAttachments.length === 0) || session.busy || !session.ready) {
if (session.busy && value.trim() === '/stop') {
session.sendRequest({type: 'interrupt'});
session.setBusyLabel('Stopping current operation...');
@@ -417,16 +468,19 @@ function AppInner({config}: {config: FrontendConfig}): React.JSX.Element {
return;
}
// Check if it's an interactive command
if (handleCommand(value)) {
if (imageAttachments.length === 0 && handleCommand(value)) {
setHistory((items) => [...items, value]);
setHistoryIndex(-1);
setInput('');
return;
}
session.sendRequest({type: 'submit_line', line: value});
setHistory((items) => [...items, value]);
session.sendRequest({type: 'submit_line', line: value, images: imagePayloads()});
if (value.trim()) {
setHistory((items) => [...items, value]);
}
setHistoryIndex(-1);
setInput('');
setImageAttachments([]);
session.setBusy(true);
};
@@ -511,6 +565,8 @@ function AppInner({config}: {config: FrontendConfig}): React.JSX.Element {
toolName={session.busy ? currentToolName : undefined}
statusLabel={session.busy ? (session.busyLabel ?? (currentToolName ? `Running ${currentToolName}...` : 'Running agent loop...')) : undefined}
suppressSubmit={showPicker}
imageAttachmentLabels={imageAttachments.map((image) => image.label)}
clipboardStatus={clipboardStatus}
/>
)}
+173
View File
@@ -0,0 +1,173 @@
import {execFile} from 'node:child_process';
import {mkdtemp, readFile, rm, stat} from 'node:fs/promises';
import {tmpdir} from 'node:os';
import {join} from 'node:path';
export type ImageAttachment = {
id: string;
label: string;
media_type: string;
data: string;
source_path?: string;
size_bytes?: number;
};
const MAX_CLIPBOARD_IMAGE_BYTES = 15 * 1024 * 1024;
const EXEC_TIMEOUT_MS = 2500;
type ClipboardImageRead = {
data: Buffer;
mediaType: string;
label: string;
};
export async function readClipboardImage(): Promise<ImageAttachment | null> {
const image = await readClipboardImageData();
if (!image) {
return null;
}
return {
id: `clipboard-${Date.now()}-${Math.random().toString(16).slice(2)}`,
label: image.label,
media_type: image.mediaType,
data: image.data.toString('base64'),
source_path: `clipboard:${image.label}`,
size_bytes: image.data.length,
};
}
async function readClipboardImageData(): Promise<ClipboardImageRead | null> {
if (process.platform === 'darwin') {
return readMacClipboardImage();
}
if (process.platform === 'win32') {
return readWindowsClipboardImage();
}
return readLinuxClipboardImage();
}
async function readMacClipboardImage(): Promise<ClipboardImageRead | null> {
const tempDir = await mkdtemp(join(tmpdir(), 'openharness-clipboard-'));
try {
const pngPath = join(tempDir, 'clipboard.png');
if (await runFileCommand('pngpaste', [pngPath])) {
return await readImageFile(pngPath, 'image/png', 'clipboard.png');
}
if (await writeMacClipboardClass('PNGf', pngPath)) {
return await readImageFile(pngPath, 'image/png', 'clipboard.png');
}
const tiffPath = join(tempDir, 'clipboard.tiff');
if (await writeMacClipboardClass('TIFF', tiffPath)) {
if (await runFileCommand('sips', ['-s', 'format', 'png', tiffPath, '--out', pngPath])) {
return await readImageFile(pngPath, 'image/png', 'clipboard.png');
}
return await readImageFile(tiffPath, 'image/tiff', 'clipboard.tiff');
}
return null;
} finally {
await rm(tempDir, {recursive: true, force: true});
}
}
async function writeMacClipboardClass(classCode: 'PNGf' | 'TIFF', outputPath: string): Promise<boolean> {
const appleClass = `${String.fromCharCode(0xab)}class ${classCode}${String.fromCharCode(0xbb)}`;
const escapedPath = outputPath.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
return runFileCommand('osascript', [
'-e',
`set clipboardData to the clipboard as ${appleClass}`,
'-e',
`set outFile to open for access POSIX file "${escapedPath}" with write permission`,
'-e',
'set eof outFile to 0',
'-e',
'write clipboardData to outFile',
'-e',
'close access outFile',
]);
}
async function readWindowsClipboardImage(): Promise<ClipboardImageRead | null> {
const tempDir = await mkdtemp(join(tmpdir(), 'openharness-clipboard-'));
try {
const pngPath = join(tempDir, 'clipboard.png');
const escapedPath = pngPath.replace(/'/g, "''");
const powershell = join(
process.env.SystemRoot ?? 'C:\\Windows',
'System32',
'WindowsPowerShell',
'v1.0',
'powershell.exe',
);
const script = [
'Add-Type -AssemblyName System.Windows.Forms',
'Add-Type -AssemblyName System.Drawing',
'if (-not [Windows.Forms.Clipboard]::ContainsImage()) { exit 2 }',
'$image = [Windows.Forms.Clipboard]::GetImage()',
`$image.Save('${escapedPath}', [Drawing.Imaging.ImageFormat]::Png)`,
].join('; ');
if (!(await runFileCommand(powershell, ['-NoProfile', '-STA', '-Command', script]))) {
return null;
}
return await readImageFile(pngPath, 'image/png', 'clipboard.png');
} finally {
await rm(tempDir, {recursive: true, force: true});
}
}
async function readLinuxClipboardImage(): Promise<ClipboardImageRead | null> {
const attempts: Array<[string, string[], string, string]> = [
['wl-paste', ['--no-newline', '--type', 'image/png'], 'image/png', 'clipboard.png'],
['wl-paste', ['--no-newline', '--type', 'image/jpeg'], 'image/jpeg', 'clipboard.jpg'],
['xclip', ['-selection', 'clipboard', '-target', 'image/png', '-out'], 'image/png', 'clipboard.png'],
['xclip', ['-selection', 'clipboard', '-target', 'image/jpeg', '-out'], 'image/jpeg', 'clipboard.jpg'],
['xsel', ['--clipboard', '--output', '--mime-type', 'image/png'], 'image/png', 'clipboard.png'],
['xsel', ['--clipboard', '--output', '--mime-type', 'image/jpeg'], 'image/jpeg', 'clipboard.jpg'],
];
for (const [command, args, mediaType, label] of attempts) {
const data = await runBufferCommand(command, args);
if (data && data.length > 0) {
return {data, mediaType, label};
}
}
return null;
}
async function readImageFile(path: string, mediaType: string, label: string): Promise<ClipboardImageRead | null> {
const fileStat = await stat(path).catch(() => null);
if (!fileStat || fileStat.size <= 0 || fileStat.size > MAX_CLIPBOARD_IMAGE_BYTES) {
return null;
}
const data = await readFile(path);
return {data, mediaType, label};
}
async function runFileCommand(command: string, args: string[]): Promise<boolean> {
return new Promise((resolve) => {
execFile(command, args, {timeout: EXEC_TIMEOUT_MS, windowsHide: true}, (error) => {
resolve(!error);
});
});
}
async function runBufferCommand(command: string, args: string[]): Promise<Buffer | null> {
return new Promise((resolve) => {
execFile(
command,
args,
{
encoding: 'buffer',
maxBuffer: MAX_CLIPBOARD_IMAGE_BYTES + 1024,
timeout: EXEC_TIMEOUT_MS,
windowsHide: true,
},
(error, stdout) => {
if (error || !Buffer.isBuffer(stdout) || stdout.length > MAX_CLIPBOARD_IMAGE_BYTES) {
resolve(null);
return;
}
resolve(stdout);
},
);
});
}
@@ -173,6 +173,39 @@ test('keeps forward delete behavior when cursor is inside the line', async () =>
}
});
test('ignores ctrl+v so the app can attach clipboard images', async () => {
const stdin = createTestStdin();
const stdout = createTestStdout();
let currentValue = '';
const instance = render(<PromptHarness onInputChange={(value) => {
currentValue = value;
}} />, {
stdin: stdin as unknown as NodeJS.ReadStream & {fd: 0},
stdout: stdout as unknown as NodeJS.WriteStream,
debug: true,
patchConsole: false,
});
const exitPromise = instance.waitUntilExit();
try {
await nextLoopTurn();
await sendKey(stdin, Buffer.from([0x16]));
await nextLoopTurn();
assert.equal(currentValue, '');
await sendKey(stdin, 'v');
await waitForValue(() => currentValue, 'v');
} finally {
instance.unmount();
await exitPromise;
instance.cleanup();
stdin.destroy();
stdout.destroy();
}
});
test('accepts explicit /stop submission while busy', async () => {
const stdin = createTestStdin();
const stdout = createTestStdout();
@@ -78,7 +78,14 @@ function MultilineTextInput({
return;
}
if (key.upArrow || key.downArrow || key.tab || (key.shift && key.tab) || key.escape || (key.ctrl && input === 'c')) {
if (
key.upArrow ||
key.downArrow ||
key.tab ||
(key.shift && key.tab) ||
key.escape ||
(key.ctrl && (input === 'c' || input === 'v'))
) {
return;
}
@@ -197,6 +204,8 @@ export function PromptInput({
toolName,
suppressSubmit,
statusLabel,
imageAttachmentLabels = [],
clipboardStatus,
}: {
busy: boolean;
input: string;
@@ -205,6 +214,8 @@ export function PromptInput({
toolName?: string;
suppressSubmit?: boolean;
statusLabel?: string;
imageAttachmentLabels?: string[];
clipboardStatus?: string | null;
}): React.JSX.Element {
const {theme} = useTheme();
const promptPrefix = busy ? '… ' : '> ';
@@ -218,6 +229,18 @@ export function PromptInput({
</Box>
</Box>
) : null}
{imageAttachmentLabels.length > 0 ? (
<Box>
<Text color={theme.colors.accent}>
{imageAttachmentLabels.map((label, index) => `[image ${index + 1}: ${label}]`).join(' ')}
</Text>
</Box>
) : null}
{clipboardStatus ? (
<Box>
<Text color={theme.colors.muted}>{clipboardStatus}</Text>
</Box>
) : null}
<MultilineTextInput
value={input}
onChange={setInput}
+6
View File
@@ -11,6 +11,12 @@ export type TranscriptItem = {
is_error?: boolean;
};
export type ImageAttachmentPayload = {
media_type: string;
data: string;
source_path?: string;
};
export type TaskSnapshot = {
id: string;
type: string;
+48 -5
View File
@@ -19,6 +19,7 @@ from openharness.commands import MemoryCommandBackend
from openharness.config.settings import CLAUDE_MODEL_ALIAS_OPTIONS, resolve_model_setting
from openharness.bridge import get_bridge_manager
from openharness.coordinator.coordinator_mode import is_coordinator_mode
from openharness.engine.messages import ConversationMessage, ImageBlock, TextBlock
from openharness.themes import list_themes
from openharness.engine.stream_events import (
AssistantTextDelta,
@@ -33,7 +34,7 @@ from openharness.engine.stream_events import (
from openharness.output_styles import load_output_styles
from openharness.tasks import get_task_manager
from openharness.ui.coordinator_drain import drain_coordinator_async_agents
from openharness.ui.protocol import BackendEvent, FrontendRequest, TranscriptItem
from openharness.ui.protocol import BackendEvent, FrontendImageAttachment, FrontendRequest, TranscriptItem
from openharness.ui.runtime import build_runtime, close_runtime, handle_line, start_runtime
from openharness.services.session_backend import SessionBackend
@@ -166,11 +167,13 @@ class ReactBackendHost:
await self._emit(BackendEvent(type="error", message="Session is busy"))
continue
line = (request.line or "").strip()
if not line:
if not line and not request.images:
continue
self._busy = True
try:
should_continue = await self._run_active_request(self._process_line(line))
should_continue = await self._run_active_request(
self._process_line(line, images=request.images)
)
finally:
self._busy = False
if not should_continue:
@@ -244,10 +247,23 @@ class ReactBackendHost:
return
task.cancel()
async def _process_line(self, line: str, *, transcript_line: str | None = None) -> bool:
async def _process_line(
self,
line: str,
*,
transcript_line: str | None = None,
images: list[FrontendImageAttachment] | None = None,
) -> bool:
assert self._bundle is not None
user_message = _build_user_message_with_images(line, images or [])
await self._emit(
BackendEvent(type="transcript_item", item=TranscriptItem(role="user", text=transcript_line or line))
BackendEvent(
type="transcript_item",
item=TranscriptItem(
role="user",
text=transcript_line or _format_transcript_line(line, images or []),
),
)
)
async def _print_system(message: str) -> None:
@@ -358,6 +374,7 @@ class ReactBackendHost:
print_system=_print_system,
render_event=_render_event,
clear_output=_clear_output,
user_message=user_message,
)
if is_coordinator_mode():
await drain_coordinator_async_agents(
@@ -837,6 +854,32 @@ class ReactBackendHost:
sys.stdout.flush()
def _build_user_message_with_images(
line: str,
images: list[FrontendImageAttachment],
) -> ConversationMessage | None:
if not images:
return None
content = [TextBlock(text=line or "Please analyze the attached image.")]
content.extend(
ImageBlock(
media_type=image.media_type,
data=image.data,
source_path=image.source_path or "",
)
for image in images
)
return ConversationMessage.from_user_content(content)
def _format_transcript_line(line: str, images: list[FrontendImageAttachment]) -> str:
if not images:
return line
noun = "image" if len(images) == 1 else "images"
attachment_line = f"[{len(images)} {noun} attached]"
return f"{line}\n{attachment_line}" if line else attachment_line
async def run_backend_host(
*,
model: str | None = None,
+25 -1
View File
@@ -4,7 +4,7 @@ from __future__ import annotations
from typing import Any, Literal
from pydantic import BaseModel, Field
from pydantic import BaseModel, Field, field_validator
from openharness.state.app_state import AppState
from openharness.bridge.manager import BridgeSessionRecord
@@ -12,6 +12,28 @@ from openharness.mcp.types import McpConnectionStatus
from openharness.tasks.types import TaskRecord
class FrontendImageAttachment(BaseModel):
"""Base64 image payload submitted from the React TUI."""
media_type: str
data: str
source_path: str | None = None
@field_validator("media_type")
@classmethod
def _validate_media_type(cls, value: str) -> str:
if not value.startswith("image/"):
raise ValueError("image attachment media_type must start with image/")
return value
@field_validator("data")
@classmethod
def _validate_data(cls, value: str) -> str:
if not value.strip():
raise ValueError("image attachment data is required")
return value
class FrontendRequest(BaseModel):
"""One request sent from the React frontend to the Python backend."""
@@ -32,6 +54,7 @@ class FrontendRequest(BaseModel):
allowed: bool | None = None
permission_reply: str | None = None
answer: str | None = None
images: list[FrontendImageAttachment] = Field(default_factory=list)
class TranscriptItem(BaseModel):
@@ -217,6 +240,7 @@ def _format_permission_mode(raw: str) -> str:
__all__ = [
"BackendEvent",
"FrontendImageAttachment",
"FrontendRequest",
"TaskSnapshot",
"TranscriptItem",
+7 -3
View File
@@ -583,6 +583,7 @@ async def handle_line(
print_system: SystemPrinter,
render_event: StreamRenderer,
clear_output: ClearHandler,
user_message: ConversationMessage | None = None,
) -> bool:
"""Handle one submitted line for either headless or TUI rendering."""
if not bundle.external_api_client:
@@ -605,7 +606,9 @@ async def handle_line(
memory_backend=bundle.memory_backend,
include_project_memory=bundle.include_project_memory,
)
parsed = bundle.commands.lookup(line) or lookup_skill_slash_command(line, command_context)
parsed = None if user_message is not None else (
bundle.commands.lookup(line) or lookup_skill_slash_command(line, command_context)
)
if parsed is not None:
command, args = parsed
result = await command.handler(
@@ -687,17 +690,18 @@ async def handle_line(
settings = bundle.current_settings()
if bundle.enforce_max_turns:
bundle.engine.set_max_turns(settings.max_turns)
latest_user_prompt = line or (user_message.text if user_message is not None else "")
system_prompt = build_runtime_system_prompt(
settings,
cwd=bundle.cwd,
latest_user_prompt=line,
latest_user_prompt=latest_user_prompt,
extra_skill_dirs=bundle.extra_skill_dirs,
extra_plugin_roots=bundle.extra_plugin_roots,
include_project_memory=bundle.include_project_memory,
)
bundle.engine.set_system_prompt(system_prompt)
try:
async for event in bundle.engine.submit_message(line):
async for event in bundle.engine.submit_message(user_message or line):
await render_event(event)
except MaxTurnsExceeded as exc:
await print_system(f"Stopped after {exc.max_turns} turns (max_turns).")
+136 -5
View File
@@ -12,10 +12,16 @@ import pytest
from openharness.api.client import ApiMessageCompleteEvent
from openharness.api.usage import UsageSnapshot
from openharness.engine.stream_events import CompactProgressEvent
from openharness.engine.messages import ConversationMessage, TextBlock
from openharness.engine.messages import ConversationMessage, ImageBlock, TextBlock
from openharness.config.settings import Settings, save_settings
from openharness.ui.backend_host import BackendHostConfig, ReactBackendHost, run_backend_host
from openharness.ui.protocol import BackendEvent
from openharness.ui.backend_host import (
BackendHostConfig,
ReactBackendHost,
_build_user_message_with_images,
_format_transcript_line,
run_backend_host,
)
from openharness.ui.protocol import BackendEvent, FrontendImageAttachment, FrontendRequest
from openharness.ui.runtime import build_runtime, close_runtime, start_runtime
@@ -34,6 +40,19 @@ class StaticApiClient:
)
class CapturingApiClient(StaticApiClient):
"""Fake streaming client that records model requests."""
def __init__(self, text: str) -> None:
super().__init__(text)
self.requests = []
async def stream_message(self, request):
self.requests.append(request)
async for event in super().stream_message(request):
yield event
class FailingApiClient:
"""Fake client that triggers the query-loop ErrorEvent path."""
@@ -57,6 +76,71 @@ class FakeBinaryStdout:
return None
def test_frontend_request_accepts_image_attachments():
request = FrontendRequest.model_validate(
{
"type": "submit_line",
"line": "describe this",
"images": [
{
"media_type": "image/png",
"data": "aGVsbG8=",
"source_path": "clipboard:clipboard.png",
}
],
}
)
assert request.images[0].media_type == "image/png"
assert request.images[0].data == "aGVsbG8="
def test_frontend_request_rejects_non_image_attachments():
with pytest.raises(ValueError):
FrontendRequest.model_validate(
{
"type": "submit_line",
"images": [
{
"media_type": "text/plain",
"data": "aGVsbG8=",
}
],
}
)
def test_build_user_message_with_images():
message = _build_user_message_with_images(
"What is in this screenshot?",
[
FrontendImageAttachment(
media_type="image/png",
data="aGVsbG8=",
source_path="clipboard:clipboard.png",
)
],
)
assert message is not None
assert message.text == "What is in this screenshot?"
assert isinstance(message.content[1], ImageBlock)
assert message.content[1].media_type == "image/png"
assert message.content[1].source_path == "clipboard:clipboard.png"
def test_format_transcript_line_mentions_attached_images():
assert _format_transcript_line("inspect", []) == "inspect"
assert _format_transcript_line("", [FrontendImageAttachment(media_type="image/png", data="x")]) == "[1 image attached]"
assert _format_transcript_line(
"inspect",
[
FrontendImageAttachment(media_type="image/png", data="x"),
FrontendImageAttachment(media_type="image/jpeg", data="y"),
],
) == "inspect\n[2 images attached]"
@pytest.mark.asyncio
async def test_run_backend_host_accepts_permission_mode(monkeypatch):
captured: dict[str, str | None] = {}
@@ -278,6 +362,51 @@ async def test_backend_host_processes_model_turn(tmp_path, monkeypatch):
)
@pytest.mark.asyncio
async def test_backend_host_processes_image_turn(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
monkeypatch.setenv("OPENHARNESS_CONFIG_DIR", str(tmp_path / "config"))
monkeypatch.setenv("OPENHARNESS_DATA_DIR", str(tmp_path / "data"))
client = CapturingApiClient("image received")
host = ReactBackendHost(BackendHostConfig(api_client=client))
host._bundle = await build_runtime(api_client=client)
events = []
async def _emit(event):
events.append(event)
host._emit = _emit # type: ignore[method-assign]
await start_runtime(host._bundle)
try:
should_continue = await host._process_line(
"",
images=[
FrontendImageAttachment(
media_type="image/png",
data="aGVsbG8=",
source_path="clipboard:clipboard.png",
)
],
)
finally:
await close_runtime(host._bundle)
assert should_continue is True
assert len(client.requests) == 1
message = client.requests[0].messages[-2]
assert message.text == "Please analyze the attached image."
assert isinstance(message.content[1], ImageBlock)
assert message.content[1].data == "aGVsbG8="
assert any(
event.type == "transcript_item"
and event.item
and event.item.role == "user"
and event.item.text == "[1 image attached]"
for event in events
)
@pytest.mark.asyncio
async def test_backend_host_emits_compact_progress_event(tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
@@ -291,8 +420,10 @@ async def test_backend_host_emits_compact_progress_event(tmp_path, monkeypatch):
async def _emit(event):
events.append(event)
async def _fake_handle_line(bundle, line, print_system, render_event, clear_output):
del bundle, line, print_system, clear_output
async def _fake_handle_line(
bundle, line, print_system, render_event, clear_output, user_message=None
):
del bundle, line, print_system, clear_output, user_message
await render_event(
CompactProgressEvent(
phase="compact_start",