@@ -1,6 +1,7 @@
|
||||
import React, {useDeferredValue, useEffect, useMemo, useState} from 'react';
|
||||
import React, {useDeferredValue, useEffect, useMemo, useRef, useState} from 'react';
|
||||
import {Box, Text, useApp, useInput} from 'ink';
|
||||
|
||||
import {readClipboardImage, type ImageAttachment} from './clipboardImage.js';
|
||||
import {CommandPicker} from './components/CommandPicker.js';
|
||||
import {ConversationView} from './components/ConversationView.js';
|
||||
import {ModalHost} from './components/ModalHost.js';
|
||||
@@ -11,7 +12,7 @@ import {SwarmPanel} from './components/SwarmPanel.js';
|
||||
import {TodoPanel} from './components/TodoPanel.js';
|
||||
import {useBackendSession} from './hooks/useBackendSession.js';
|
||||
import {ThemeProvider, useTheme} from './theme/ThemeContext.js';
|
||||
import type {FrontendConfig} from './types.js';
|
||||
import type {FrontendConfig, ImageAttachmentPayload} from './types.js';
|
||||
|
||||
const rawReturnSubmit = process.env.OPENHARNESS_FRONTEND_RAW_RETURN === '1';
|
||||
const scriptedSteps = (() => {
|
||||
@@ -64,6 +65,8 @@ function AppInner({config}: {config: FrontendConfig}): React.JSX.Element {
|
||||
const [modalInput, setModalInput] = useState('');
|
||||
const [history, setHistory] = useState<string[]>([]);
|
||||
const [historyIndex, setHistoryIndex] = useState(-1);
|
||||
const [imageAttachments, setImageAttachments] = useState<ImageAttachment[]>([]);
|
||||
const [clipboardStatus, setClipboardStatus] = useState<string | null>(null);
|
||||
const [lastEscapeAt, setLastEscapeAt] = useState(0);
|
||||
const [scriptIndex, setScriptIndex] = useState(0);
|
||||
const [pickerIndex, setPickerIndex] = useState(0);
|
||||
@@ -77,6 +80,7 @@ function AppInner({config}: {config: FrontendConfig}): React.JSX.Element {
|
||||
const deferredTodoMarkdown = useDeferredValue(session.todoMarkdown);
|
||||
const deferredSwarmTeammates = useDeferredValue(session.swarmTeammates);
|
||||
const deferredSwarmNotifications = useDeferredValue(session.swarmNotifications);
|
||||
const clipboardStatusTimerRef = useRef<NodeJS.Timeout | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const nextTheme = session.status.theme;
|
||||
@@ -85,6 +89,47 @@ function AppInner({config}: {config: FrontendConfig}): React.JSX.Element {
|
||||
}
|
||||
}, [session.status.theme, setThemeName]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (clipboardStatusTimerRef.current) {
|
||||
clearTimeout(clipboardStatusTimerRef.current);
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
const setTemporaryClipboardStatus = (message: string): void => {
|
||||
setClipboardStatus(message);
|
||||
if (clipboardStatusTimerRef.current) {
|
||||
clearTimeout(clipboardStatusTimerRef.current);
|
||||
}
|
||||
clipboardStatusTimerRef.current = setTimeout(() => {
|
||||
setClipboardStatus(null);
|
||||
clipboardStatusTimerRef.current = null;
|
||||
}, 2500);
|
||||
};
|
||||
|
||||
const attachClipboardImage = (): void => {
|
||||
void (async () => {
|
||||
const image = await readClipboardImage();
|
||||
if (!image) {
|
||||
setTemporaryClipboardStatus('No image found in clipboard');
|
||||
return;
|
||||
}
|
||||
setImageAttachments((items) => [...items, image]);
|
||||
setTemporaryClipboardStatus(`Attached ${image.label}`);
|
||||
})().catch((error: unknown) => {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
setTemporaryClipboardStatus(`Clipboard image unavailable: ${message}`);
|
||||
});
|
||||
};
|
||||
|
||||
const imagePayloads = (): ImageAttachmentPayload[] =>
|
||||
imageAttachments.map((image) => ({
|
||||
media_type: image.media_type,
|
||||
data: image.data,
|
||||
source_path: image.source_path,
|
||||
}));
|
||||
|
||||
// Current tool name for spinner
|
||||
const currentToolName = useMemo(() => {
|
||||
for (let i = deferredTranscript.length - 1; i >= 0; i--) {
|
||||
@@ -191,6 +236,11 @@ function AppInner({config}: {config: FrontendConfig}): React.JSX.Element {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!session.busy && key.ctrl && chunk === 'v') {
|
||||
attachClipboardImage();
|
||||
return;
|
||||
}
|
||||
|
||||
// Let ink-text-input handle pasted text directly.
|
||||
if (isPaste) {
|
||||
return;
|
||||
@@ -367,8 +417,9 @@ function AppInner({config}: {config: FrontendConfig}): React.JSX.Element {
|
||||
|
||||
if (isEscape) {
|
||||
const now = Date.now();
|
||||
if (input && now - lastEscapeAt < 500) {
|
||||
if ((input || imageAttachments.length > 0) && now - lastEscapeAt < 500) {
|
||||
setInput('');
|
||||
setImageAttachments([]);
|
||||
setHistoryIndex(-1);
|
||||
setLastEscapeAt(0);
|
||||
return;
|
||||
@@ -408,7 +459,7 @@ function AppInner({config}: {config: FrontendConfig}): React.JSX.Element {
|
||||
setModalInput('');
|
||||
return;
|
||||
}
|
||||
if (!value.trim() || session.busy || !session.ready) {
|
||||
if ((!value.trim() && imageAttachments.length === 0) || session.busy || !session.ready) {
|
||||
if (session.busy && value.trim() === '/stop') {
|
||||
session.sendRequest({type: 'interrupt'});
|
||||
session.setBusyLabel('Stopping current operation...');
|
||||
@@ -417,16 +468,19 @@ function AppInner({config}: {config: FrontendConfig}): React.JSX.Element {
|
||||
return;
|
||||
}
|
||||
// Check if it's an interactive command
|
||||
if (handleCommand(value)) {
|
||||
if (imageAttachments.length === 0 && handleCommand(value)) {
|
||||
setHistory((items) => [...items, value]);
|
||||
setHistoryIndex(-1);
|
||||
setInput('');
|
||||
return;
|
||||
}
|
||||
session.sendRequest({type: 'submit_line', line: value});
|
||||
setHistory((items) => [...items, value]);
|
||||
session.sendRequest({type: 'submit_line', line: value, images: imagePayloads()});
|
||||
if (value.trim()) {
|
||||
setHistory((items) => [...items, value]);
|
||||
}
|
||||
setHistoryIndex(-1);
|
||||
setInput('');
|
||||
setImageAttachments([]);
|
||||
session.setBusy(true);
|
||||
};
|
||||
|
||||
@@ -511,6 +565,8 @@ function AppInner({config}: {config: FrontendConfig}): React.JSX.Element {
|
||||
toolName={session.busy ? currentToolName : undefined}
|
||||
statusLabel={session.busy ? (session.busyLabel ?? (currentToolName ? `Running ${currentToolName}...` : 'Running agent loop...')) : undefined}
|
||||
suppressSubmit={showPicker}
|
||||
imageAttachmentLabels={imageAttachments.map((image) => image.label)}
|
||||
clipboardStatus={clipboardStatus}
|
||||
/>
|
||||
)}
|
||||
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
import {execFile} from 'node:child_process';
|
||||
import {mkdtemp, readFile, rm, stat} from 'node:fs/promises';
|
||||
import {tmpdir} from 'node:os';
|
||||
import {join} from 'node:path';
|
||||
|
||||
export type ImageAttachment = {
|
||||
id: string;
|
||||
label: string;
|
||||
media_type: string;
|
||||
data: string;
|
||||
source_path?: string;
|
||||
size_bytes?: number;
|
||||
};
|
||||
|
||||
const MAX_CLIPBOARD_IMAGE_BYTES = 15 * 1024 * 1024;
|
||||
const EXEC_TIMEOUT_MS = 2500;
|
||||
|
||||
type ClipboardImageRead = {
|
||||
data: Buffer;
|
||||
mediaType: string;
|
||||
label: string;
|
||||
};
|
||||
|
||||
export async function readClipboardImage(): Promise<ImageAttachment | null> {
|
||||
const image = await readClipboardImageData();
|
||||
if (!image) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
id: `clipboard-${Date.now()}-${Math.random().toString(16).slice(2)}`,
|
||||
label: image.label,
|
||||
media_type: image.mediaType,
|
||||
data: image.data.toString('base64'),
|
||||
source_path: `clipboard:${image.label}`,
|
||||
size_bytes: image.data.length,
|
||||
};
|
||||
}
|
||||
|
||||
async function readClipboardImageData(): Promise<ClipboardImageRead | null> {
|
||||
if (process.platform === 'darwin') {
|
||||
return readMacClipboardImage();
|
||||
}
|
||||
if (process.platform === 'win32') {
|
||||
return readWindowsClipboardImage();
|
||||
}
|
||||
return readLinuxClipboardImage();
|
||||
}
|
||||
|
||||
async function readMacClipboardImage(): Promise<ClipboardImageRead | null> {
|
||||
const tempDir = await mkdtemp(join(tmpdir(), 'openharness-clipboard-'));
|
||||
try {
|
||||
const pngPath = join(tempDir, 'clipboard.png');
|
||||
if (await runFileCommand('pngpaste', [pngPath])) {
|
||||
return await readImageFile(pngPath, 'image/png', 'clipboard.png');
|
||||
}
|
||||
if (await writeMacClipboardClass('PNGf', pngPath)) {
|
||||
return await readImageFile(pngPath, 'image/png', 'clipboard.png');
|
||||
}
|
||||
|
||||
const tiffPath = join(tempDir, 'clipboard.tiff');
|
||||
if (await writeMacClipboardClass('TIFF', tiffPath)) {
|
||||
if (await runFileCommand('sips', ['-s', 'format', 'png', tiffPath, '--out', pngPath])) {
|
||||
return await readImageFile(pngPath, 'image/png', 'clipboard.png');
|
||||
}
|
||||
return await readImageFile(tiffPath, 'image/tiff', 'clipboard.tiff');
|
||||
}
|
||||
return null;
|
||||
} finally {
|
||||
await rm(tempDir, {recursive: true, force: true});
|
||||
}
|
||||
}
|
||||
|
||||
async function writeMacClipboardClass(classCode: 'PNGf' | 'TIFF', outputPath: string): Promise<boolean> {
|
||||
const appleClass = `${String.fromCharCode(0xab)}class ${classCode}${String.fromCharCode(0xbb)}`;
|
||||
const escapedPath = outputPath.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
|
||||
return runFileCommand('osascript', [
|
||||
'-e',
|
||||
`set clipboardData to the clipboard as ${appleClass}`,
|
||||
'-e',
|
||||
`set outFile to open for access POSIX file "${escapedPath}" with write permission`,
|
||||
'-e',
|
||||
'set eof outFile to 0',
|
||||
'-e',
|
||||
'write clipboardData to outFile',
|
||||
'-e',
|
||||
'close access outFile',
|
||||
]);
|
||||
}
|
||||
|
||||
async function readWindowsClipboardImage(): Promise<ClipboardImageRead | null> {
|
||||
const tempDir = await mkdtemp(join(tmpdir(), 'openharness-clipboard-'));
|
||||
try {
|
||||
const pngPath = join(tempDir, 'clipboard.png');
|
||||
const escapedPath = pngPath.replace(/'/g, "''");
|
||||
const powershell = join(
|
||||
process.env.SystemRoot ?? 'C:\\Windows',
|
||||
'System32',
|
||||
'WindowsPowerShell',
|
||||
'v1.0',
|
||||
'powershell.exe',
|
||||
);
|
||||
const script = [
|
||||
'Add-Type -AssemblyName System.Windows.Forms',
|
||||
'Add-Type -AssemblyName System.Drawing',
|
||||
'if (-not [Windows.Forms.Clipboard]::ContainsImage()) { exit 2 }',
|
||||
'$image = [Windows.Forms.Clipboard]::GetImage()',
|
||||
`$image.Save('${escapedPath}', [Drawing.Imaging.ImageFormat]::Png)`,
|
||||
].join('; ');
|
||||
if (!(await runFileCommand(powershell, ['-NoProfile', '-STA', '-Command', script]))) {
|
||||
return null;
|
||||
}
|
||||
return await readImageFile(pngPath, 'image/png', 'clipboard.png');
|
||||
} finally {
|
||||
await rm(tempDir, {recursive: true, force: true});
|
||||
}
|
||||
}
|
||||
|
||||
async function readLinuxClipboardImage(): Promise<ClipboardImageRead | null> {
|
||||
const attempts: Array<[string, string[], string, string]> = [
|
||||
['wl-paste', ['--no-newline', '--type', 'image/png'], 'image/png', 'clipboard.png'],
|
||||
['wl-paste', ['--no-newline', '--type', 'image/jpeg'], 'image/jpeg', 'clipboard.jpg'],
|
||||
['xclip', ['-selection', 'clipboard', '-target', 'image/png', '-out'], 'image/png', 'clipboard.png'],
|
||||
['xclip', ['-selection', 'clipboard', '-target', 'image/jpeg', '-out'], 'image/jpeg', 'clipboard.jpg'],
|
||||
['xsel', ['--clipboard', '--output', '--mime-type', 'image/png'], 'image/png', 'clipboard.png'],
|
||||
['xsel', ['--clipboard', '--output', '--mime-type', 'image/jpeg'], 'image/jpeg', 'clipboard.jpg'],
|
||||
];
|
||||
for (const [command, args, mediaType, label] of attempts) {
|
||||
const data = await runBufferCommand(command, args);
|
||||
if (data && data.length > 0) {
|
||||
return {data, mediaType, label};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function readImageFile(path: string, mediaType: string, label: string): Promise<ClipboardImageRead | null> {
|
||||
const fileStat = await stat(path).catch(() => null);
|
||||
if (!fileStat || fileStat.size <= 0 || fileStat.size > MAX_CLIPBOARD_IMAGE_BYTES) {
|
||||
return null;
|
||||
}
|
||||
const data = await readFile(path);
|
||||
return {data, mediaType, label};
|
||||
}
|
||||
|
||||
async function runFileCommand(command: string, args: string[]): Promise<boolean> {
|
||||
return new Promise((resolve) => {
|
||||
execFile(command, args, {timeout: EXEC_TIMEOUT_MS, windowsHide: true}, (error) => {
|
||||
resolve(!error);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function runBufferCommand(command: string, args: string[]): Promise<Buffer | null> {
|
||||
return new Promise((resolve) => {
|
||||
execFile(
|
||||
command,
|
||||
args,
|
||||
{
|
||||
encoding: 'buffer',
|
||||
maxBuffer: MAX_CLIPBOARD_IMAGE_BYTES + 1024,
|
||||
timeout: EXEC_TIMEOUT_MS,
|
||||
windowsHide: true,
|
||||
},
|
||||
(error, stdout) => {
|
||||
if (error || !Buffer.isBuffer(stdout) || stdout.length > MAX_CLIPBOARD_IMAGE_BYTES) {
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
resolve(stdout);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
@@ -173,6 +173,39 @@ test('keeps forward delete behavior when cursor is inside the line', async () =>
|
||||
}
|
||||
});
|
||||
|
||||
test('ignores ctrl+v so the app can attach clipboard images', async () => {
|
||||
const stdin = createTestStdin();
|
||||
const stdout = createTestStdout();
|
||||
let currentValue = '';
|
||||
|
||||
const instance = render(<PromptHarness onInputChange={(value) => {
|
||||
currentValue = value;
|
||||
}} />, {
|
||||
stdin: stdin as unknown as NodeJS.ReadStream & {fd: 0},
|
||||
stdout: stdout as unknown as NodeJS.WriteStream,
|
||||
debug: true,
|
||||
patchConsole: false,
|
||||
});
|
||||
const exitPromise = instance.waitUntilExit();
|
||||
|
||||
try {
|
||||
await nextLoopTurn();
|
||||
|
||||
await sendKey(stdin, Buffer.from([0x16]));
|
||||
await nextLoopTurn();
|
||||
assert.equal(currentValue, '');
|
||||
|
||||
await sendKey(stdin, 'v');
|
||||
await waitForValue(() => currentValue, 'v');
|
||||
} finally {
|
||||
instance.unmount();
|
||||
await exitPromise;
|
||||
instance.cleanup();
|
||||
stdin.destroy();
|
||||
stdout.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('accepts explicit /stop submission while busy', async () => {
|
||||
const stdin = createTestStdin();
|
||||
const stdout = createTestStdout();
|
||||
|
||||
@@ -78,7 +78,14 @@ function MultilineTextInput({
|
||||
return;
|
||||
}
|
||||
|
||||
if (key.upArrow || key.downArrow || key.tab || (key.shift && key.tab) || key.escape || (key.ctrl && input === 'c')) {
|
||||
if (
|
||||
key.upArrow ||
|
||||
key.downArrow ||
|
||||
key.tab ||
|
||||
(key.shift && key.tab) ||
|
||||
key.escape ||
|
||||
(key.ctrl && (input === 'c' || input === 'v'))
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -197,6 +204,8 @@ export function PromptInput({
|
||||
toolName,
|
||||
suppressSubmit,
|
||||
statusLabel,
|
||||
imageAttachmentLabels = [],
|
||||
clipboardStatus,
|
||||
}: {
|
||||
busy: boolean;
|
||||
input: string;
|
||||
@@ -205,6 +214,8 @@ export function PromptInput({
|
||||
toolName?: string;
|
||||
suppressSubmit?: boolean;
|
||||
statusLabel?: string;
|
||||
imageAttachmentLabels?: string[];
|
||||
clipboardStatus?: string | null;
|
||||
}): React.JSX.Element {
|
||||
const {theme} = useTheme();
|
||||
const promptPrefix = busy ? '… ' : '> ';
|
||||
@@ -218,6 +229,18 @@ export function PromptInput({
|
||||
</Box>
|
||||
</Box>
|
||||
) : null}
|
||||
{imageAttachmentLabels.length > 0 ? (
|
||||
<Box>
|
||||
<Text color={theme.colors.accent}>
|
||||
{imageAttachmentLabels.map((label, index) => `[image ${index + 1}: ${label}]`).join(' ')}
|
||||
</Text>
|
||||
</Box>
|
||||
) : null}
|
||||
{clipboardStatus ? (
|
||||
<Box>
|
||||
<Text color={theme.colors.muted}>{clipboardStatus}</Text>
|
||||
</Box>
|
||||
) : null}
|
||||
<MultilineTextInput
|
||||
value={input}
|
||||
onChange={setInput}
|
||||
|
||||
@@ -11,6 +11,12 @@ export type TranscriptItem = {
|
||||
is_error?: boolean;
|
||||
};
|
||||
|
||||
export type ImageAttachmentPayload = {
|
||||
media_type: string;
|
||||
data: string;
|
||||
source_path?: string;
|
||||
};
|
||||
|
||||
export type TaskSnapshot = {
|
||||
id: string;
|
||||
type: string;
|
||||
|
||||
@@ -19,6 +19,7 @@ from openharness.commands import MemoryCommandBackend
|
||||
from openharness.config.settings import CLAUDE_MODEL_ALIAS_OPTIONS, resolve_model_setting
|
||||
from openharness.bridge import get_bridge_manager
|
||||
from openharness.coordinator.coordinator_mode import is_coordinator_mode
|
||||
from openharness.engine.messages import ConversationMessage, ImageBlock, TextBlock
|
||||
from openharness.themes import list_themes
|
||||
from openharness.engine.stream_events import (
|
||||
AssistantTextDelta,
|
||||
@@ -33,7 +34,7 @@ from openharness.engine.stream_events import (
|
||||
from openharness.output_styles import load_output_styles
|
||||
from openharness.tasks import get_task_manager
|
||||
from openharness.ui.coordinator_drain import drain_coordinator_async_agents
|
||||
from openharness.ui.protocol import BackendEvent, FrontendRequest, TranscriptItem
|
||||
from openharness.ui.protocol import BackendEvent, FrontendImageAttachment, FrontendRequest, TranscriptItem
|
||||
from openharness.ui.runtime import build_runtime, close_runtime, handle_line, start_runtime
|
||||
from openharness.services.session_backend import SessionBackend
|
||||
|
||||
@@ -166,11 +167,13 @@ class ReactBackendHost:
|
||||
await self._emit(BackendEvent(type="error", message="Session is busy"))
|
||||
continue
|
||||
line = (request.line or "").strip()
|
||||
if not line:
|
||||
if not line and not request.images:
|
||||
continue
|
||||
self._busy = True
|
||||
try:
|
||||
should_continue = await self._run_active_request(self._process_line(line))
|
||||
should_continue = await self._run_active_request(
|
||||
self._process_line(line, images=request.images)
|
||||
)
|
||||
finally:
|
||||
self._busy = False
|
||||
if not should_continue:
|
||||
@@ -244,10 +247,23 @@ class ReactBackendHost:
|
||||
return
|
||||
task.cancel()
|
||||
|
||||
async def _process_line(self, line: str, *, transcript_line: str | None = None) -> bool:
|
||||
async def _process_line(
|
||||
self,
|
||||
line: str,
|
||||
*,
|
||||
transcript_line: str | None = None,
|
||||
images: list[FrontendImageAttachment] | None = None,
|
||||
) -> bool:
|
||||
assert self._bundle is not None
|
||||
user_message = _build_user_message_with_images(line, images or [])
|
||||
await self._emit(
|
||||
BackendEvent(type="transcript_item", item=TranscriptItem(role="user", text=transcript_line or line))
|
||||
BackendEvent(
|
||||
type="transcript_item",
|
||||
item=TranscriptItem(
|
||||
role="user",
|
||||
text=transcript_line or _format_transcript_line(line, images or []),
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
async def _print_system(message: str) -> None:
|
||||
@@ -358,6 +374,7 @@ class ReactBackendHost:
|
||||
print_system=_print_system,
|
||||
render_event=_render_event,
|
||||
clear_output=_clear_output,
|
||||
user_message=user_message,
|
||||
)
|
||||
if is_coordinator_mode():
|
||||
await drain_coordinator_async_agents(
|
||||
@@ -837,6 +854,32 @@ class ReactBackendHost:
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def _build_user_message_with_images(
|
||||
line: str,
|
||||
images: list[FrontendImageAttachment],
|
||||
) -> ConversationMessage | None:
|
||||
if not images:
|
||||
return None
|
||||
content = [TextBlock(text=line or "Please analyze the attached image.")]
|
||||
content.extend(
|
||||
ImageBlock(
|
||||
media_type=image.media_type,
|
||||
data=image.data,
|
||||
source_path=image.source_path or "",
|
||||
)
|
||||
for image in images
|
||||
)
|
||||
return ConversationMessage.from_user_content(content)
|
||||
|
||||
|
||||
def _format_transcript_line(line: str, images: list[FrontendImageAttachment]) -> str:
|
||||
if not images:
|
||||
return line
|
||||
noun = "image" if len(images) == 1 else "images"
|
||||
attachment_line = f"[{len(images)} {noun} attached]"
|
||||
return f"{line}\n{attachment_line}" if line else attachment_line
|
||||
|
||||
|
||||
async def run_backend_host(
|
||||
*,
|
||||
model: str | None = None,
|
||||
|
||||
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
from openharness.state.app_state import AppState
|
||||
from openharness.bridge.manager import BridgeSessionRecord
|
||||
@@ -12,6 +12,28 @@ from openharness.mcp.types import McpConnectionStatus
|
||||
from openharness.tasks.types import TaskRecord
|
||||
|
||||
|
||||
class FrontendImageAttachment(BaseModel):
|
||||
"""Base64 image payload submitted from the React TUI."""
|
||||
|
||||
media_type: str
|
||||
data: str
|
||||
source_path: str | None = None
|
||||
|
||||
@field_validator("media_type")
|
||||
@classmethod
|
||||
def _validate_media_type(cls, value: str) -> str:
|
||||
if not value.startswith("image/"):
|
||||
raise ValueError("image attachment media_type must start with image/")
|
||||
return value
|
||||
|
||||
@field_validator("data")
|
||||
@classmethod
|
||||
def _validate_data(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("image attachment data is required")
|
||||
return value
|
||||
|
||||
|
||||
class FrontendRequest(BaseModel):
|
||||
"""One request sent from the React frontend to the Python backend."""
|
||||
|
||||
@@ -32,6 +54,7 @@ class FrontendRequest(BaseModel):
|
||||
allowed: bool | None = None
|
||||
permission_reply: str | None = None
|
||||
answer: str | None = None
|
||||
images: list[FrontendImageAttachment] = Field(default_factory=list)
|
||||
|
||||
|
||||
class TranscriptItem(BaseModel):
|
||||
@@ -217,6 +240,7 @@ def _format_permission_mode(raw: str) -> str:
|
||||
|
||||
__all__ = [
|
||||
"BackendEvent",
|
||||
"FrontendImageAttachment",
|
||||
"FrontendRequest",
|
||||
"TaskSnapshot",
|
||||
"TranscriptItem",
|
||||
|
||||
@@ -583,6 +583,7 @@ async def handle_line(
|
||||
print_system: SystemPrinter,
|
||||
render_event: StreamRenderer,
|
||||
clear_output: ClearHandler,
|
||||
user_message: ConversationMessage | None = None,
|
||||
) -> bool:
|
||||
"""Handle one submitted line for either headless or TUI rendering."""
|
||||
if not bundle.external_api_client:
|
||||
@@ -605,7 +606,9 @@ async def handle_line(
|
||||
memory_backend=bundle.memory_backend,
|
||||
include_project_memory=bundle.include_project_memory,
|
||||
)
|
||||
parsed = bundle.commands.lookup(line) or lookup_skill_slash_command(line, command_context)
|
||||
parsed = None if user_message is not None else (
|
||||
bundle.commands.lookup(line) or lookup_skill_slash_command(line, command_context)
|
||||
)
|
||||
if parsed is not None:
|
||||
command, args = parsed
|
||||
result = await command.handler(
|
||||
@@ -687,17 +690,18 @@ async def handle_line(
|
||||
settings = bundle.current_settings()
|
||||
if bundle.enforce_max_turns:
|
||||
bundle.engine.set_max_turns(settings.max_turns)
|
||||
latest_user_prompt = line or (user_message.text if user_message is not None else "")
|
||||
system_prompt = build_runtime_system_prompt(
|
||||
settings,
|
||||
cwd=bundle.cwd,
|
||||
latest_user_prompt=line,
|
||||
latest_user_prompt=latest_user_prompt,
|
||||
extra_skill_dirs=bundle.extra_skill_dirs,
|
||||
extra_plugin_roots=bundle.extra_plugin_roots,
|
||||
include_project_memory=bundle.include_project_memory,
|
||||
)
|
||||
bundle.engine.set_system_prompt(system_prompt)
|
||||
try:
|
||||
async for event in bundle.engine.submit_message(line):
|
||||
async for event in bundle.engine.submit_message(user_message or line):
|
||||
await render_event(event)
|
||||
except MaxTurnsExceeded as exc:
|
||||
await print_system(f"Stopped after {exc.max_turns} turns (max_turns).")
|
||||
|
||||
@@ -12,10 +12,16 @@ import pytest
|
||||
from openharness.api.client import ApiMessageCompleteEvent
|
||||
from openharness.api.usage import UsageSnapshot
|
||||
from openharness.engine.stream_events import CompactProgressEvent
|
||||
from openharness.engine.messages import ConversationMessage, TextBlock
|
||||
from openharness.engine.messages import ConversationMessage, ImageBlock, TextBlock
|
||||
from openharness.config.settings import Settings, save_settings
|
||||
from openharness.ui.backend_host import BackendHostConfig, ReactBackendHost, run_backend_host
|
||||
from openharness.ui.protocol import BackendEvent
|
||||
from openharness.ui.backend_host import (
|
||||
BackendHostConfig,
|
||||
ReactBackendHost,
|
||||
_build_user_message_with_images,
|
||||
_format_transcript_line,
|
||||
run_backend_host,
|
||||
)
|
||||
from openharness.ui.protocol import BackendEvent, FrontendImageAttachment, FrontendRequest
|
||||
from openharness.ui.runtime import build_runtime, close_runtime, start_runtime
|
||||
|
||||
|
||||
@@ -34,6 +40,19 @@ class StaticApiClient:
|
||||
)
|
||||
|
||||
|
||||
class CapturingApiClient(StaticApiClient):
|
||||
"""Fake streaming client that records model requests."""
|
||||
|
||||
def __init__(self, text: str) -> None:
|
||||
super().__init__(text)
|
||||
self.requests = []
|
||||
|
||||
async def stream_message(self, request):
|
||||
self.requests.append(request)
|
||||
async for event in super().stream_message(request):
|
||||
yield event
|
||||
|
||||
|
||||
class FailingApiClient:
|
||||
"""Fake client that triggers the query-loop ErrorEvent path."""
|
||||
|
||||
@@ -57,6 +76,71 @@ class FakeBinaryStdout:
|
||||
return None
|
||||
|
||||
|
||||
def test_frontend_request_accepts_image_attachments():
|
||||
request = FrontendRequest.model_validate(
|
||||
{
|
||||
"type": "submit_line",
|
||||
"line": "describe this",
|
||||
"images": [
|
||||
{
|
||||
"media_type": "image/png",
|
||||
"data": "aGVsbG8=",
|
||||
"source_path": "clipboard:clipboard.png",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
assert request.images[0].media_type == "image/png"
|
||||
assert request.images[0].data == "aGVsbG8="
|
||||
|
||||
|
||||
def test_frontend_request_rejects_non_image_attachments():
|
||||
with pytest.raises(ValueError):
|
||||
FrontendRequest.model_validate(
|
||||
{
|
||||
"type": "submit_line",
|
||||
"images": [
|
||||
{
|
||||
"media_type": "text/plain",
|
||||
"data": "aGVsbG8=",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_build_user_message_with_images():
|
||||
message = _build_user_message_with_images(
|
||||
"What is in this screenshot?",
|
||||
[
|
||||
FrontendImageAttachment(
|
||||
media_type="image/png",
|
||||
data="aGVsbG8=",
|
||||
source_path="clipboard:clipboard.png",
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert message is not None
|
||||
assert message.text == "What is in this screenshot?"
|
||||
assert isinstance(message.content[1], ImageBlock)
|
||||
assert message.content[1].media_type == "image/png"
|
||||
assert message.content[1].source_path == "clipboard:clipboard.png"
|
||||
|
||||
|
||||
def test_format_transcript_line_mentions_attached_images():
|
||||
assert _format_transcript_line("inspect", []) == "inspect"
|
||||
assert _format_transcript_line("", [FrontendImageAttachment(media_type="image/png", data="x")]) == "[1 image attached]"
|
||||
assert _format_transcript_line(
|
||||
"inspect",
|
||||
[
|
||||
FrontendImageAttachment(media_type="image/png", data="x"),
|
||||
FrontendImageAttachment(media_type="image/jpeg", data="y"),
|
||||
],
|
||||
) == "inspect\n[2 images attached]"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_backend_host_accepts_permission_mode(monkeypatch):
|
||||
captured: dict[str, str | None] = {}
|
||||
@@ -278,6 +362,51 @@ async def test_backend_host_processes_model_turn(tmp_path, monkeypatch):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backend_host_processes_image_turn(tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setenv("OPENHARNESS_CONFIG_DIR", str(tmp_path / "config"))
|
||||
monkeypatch.setenv("OPENHARNESS_DATA_DIR", str(tmp_path / "data"))
|
||||
|
||||
client = CapturingApiClient("image received")
|
||||
host = ReactBackendHost(BackendHostConfig(api_client=client))
|
||||
host._bundle = await build_runtime(api_client=client)
|
||||
events = []
|
||||
|
||||
async def _emit(event):
|
||||
events.append(event)
|
||||
|
||||
host._emit = _emit # type: ignore[method-assign]
|
||||
await start_runtime(host._bundle)
|
||||
try:
|
||||
should_continue = await host._process_line(
|
||||
"",
|
||||
images=[
|
||||
FrontendImageAttachment(
|
||||
media_type="image/png",
|
||||
data="aGVsbG8=",
|
||||
source_path="clipboard:clipboard.png",
|
||||
)
|
||||
],
|
||||
)
|
||||
finally:
|
||||
await close_runtime(host._bundle)
|
||||
|
||||
assert should_continue is True
|
||||
assert len(client.requests) == 1
|
||||
message = client.requests[0].messages[-2]
|
||||
assert message.text == "Please analyze the attached image."
|
||||
assert isinstance(message.content[1], ImageBlock)
|
||||
assert message.content[1].data == "aGVsbG8="
|
||||
assert any(
|
||||
event.type == "transcript_item"
|
||||
and event.item
|
||||
and event.item.role == "user"
|
||||
and event.item.text == "[1 image attached]"
|
||||
for event in events
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backend_host_emits_compact_progress_event(tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
@@ -291,8 +420,10 @@ async def test_backend_host_emits_compact_progress_event(tmp_path, monkeypatch):
|
||||
async def _emit(event):
|
||||
events.append(event)
|
||||
|
||||
async def _fake_handle_line(bundle, line, print_system, render_event, clear_output):
|
||||
del bundle, line, print_system, clear_output
|
||||
async def _fake_handle_line(
|
||||
bundle, line, print_system, render_event, clear_output, user_message=None
|
||||
):
|
||||
del bundle, line, print_system, clear_output, user_message
|
||||
await render_event(
|
||||
CompactProgressEvent(
|
||||
phase="compact_start",
|
||||
|
||||
Reference in New Issue
Block a user