wehub-resource-sync
410 lines
16 KiB
TypeScript
410 lines
16 KiB
TypeScript
import { BACKGROUND_MESSAGE_TYPES } from '@/common/message-types';
|
|
import type {
|
|
UpsertMarkerRequest,
|
|
ElementMarker,
|
|
MarkerValidationRequest,
|
|
MarkerValidationAction,
|
|
} from '@/common/element-marker-types';
|
|
import {
|
|
deleteMarker,
|
|
listAllMarkers,
|
|
listMarkersForUrl,
|
|
saveMarker,
|
|
updateMarker,
|
|
} from './element-marker-storage';
|
|
import { computerTool } from '@/entrypoints/background/tools/browser/computer';
|
|
import { clickTool } from '@/entrypoints/background/tools/browser/interaction';
|
|
import { keyboardTool } from '@/entrypoints/background/tools/browser/keyboard';
|
|
|
|
const CONTEXT_MENU_ID = 'element_marker_mark';
|
|
|
|
/**
|
|
* Extract error message from MCP tool result
|
|
*/
|
|
function extractToolError(result: any): string | undefined {
|
|
if (!result) return undefined;
|
|
|
|
// Check for error in result content array
|
|
if (Array.isArray(result.content)) {
|
|
for (const item of result.content) {
|
|
if (item?.text) {
|
|
try {
|
|
const parsed = JSON.parse(item.text);
|
|
if (parsed?.error) return parsed.error;
|
|
if (parsed?.message) return parsed.message;
|
|
} catch {
|
|
// Not JSON, use as-is
|
|
return item.text;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Fallback to direct error field
|
|
return result.error || (result.isError ? 'unknown tool error' : undefined);
|
|
}
|
|
|
|
async function ensureContextMenu() {
|
|
try {
|
|
// Guard: contextMenus permission may be missing
|
|
if (!(chrome as any).contextMenus?.create) return;
|
|
// Remove and re-create our single menu to avoid duplication
|
|
try {
|
|
await chrome.contextMenus.remove(CONTEXT_MENU_ID);
|
|
} catch {}
|
|
await chrome.contextMenus.create({
|
|
id: CONTEXT_MENU_ID,
|
|
title: '标注元素',
|
|
contexts: ['all'],
|
|
});
|
|
} catch (e) {
|
|
console.warn('ElementMarker: ensureContextMenu failed:', e);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check if element-marker.js is already injected in the tab
|
|
* Uses a short timeout to avoid hanging on unresponsive tabs
|
|
*/
|
|
async function isMarkerInjected(tabId: number): Promise<boolean> {
|
|
try {
|
|
const response = await Promise.race([
|
|
chrome.tabs.sendMessage(tabId, { action: 'element_marker_ping' }),
|
|
new Promise<null>((resolve) => setTimeout(() => resolve(null), 300)),
|
|
]);
|
|
return response?.status === 'pong';
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Inject element-marker.js into the tab if not already injected
|
|
*/
|
|
async function injectMarkerHelper(tabId: number) {
|
|
// Check if already injected via ping
|
|
const alreadyInjected = await isMarkerInjected(tabId);
|
|
|
|
if (!alreadyInjected) {
|
|
try {
|
|
await chrome.scripting.executeScript({
|
|
target: { tabId, allFrames: true },
|
|
files: ['inject-scripts/element-marker.js'],
|
|
world: 'ISOLATED',
|
|
} as any);
|
|
} catch (e) {
|
|
// Script injection may fail on some pages (e.g., chrome:// URLs)
|
|
console.warn('ElementMarker: script injection failed:', e);
|
|
}
|
|
}
|
|
|
|
try {
|
|
await chrome.tabs.sendMessage(tabId, { action: 'element_marker_start' } as any);
|
|
} catch (e) {
|
|
console.warn('ElementMarker: start overlay failed:', e);
|
|
}
|
|
}
|
|
|
|
export function initElementMarkerListeners() {
|
|
// Ensure context menu on startup
|
|
ensureContextMenu().catch(() => {});
|
|
|
|
// Respond to RR triggers refresh by re-ensuring our menu a bit later
|
|
chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {
|
|
try {
|
|
switch (message?.type) {
|
|
// Handle element marker start from popup
|
|
case BACKGROUND_MESSAGE_TYPES.ELEMENT_MARKER_START: {
|
|
const tabId = message.tabId;
|
|
if (typeof tabId !== 'number') {
|
|
sendResponse({ success: false, error: 'invalid tabId' });
|
|
return true;
|
|
}
|
|
injectMarkerHelper(tabId)
|
|
.then(() => sendResponse({ success: true }))
|
|
.catch((e) => sendResponse({ success: false, error: e?.message || String(e) }));
|
|
return true;
|
|
}
|
|
case BACKGROUND_MESSAGE_TYPES.ELEMENT_MARKER_LIST_ALL: {
|
|
listAllMarkers()
|
|
.then((markers) => sendResponse({ success: true, markers }))
|
|
.catch((e) => sendResponse({ success: false, error: e?.message || String(e) }));
|
|
return true;
|
|
}
|
|
case BACKGROUND_MESSAGE_TYPES.ELEMENT_MARKER_LIST_FOR_URL: {
|
|
const url = String(message.url || '');
|
|
listMarkersForUrl(url)
|
|
.then((markers) => sendResponse({ success: true, markers }))
|
|
.catch((e) => sendResponse({ success: false, error: e?.message || String(e) }));
|
|
return true;
|
|
}
|
|
case BACKGROUND_MESSAGE_TYPES.ELEMENT_MARKER_SAVE: {
|
|
const req = message.marker as UpsertMarkerRequest;
|
|
saveMarker(req)
|
|
.then((marker) => sendResponse({ success: true, marker }))
|
|
.catch((e) => sendResponse({ success: false, error: e?.message || String(e) }));
|
|
return true;
|
|
}
|
|
case BACKGROUND_MESSAGE_TYPES.ELEMENT_MARKER_UPDATE: {
|
|
const marker = message.marker as ElementMarker;
|
|
updateMarker(marker)
|
|
.then(() => sendResponse({ success: true }))
|
|
.catch((e) => sendResponse({ success: false, error: e?.message || String(e) }));
|
|
return true;
|
|
}
|
|
case BACKGROUND_MESSAGE_TYPES.ELEMENT_MARKER_DELETE: {
|
|
const id = String(message.id || '');
|
|
if (!id) {
|
|
sendResponse({ success: false, error: 'invalid id' });
|
|
return true;
|
|
}
|
|
deleteMarker(id)
|
|
.then(() => sendResponse({ success: true }))
|
|
.catch((e) => sendResponse({ success: false, error: e?.message || String(e) }));
|
|
return true;
|
|
}
|
|
case BACKGROUND_MESSAGE_TYPES.ELEMENT_MARKER_VALIDATE: {
|
|
// Validate via MCP tool chain
|
|
(async () => {
|
|
const req = message as {
|
|
selector: string;
|
|
selectorType?: 'css' | 'xpath';
|
|
action: MarkerValidationAction;
|
|
listMode?: boolean;
|
|
text?: string;
|
|
keys?: string;
|
|
button?: 'left' | 'right' | 'middle';
|
|
bubbles?: boolean;
|
|
cancelable?: boolean;
|
|
modifiers?: any;
|
|
coordinates?: { x: number; y: number };
|
|
offsetX?: number;
|
|
offsetY?: number;
|
|
relativeTo?: 'element' | 'viewport';
|
|
};
|
|
// enrich typing with optional nav + scroll params
|
|
(req as any).waitForNavigation = (message as any).waitForNavigation;
|
|
(req as any).timeoutMs = (message as any).timeoutMs;
|
|
(req as any).scrollDirection = (message as any).scrollDirection;
|
|
(req as any).scrollAmount = (message as any).scrollAmount;
|
|
const selector = String(req.selector || '').trim();
|
|
const selectorType = (req.selectorType || 'css') as 'css' | 'xpath';
|
|
const action = req.action as MarkerValidationAction;
|
|
if (!selector) return sendResponse({ success: false, error: 'selector is required' });
|
|
const tabs = await chrome.tabs.query({ active: true, currentWindow: true });
|
|
const tab = tabs[0];
|
|
if (!tab?.id) return sendResponse({ success: false, error: 'active tab not found' });
|
|
|
|
// 1) Ensure helper
|
|
try {
|
|
await chrome.scripting.executeScript({
|
|
target: { tabId: tab.id, allFrames: true },
|
|
files: ['inject-scripts/accessibility-tree-helper.js'],
|
|
world: 'ISOLATED',
|
|
} as any);
|
|
} catch {}
|
|
|
|
// 2) Resolve selector -> ref/center via helper (same as tools)
|
|
let ensured: any;
|
|
try {
|
|
ensured = await chrome.tabs.sendMessage(tab.id, {
|
|
action: 'ensureRefForSelector',
|
|
selector,
|
|
isXPath: selectorType === 'xpath',
|
|
allowMultiple: !!req.listMode,
|
|
} as any);
|
|
} catch (e) {
|
|
return sendResponse({
|
|
success: false,
|
|
error: String(e instanceof Error ? e.message : e),
|
|
});
|
|
}
|
|
if (!ensured || !ensured.success || !ensured.ref) {
|
|
return sendResponse({
|
|
success: false,
|
|
error: ensured?.error || 'failed to resolve selector',
|
|
});
|
|
}
|
|
|
|
const base = {
|
|
success: true,
|
|
resolved: true,
|
|
ref: ensured.ref,
|
|
center: ensured.center,
|
|
} as any;
|
|
|
|
// Compute optional coordinates from offsets
|
|
let coords: { x: number; y: number } | undefined = undefined;
|
|
if (
|
|
req.coordinates &&
|
|
typeof req.coordinates.x === 'number' &&
|
|
typeof req.coordinates.y === 'number'
|
|
) {
|
|
coords = { x: Math.round(req.coordinates.x), y: Math.round(req.coordinates.y) };
|
|
} else if (
|
|
req.relativeTo === 'element' &&
|
|
ensured.center &&
|
|
(typeof req.offsetX === 'number' || typeof req.offsetY === 'number')
|
|
) {
|
|
const dx = Number.isFinite(req.offsetX as any) ? (req.offsetX as number) : 0;
|
|
const dy = Number.isFinite(req.offsetY as any) ? (req.offsetY as number) : 0;
|
|
coords = { x: ensured.center.x + dx, y: ensured.center.y + dy };
|
|
}
|
|
|
|
// 3) Dispatch to appropriate tool for end-to-end validation
|
|
try {
|
|
switch (action) {
|
|
case 'hover': {
|
|
const r = await computerTool.execute(
|
|
coords
|
|
? { action: 'hover', coordinates: coords }
|
|
: ({ action: 'hover', ref: ensured.ref } as any),
|
|
);
|
|
const error = r.isError ? extractToolError(r) : undefined;
|
|
base.tool = { name: 'computer.hover', ok: !r.isError, error };
|
|
break;
|
|
}
|
|
case 'left_click': {
|
|
const r = await clickTool.execute({
|
|
...(coords ? { coordinates: coords } : { ref: ensured.ref }),
|
|
waitForNavigation: !!req.waitForNavigation,
|
|
timeout: Number.isFinite(req.timeoutMs as any)
|
|
? (req.timeoutMs as number)
|
|
: 3000,
|
|
button: (req.button || 'left') as any,
|
|
modifiers: req.modifiers || {},
|
|
} as any);
|
|
const error = r.isError ? extractToolError(r) : undefined;
|
|
base.tool = { name: 'interaction.click', ok: !r.isError, error };
|
|
break;
|
|
}
|
|
case 'double_click': {
|
|
const r = await clickTool.execute({
|
|
...(coords ? { coordinates: coords } : { ref: ensured.ref }),
|
|
double: true,
|
|
waitForNavigation: !!req.waitForNavigation,
|
|
timeout: Number.isFinite(req.timeoutMs as any)
|
|
? (req.timeoutMs as number)
|
|
: 3000,
|
|
button: (req.button || 'left') as any,
|
|
modifiers: req.modifiers || {},
|
|
} as any);
|
|
const error = r.isError ? extractToolError(r) : undefined;
|
|
base.tool = { name: 'interaction.click(double)', ok: !r.isError, error };
|
|
break;
|
|
}
|
|
case 'right_click': {
|
|
const r = await clickTool.execute({
|
|
...(coords ? { coordinates: coords } : { ref: ensured.ref }),
|
|
waitForNavigation: !!req.waitForNavigation,
|
|
timeout: Number.isFinite(req.timeoutMs as any)
|
|
? (req.timeoutMs as number)
|
|
: 3000,
|
|
button: 'right',
|
|
modifiers: req.modifiers || {},
|
|
} as any);
|
|
const error = r.isError ? extractToolError(r) : undefined;
|
|
base.tool = { name: 'interaction.click(right)', ok: !r.isError, error };
|
|
break;
|
|
}
|
|
case 'scroll': {
|
|
const direction = (req as any).scrollDirection || 'down';
|
|
const amount = Number.isFinite((req as any).scrollAmount)
|
|
? Number((req as any).scrollAmount)
|
|
: 300;
|
|
const payload = coords
|
|
? {
|
|
action: 'scroll',
|
|
scrollDirection: direction,
|
|
scrollAmount: amount,
|
|
coordinates: coords,
|
|
}
|
|
: ({
|
|
action: 'scroll',
|
|
scrollDirection: direction,
|
|
scrollAmount: amount,
|
|
ref: ensured.ref,
|
|
} as any);
|
|
const r = await computerTool.execute(payload as any);
|
|
const error = r.isError ? extractToolError(r) : undefined;
|
|
base.tool = { name: 'computer.scroll', ok: !r.isError, error };
|
|
break;
|
|
}
|
|
case 'type_text': {
|
|
const text = String(req.text || '');
|
|
const r = await computerTool.execute({ action: 'type', ref: ensured.ref, text });
|
|
const error = r.isError ? extractToolError(r) : undefined;
|
|
base.tool = { name: 'computer.type', ok: !r.isError, error };
|
|
break;
|
|
}
|
|
case 'press_keys': {
|
|
const keys = String(req.keys || '');
|
|
// Focus first by ref to ensure key target
|
|
try {
|
|
await clickTool.execute({
|
|
ref: ensured.ref,
|
|
waitForNavigation: false,
|
|
timeout: 2000,
|
|
});
|
|
} catch {}
|
|
const r = await keyboardTool.execute({ keys, delay: 0 } as any);
|
|
const error = r.isError ? extractToolError(r) : undefined;
|
|
base.tool = { name: 'keyboard.simulate', ok: !r.isError, error };
|
|
break;
|
|
}
|
|
default: {
|
|
base.tool = { name: 'noop', ok: true };
|
|
}
|
|
}
|
|
} catch (e) {
|
|
console.warn('[ElementMarker] Validation failed before tool execution', e);
|
|
base.tool = {
|
|
name: 'unknown',
|
|
ok: false,
|
|
error: String(e instanceof Error ? e.message : e),
|
|
};
|
|
}
|
|
|
|
// Log tool failures for debugging
|
|
if (base.tool && base.tool.ok === false) {
|
|
console.warn('[ElementMarker] Tool validation failure', {
|
|
action,
|
|
toolName: base.tool.name,
|
|
error: base.tool.error,
|
|
selector,
|
|
selectorType,
|
|
});
|
|
}
|
|
|
|
return sendResponse(base);
|
|
})();
|
|
return true;
|
|
}
|
|
// When RR refresh (or similar) happens, re-add our menu
|
|
case BACKGROUND_MESSAGE_TYPES.RR_REFRESH_TRIGGERS:
|
|
case BACKGROUND_MESSAGE_TYPES.RR_SAVE_TRIGGER:
|
|
case BACKGROUND_MESSAGE_TYPES.RR_DELETE_TRIGGER: {
|
|
setTimeout(() => ensureContextMenu().catch(() => {}), 300);
|
|
break;
|
|
}
|
|
}
|
|
} catch (e) {
|
|
sendResponse({ success: false, error: (e as any)?.message || String(e) });
|
|
}
|
|
return false;
|
|
});
|
|
|
|
// Context menu click routing
|
|
if ((chrome as any).contextMenus?.onClicked?.addListener) {
|
|
chrome.contextMenus.onClicked.addListener(async (info, tab) => {
|
|
try {
|
|
if (info.menuItemId === CONTEXT_MENU_ID && tab?.id) {
|
|
await injectMarkerHelper(tab.id);
|
|
}
|
|
} catch (e) {
|
|
console.warn('ElementMarker: context menu click failed:', e);
|
|
}
|
|
});
|
|
}
|
|
}
|