chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,248 @@
|
||||
"""L4 Session Log Processor — compress & extract history.
|
||||
Format A (JSON): kept as-is. Format B (Raw): strip sys prompt & assistant echo.
|
||||
"""
|
||||
import re, os, json, ast
|
||||
from datetime import datetime
|
||||
|
||||
L4_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
_RE_PROMPT = re.compile(r'^=== Prompt ===(?: (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}))?', re.M)
|
||||
_RE_RESPONSE = re.compile(r'^=== Response ===(?: (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}))?', re.M)
|
||||
_RE_USER = re.compile(r'^=== USER ===$', re.M)
|
||||
_RE_ASST = re.compile(r'^=== ASSISTANT ===$', re.M)
|
||||
_RE_ANY_MARKER = re.compile(r'^=== (?:Prompt|Response|USER|ASSISTANT) ===(?:.*)?$', re.M)
|
||||
|
||||
def _ts_fmt(ts_str):
|
||||
"""'2026-04-03 20:13:06' → '0403_2013'"""
|
||||
try: return datetime.strptime(ts_str.strip(), '%Y-%m-%d %H:%M:%S').strftime('%m%d_%H%M')
|
||||
except Exception: return None
|
||||
|
||||
def _detect_format(text):
|
||||
"""Detect A (json) vs B (raw) by checking content after first Prompt marker."""
|
||||
m = _RE_PROMPT.search(text)
|
||||
if not m: return 'unknown'
|
||||
return 'json' if re.match(r'\s*\{', text[m.end():m.end()+200]) else 'raw'
|
||||
|
||||
def _parse_sections(text):
|
||||
"""Split text into (type, marker_line, body) tuples."""
|
||||
markers = list(_RE_ANY_MARKER.finditer(text))
|
||||
if not markers:
|
||||
return [('preamble', '', text)]
|
||||
_MAP = {'Prompt': 'prompt', 'Response': 'response', 'USER': 'user', 'ASSISTANT': 'assistant'}
|
||||
sections = []
|
||||
if markers[0].start() > 0:
|
||||
sections.append(('preamble', '', text[:markers[0].start()]))
|
||||
for i, m in enumerate(markers):
|
||||
line = m.group()
|
||||
end = markers[i+1].start() if i+1 < len(markers) else len(text)
|
||||
typ = next((v for k, v in _MAP.items() if line.startswith(f'=== {k}')), None)
|
||||
if typ:
|
||||
sections.append((typ, line, text[m.end():end]))
|
||||
return sections
|
||||
|
||||
def compress_session(src, dst_dir=None):
|
||||
"""Compress model_responses_xxx.txt → MMDD_HHMM-MMDD_HHMM.txt. Returns (dst, stats) or (None, reason)."""
|
||||
dst_dir = dst_dir or L4_DIR
|
||||
with open(src, 'r', encoding='utf-8', errors='replace') as f:
|
||||
text = f.read()
|
||||
timestamps = [m.group(1) for m in _RE_PROMPT.finditer(text) if m.group(1)]
|
||||
if not timestamps: # fallback to Response timestamps
|
||||
timestamps = [m.group(1) for m in _RE_RESPONSE.finditer(text) if m.group(1)]
|
||||
if not timestamps:
|
||||
return None, 'no timestamps found'
|
||||
ts_first, ts_last = _ts_fmt(timestamps[0]), _ts_fmt(timestamps[-1])
|
||||
if not ts_first:
|
||||
return None, 'bad timestamp format'
|
||||
name = f"{ts_first}-{ts_last or ts_first}.txt"
|
||||
fmt = _detect_format(text)
|
||||
compressed = _compress_raw(text) if fmt == 'raw' else text
|
||||
if len(compressed.encode('utf-8')) < 4500:
|
||||
return None, f'too small after compress ({len(compressed)}B)'
|
||||
dst = os.path.join(dst_dir, name)
|
||||
with open(dst, 'w', encoding='utf-8', newline='') as f:
|
||||
f.write(compressed)
|
||||
orig_kb, new_kb = os.path.getsize(src) // 1024, os.path.getsize(dst) // 1024
|
||||
ratio = (1 - new_kb / max(orig_kb, 1)) * 100
|
||||
return dst, {'src': os.path.basename(src), 'dst': name, 'fmt': fmt,
|
||||
'orig_kb': orig_kb, 'new_kb': new_kb, 'ratio': f'{ratio:.0f}%',
|
||||
'year': timestamps[0][:4]}
|
||||
|
||||
def _compress_raw(text):
|
||||
"""Format B: strip system prompt (Prompt→USER) and assistant echo (ASSISTANT→Response)."""
|
||||
sections = _parse_sections(text)
|
||||
out = []
|
||||
for i, (typ, line, body) in enumerate(sections):
|
||||
if typ == 'prompt':
|
||||
out.append(line + '\n')
|
||||
if not (i+1 < len(sections) and sections[i+1][0] == 'user'):
|
||||
out.append(body) # no USER follows → keep body
|
||||
elif typ in ('user', 'response'):
|
||||
out.append(line + '\n')
|
||||
out.append(body)
|
||||
elif typ == 'preamble':
|
||||
out.append(body)
|
||||
# assistant → skip (redundant echo)
|
||||
return ''.join(out)
|
||||
|
||||
_RE_HISTORY = re.compile(r'<history>(.*?)</history>', re.S)
|
||||
|
||||
def _parse_history_block(raw):
|
||||
"""Parse <history> block into ['[USER]...', '[Agent]...'] lines."""
|
||||
lines = [l.strip() for l in raw.split('\n') if l.strip()]
|
||||
parsed = [l for l in lines if l.startswith('[USER]') or l.startswith('[Agent]')]
|
||||
if len(parsed) >= 2:
|
||||
return parsed
|
||||
# JSON format: literal \\n separators
|
||||
joined = raw.strip()
|
||||
if '\\n[USER]' in joined or '\\n[Agent]' in joined:
|
||||
parts = joined.replace('\\n', '\n').split('\n')
|
||||
parsed = [p.strip() for p in parts if p.strip() and (p.strip().startswith('[USER]') or p.strip().startswith('[Agent]'))]
|
||||
if parsed: return parsed
|
||||
return parsed or []
|
||||
|
||||
def _merge_history_blocks(all_blocks):
|
||||
"""Merge sliding-window history blocks into one deduplicated list."""
|
||||
if not all_blocks: return []
|
||||
acc = list(all_blocks[0])
|
||||
for block in all_blocks[1:]:
|
||||
if not block: continue
|
||||
if not acc:
|
||||
acc = list(block); continue
|
||||
best = 0
|
||||
for k in range(1, min(len(acc), len(block)) + 1):
|
||||
if acc[-k:] == block[:k]: best = k
|
||||
if best > 0:
|
||||
acc.extend(block[best:])
|
||||
elif block[0] in acc:
|
||||
idx = len(acc) - 1 - acc[::-1].index(block[0])
|
||||
match_len = 0
|
||||
for j in range(min(len(block), len(acc) - idx)):
|
||||
if acc[idx + j] == block[j]: match_len = j + 1
|
||||
else: break
|
||||
acc.extend(block[match_len:])
|
||||
else:
|
||||
acc.extend(block)
|
||||
return acc
|
||||
|
||||
def extract_history(src, session_name=None):
|
||||
"""Extract [USER]/[Agent] history from session file."""
|
||||
with open(src, 'r', encoding='utf-8', errors='replace') as f:
|
||||
text = f.read()
|
||||
if session_name is None:
|
||||
session_name = os.path.splitext(os.path.basename(src))[0]
|
||||
all_blocks = [parsed for m in _RE_HISTORY.finditer(text)
|
||||
if (parsed := _parse_history_block(m.group(1)))]
|
||||
if all_blocks:
|
||||
return _merge_history_blocks(all_blocks)
|
||||
return []
|
||||
|
||||
def format_history_block(session_name, history_lines):
|
||||
"""Format history lines into all_histories.txt block format."""
|
||||
sep = '=' * 60
|
||||
return f"{sep}\nSESSION: {session_name}\n{sep}\n" + '\n'.join(history_lines) + '\n'
|
||||
|
||||
import tempfile, shutil, zipfile, glob
|
||||
from collections import defaultdict
|
||||
|
||||
def _existing_sessions(l4_dir):
|
||||
"""Read session names already in all_histories.txt."""
|
||||
hist_path = os.path.join(l4_dir, 'all_histories.txt')
|
||||
if not os.path.exists(hist_path): return set()
|
||||
with open(hist_path, 'r', encoding='utf-8') as f:
|
||||
return {l.strip().replace('SESSION: ', '') for l in f if l.startswith('SESSION: ')}
|
||||
|
||||
def batch_process(src, l4_dir=None, dry_run=True):
|
||||
"""Batch compress + extract history + archive. dry_run=True is safe default."""
|
||||
l4_dir = os.path.normpath(l4_dir or L4_DIR)
|
||||
raw_files = sorted(src) if isinstance(src, (list, tuple)) else \
|
||||
sorted(glob.glob(os.path.join(src, 'model_responses_*.txt')))
|
||||
raw_files = sorted(raw_files, key=os.path.getmtime)[:-10] # always retain 10 newest raw files
|
||||
if not raw_files:
|
||||
print("No raw files found"); return {'processed': 0, 'skipped': 0, 'errors': 0, 'new_sessions': 0}
|
||||
|
||||
existing = _existing_sessions(l4_dir)
|
||||
print(f"Found {len(raw_files)} raw, {len(existing)} existing in L4")
|
||||
|
||||
tmp_dir = tempfile.mkdtemp(prefix='cs_batch_')
|
||||
results, skipped, errors = [], [], []
|
||||
|
||||
import time
|
||||
cutoff = time.time() - 7200 # skip files modified within 2h
|
||||
|
||||
# Phase 1: Compress + Extract (to temp dir)
|
||||
for fp in raw_files:
|
||||
fname = os.path.basename(fp)
|
||||
if os.path.getmtime(fp) > cutoff:
|
||||
skipped.append((fname, 'recent(<2h)')); continue
|
||||
try:
|
||||
dst, info = compress_session(fp, tmp_dir)
|
||||
if dst is None:
|
||||
skipped.append((fname, info)); continue
|
||||
sn = os.path.splitext(os.path.basename(dst))[0]
|
||||
if sn in existing:
|
||||
skipped.append((fname, f'dup:{sn}')); os.remove(dst); continue
|
||||
results.append((sn, dst, extract_history(dst), info, fp))
|
||||
except Exception as e:
|
||||
errors.append((fname, str(e)))
|
||||
results.sort(key=lambda x: x[0])
|
||||
|
||||
print(f"\nP1: {len(results)} new, {len(skipped)} skip, {len(errors)} err")
|
||||
for f, r in skipped[:5]: print(f" SKIP {f}: {r}")
|
||||
for f, e in errors[:5]: print(f" ERR {f}: {e}")
|
||||
if results: print(f" Range: {results[0][0]} → {results[-1][0]}")
|
||||
|
||||
if dry_run:
|
||||
print("\n[DRY RUN] Pass dry_run=False to execute.")
|
||||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||||
return {'processed': len(results), 'skipped': len(skipped),
|
||||
'errors': len(errors), 'new_sessions': len(results),
|
||||
'sessions': [r[0] for r in results]}
|
||||
|
||||
# Phase 2: Append history
|
||||
with open(os.path.join(l4_dir, 'all_histories.txt'), 'a', encoding='utf-8') as f:
|
||||
for sn, _, hist, _, _ in results:
|
||||
if hist: f.write('\n' + format_history_block(sn, hist))
|
||||
print(f"Appended {len(results)} sessions to all_histories.txt")
|
||||
|
||||
# Phase 3: Archive to monthly zips
|
||||
by_month = defaultdict(list)
|
||||
for sn, cpath, _, info, _ in results:
|
||||
year = info.get('year', '2026') if isinstance(info, dict) else '2026'
|
||||
by_month[f"{year}-{sn[:2]}"].append((sn, cpath))
|
||||
for mk, items in sorted(by_month.items()):
|
||||
zpath = os.path.join(l4_dir, f"{mk}.zip")
|
||||
mode = 'a' if os.path.exists(zpath) else 'w'
|
||||
with zipfile.ZipFile(zpath, mode, zipfile.ZIP_DEFLATED) as zf:
|
||||
names = set(zf.namelist()) if mode == 'a' else set()
|
||||
for sn, cp in items:
|
||||
if f"{sn}.txt" not in names: zf.write(cp, f"{sn}.txt")
|
||||
print(f" {mk}.zip: +{len(items)}")
|
||||
|
||||
# Phase 4: Delete raw files
|
||||
to_del = [rp for *_, rp in results]
|
||||
for fname, reason in skipped:
|
||||
if 'recent' in reason: continue # active session still being written
|
||||
m = [f for f in raw_files if os.path.basename(f) == fname]
|
||||
if m: to_del.append(m[0])
|
||||
deleted = 0
|
||||
for rp in to_del:
|
||||
try: os.remove(rp); deleted += 1
|
||||
except Exception: pass
|
||||
print(f"Deleted {deleted}/{len(to_del)} raw files")
|
||||
|
||||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||||
report = {'processed': len(results), 'skipped': len(skipped),
|
||||
'errors': len(errors), 'new_sessions': len(results), 'deleted_raw': deleted}
|
||||
print(f"\nDone: {report}")
|
||||
return report
|
||||
|
||||
# ── CLI ──
|
||||
RAW_DIR = os.path.join(os.path.dirname(os.path.dirname(L4_DIR)), 'temp', 'model_responses')
|
||||
|
||||
if __name__ == '__main__':
|
||||
import argparse
|
||||
ap = argparse.ArgumentParser(description='L4 session archiver')
|
||||
ap.add_argument('src', nargs='?', default=RAW_DIR, help='raw files dir')
|
||||
ap.add_argument('--run', action='store_true', help='actually execute (default: dry run)')
|
||||
args = ap.parse_args()
|
||||
batch_process(args.src, dry_run=not args.run)
|
||||
@@ -0,0 +1,65 @@
|
||||
# 历史重点挖掘 SOP
|
||||
|
||||
从用户历史对话中挖掘值得长期保留的重点：情绪事件、持续活动、已消失事项。
|
||||
|
||||
## 数据源
|
||||
|
||||
`../memory/L4_raw_sessions/all_histories.txt`（`compress_session.py` 产出）。不存在则先运行脚本。只读 user 内容。
|
||||
|
||||
## 执行约束
|
||||
|
||||
- 增量：维护已处理 session 列表，每次只扫新 session
|
||||
- 分批处理时不拆分同一 session
|
||||
|
||||
## 产物
|
||||
|
||||
存 `./history_insight/`（不存在则创建），格式自定。产物是数据库，不是报告——每条发现都是下游任务的输入，遗漏即损失。
|
||||
|
||||
产物包含三个持久状态（可以是多个文件或一个文件内的多个区域）：
|
||||
1. **活动知识层** — 每次扫描读取、更新、写回。记录所有识别到的活动及其最终分类。
|
||||
2. **情绪事件列表** — 追加式，只增不改。
|
||||
3. **增量标记** — 记录最后处理到的 session 标识，下次从此处之后开始。
|
||||
|
||||
每条发现必须含：
|
||||
- session 标识（与 L4 zip 内文件名一致，如 `0403_2013-0403_2145`）
|
||||
- 关键原文片段（可 grep 定位回原日志）
|
||||
- 发现类型标签
|
||||
|
||||
## 提取标准
|
||||
|
||||
### 情绪事件
|
||||
|
||||
标记语气上的明显波动，不是内容上的。
|
||||
|
||||
**标记**：愤怒/质问/责备、讽刺挖苦、惊喜感激、反复纠正后语气突变、沮丧/无奈、预期落空后的失望或方向突变。
|
||||
|
||||
**不标记**：纯功能指令、语气平和的反馈、讨论负面话题但本人情绪稳定。
|
||||
|
||||
### 持续活动与已消失事项
|
||||
|
||||
维护一个持久的活动知识层（存 `./history_insight/`）。这是跨次运行的持久状态——每次扫描读取它、更新它、写回它。它不是报告，是数据库。
|
||||
|
||||
**你在建模的是**：这个用户的生活里现在有什么、曾经有什么。活动识别的唯一证据是用户在session中主动发起的请求或讨论——系统提示词、SOP列表、记忆引用中的被动出现不构成证据，不能从"系统里存在某个SOP"推断用户在做某事。
|
||||
|
||||
- 持续活动 = 仍然存在于用户生活中的事——值得深入了解其细节
|
||||
- 已消失 = 曾经存在但已离开的事——可能导致已有记忆过时
|
||||
|
||||
判断"离开"不需要用户明确表态。事项本身的性质就是证据——有终点的事做完了就是消失了，没终点的事沉默不代表消失。
|
||||
|
||||
每个条目必须归入二者之一。
|
||||
|
||||
归类原则：
|
||||
- 相同专有名词/工具名/项目名 → 同一条目
|
||||
- 通用动作不单独成条目，除非反复出现于同一领域
|
||||
- 宁多建不错合并
|
||||
|
||||
每条记录含：涉及的 session 列表、代表性原文片段、出现频次。
|
||||
|
||||
## 坑点
|
||||
|
||||
- 用户消息的含义不在关键词里，在语气和上下文里。脚本扫描只能看到主题，看不到情绪、看不到习惯、看不到事项的生命周期变化。必须阅读原文。
|
||||
- session 标识必须与 L4 zip 内文件名一致（如 `0403_2013-0403_2145`），不能用模糊占位（如 xxxx）——无法定位回原日志的记录没有价值
|
||||
- 情绪判断看语气不看内容——讨论负面话题但本人情绪稳定不标记
|
||||
- 活动归类是"用户在做什么"的知识表示，不是对话摘要
|
||||
- 写入产物前检查一致性：同一条目不能同时出现在"持续"和"已消失"中。如果阅读时收集到矛盾信号（频次高 vs 已停止），必须做最终裁决再写入
|
||||
- "持续"和"已消失"是两个独立分类，不能合并为一个列表用子状态（如"已消退"）规避。产物中必须有明确分开的两个区域或两个文件
|
||||
@@ -0,0 +1,86 @@
|
||||
# adb_ui.py - 一键dump+解析Android UI (u2优先，原生fallback)
|
||||
# 要先看 computer_use.md；dump配合ui_detect，微信/支付宝小程序常需detect补盲
|
||||
# 搜索框一般直接输入拼音/首字母即可，别硬啃 adb 中文输入
|
||||
# u2 (uiautomator2) 不受idle限制，适合动画密集app（美团等）
|
||||
# 弹窗检测: ui(clickable_only=True, raw=True) 找全屏FrameLayout+底部小ImageView(关闭X)
|
||||
# 已知包名: 美团外卖=com.sankuai.meituan.takeoutnew 淘宝=com.taobao.taobao
|
||||
import subprocess, xml.etree.ElementTree as ET, os, re, shutil
|
||||
|
||||
ADB = shutil.which("adb") or "adb"
|
||||
LOCAL_XML = os.path.join(os.path.dirname(os.path.abspath(__file__)), "ui_mt.xml")
|
||||
|
||||
def _dump_u2():
|
||||
"""用uiautomator2 dump，不受idle限制"""
|
||||
try:
|
||||
import uiautomator2 as u2
|
||||
d = u2.connect()
|
||||
xml_str = d.dump_hierarchy()
|
||||
if xml_str and len(xml_str) > 100: return xml_str
|
||||
except Exception as e:
|
||||
print(f"[u2 fallback] {e}")
|
||||
return None
|
||||
|
||||
def _dump_native():
|
||||
"""原生uiautomator dump（需idle状态）"""
|
||||
subprocess.run([ADB, "shell", "rm", "-f", "/sdcard/ui.xml"], capture_output=True)
|
||||
r = subprocess.run([ADB, "shell", "uiautomator", "dump", "--compressed", "/sdcard/ui.xml"],
|
||||
capture_output=True, text=True, timeout=15)
|
||||
if "dumped" not in r.stdout.lower() and "dumped" not in r.stderr.lower(): print(f"dump failed: {r.stdout}{r.stderr}"); return None
|
||||
subprocess.run([ADB, "pull", "/sdcard/ui.xml", LOCAL_XML], capture_output=True, timeout=10)
|
||||
with open(LOCAL_XML, "r", encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
def _parse_xml(xml_str, keyword=None, clickable_only=False, raw=False):
|
||||
"""解析XML字符串为节点列表"""
|
||||
root = ET.fromstring(xml_str)
|
||||
nodes = []
|
||||
for n in root.iter("node"):
|
||||
pkg = n.get("package", "")
|
||||
if "termux" in pkg.lower(): continue
|
||||
text = n.get("text", "")
|
||||
desc = n.get("content-desc", "")
|
||||
bounds = n.get("bounds", "")
|
||||
click = n.get("clickable") == "true"
|
||||
cls = n.get("class", "").split(".")[-1]
|
||||
rid = n.get("resource-id", "")
|
||||
label = text or desc
|
||||
if not label and not click and not raw: continue
|
||||
if clickable_only and not click: continue
|
||||
if keyword and keyword.lower() not in label.lower(): continue
|
||||
cx, cy = 0, 0
|
||||
if bounds:
|
||||
m = re.findall(r'\[(\d+),(\d+)\]', bounds)
|
||||
if len(m) == 2:
|
||||
cx = (int(m[0][0]) + int(m[1][0])) // 2
|
||||
cy = (int(m[0][1]) + int(m[1][1])) // 2
|
||||
edit = cls == "EditText"
|
||||
nodes.append({"text": text or desc, "click": click, "edit": edit, "cx": cx, "cy": cy, "cls": cls, "rid": rid})
|
||||
return nodes
|
||||
|
||||
def ui(keyword=None, clickable_only=False, raw=False):
|
||||
"""一键dump+解析Android UI (u2优先)
|
||||
keyword: 过滤含关键词的节点
|
||||
clickable_only: 只显示可点击节点
|
||||
raw: 返回原始节点列表而非打印
|
||||
"""
|
||||
xml_str = _dump_u2() or _dump_native()
|
||||
if not xml_str: print("dump failed (both u2 and native)"); return []
|
||||
nodes = _parse_xml(xml_str, keyword, clickable_only, raw)
|
||||
if not raw:
|
||||
for n in nodes:
|
||||
flag = "E" if n.get("edit") else ("Y" if n["click"] else " ")
|
||||
coord = f"({n['cx']},{n['cy']})" if n['cx'] else ""
|
||||
display_text = n['text']
|
||||
if not display_text:
|
||||
hint = n.get('rid', '').split('/')[-1] or n.get('cls', 'icon')
|
||||
display_text = f"<{hint}>"
|
||||
print(f"[{flag}] {display_text} {coord}")
|
||||
print(f"\ntotal: {len(nodes)} nodes")
|
||||
return nodes
|
||||
|
||||
def tap(x, y):
|
||||
subprocess.run([ADB, "shell", "input", "tap", str(x), str(y)], capture_output=True)
|
||||
print(f"tap({x},{y}) ok")
|
||||
|
||||
if __name__ == "__main__":
|
||||
ui()
|
||||
@@ -0,0 +1,45 @@
|
||||
# 自主行动 SOP
|
||||
|
||||
⚠️ **路径警告**：autonomous_reports 在 temp/ 下，用`./autonomous_reports/`访问，**不是**`../memory/autonomous_reports/`或`../autonomous_reports/`！TODO在cwd下。
|
||||
报告存于 `./autonomous_reports/`，文件名 `RXX_简短描述.md`（XX从 history.txt 推断自增）。
|
||||
|
||||
授权你进行自主行动，只要不对环境造成副作用都可进行。
|
||||
|
||||
## 启动（第一步）
|
||||
- update_working_checkpoint: `自主行动｜收尾时重读SOP | from autonomous_operation_sop.helper import *; set_todo()/complete_task(tasktitle, historyline, report_path)`
|
||||
|
||||
第二步：
|
||||
```python
|
||||
from autonomous_operation_sop.helper import *
|
||||
print(get_history(40)) # 了解历史避免重复
|
||||
print(get_todo()) # 查看待办
|
||||
```
|
||||
|
||||
## 任务选择
|
||||
- 有未完成条目 → 取**一条**，直接进入执行，其他条目下次执行
|
||||
- 无 TODO → 读 `autonomous_operation_sop/task_planning.md` 规划，下次执行
|
||||
- 不连续两次选相同子任务
|
||||
- 价值公式：**「AI训练数据无法覆盖」×「对未来协作有持久收益」**
|
||||
|
||||
## 执行
|
||||
- 选定任务后 update_working_checkpoint，将选中的TODO条目和执行注意事项 append 到 checkpoint 中
|
||||
- 调用code_run准备结尾回调, script为handler._done_hooks.append("重读自主任务sop，检查你刚刚的收尾工作是否正确，不正确则改正")，inline_eval=True （秘密参数）
|
||||
- 小步快跑，边探测边实验
|
||||
- 用临时脚本验证假设；禁只读即下结论，完整验证再写报告
|
||||
- 即使失败也记录实验过程和结果，失败报告同样有价值
|
||||
- 用户不在线，遇到需要决策的问题写入报告待审，不要卡住
|
||||
|
||||
**收尾（4件事缺一不可）**：
|
||||
1. 重读本sop
|
||||
2. 在cwd写报告（文件名任意），若有记忆更新建议，附在报告末尾
|
||||
3. `from/import helper; complete_task(tasktitle, historyline, report_path)` → 自动编号+移报告到 autonomous_reports/+prepend history（historyline 格式：`类型 | 主题 | 结论`，严格单行）
|
||||
4. `set_todo()` 获取TODO路径 → 将已完成条目标记为 `[x]`（注意前缀）
|
||||
5. 结束，剩余TODO留到下次再做
|
||||
|
||||
## 权限边界
|
||||
- 无需批准：只读探测、cwd内写操作/脚本实验
|
||||
- 需写入报告待审：修改 global_mem / memory下SOP、安装软件、外部API调用、删除非临时文件
|
||||
- 绝对禁止：读取密钥、修改核心代码库、不可逆危险操作
|
||||
|
||||
## 等待用户审查
|
||||
- 用户归来后审查报告，决定批准、修改或拒绝方案
|
||||
@@ -0,0 +1,137 @@
|
||||
"""
|
||||
autonomous_task.py - 自主行动任务管理API
|
||||
放置: memory/autonomous_operation_sop/
|
||||
用法: import autonomous_task (或 from autonomous_operation_sop import autonomous_task)
|
||||
|
||||
4个函数:
|
||||
get_todo() → 返回TODO内容
|
||||
get_history(n) → 返回最近n条历史
|
||||
complete_task() → 移报告+编号+写history+返回改TODO指令
|
||||
set_todo() → 返回TODO真实路径
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
# ── 路径计算（基于模块自身位置） ──
|
||||
_MODULE_DIR = Path(__file__).resolve().parent # memory/autonomous_operation_sop/
|
||||
_MEMORY_DIR = _MODULE_DIR.parent # memory/
|
||||
_AGENT_DIR = _MEMORY_DIR.parent # GenericAgent/
|
||||
_TEMP_DIR = _AGENT_DIR / "temp" # GenericAgent/temp/
|
||||
_REPORTS_DIR = _TEMP_DIR / "autonomous_reports"
|
||||
_HISTORY_FILE = _REPORTS_DIR / "history.txt"
|
||||
_TODO_FILE = _TEMP_DIR / "TODO.txt"
|
||||
|
||||
def _next_report_number() -> int:
|
||||
"""扫 history.txt 第一行提取最大 RXX 编号，返回下一个"""
|
||||
if not _HISTORY_FILE.exists():
|
||||
return 1
|
||||
with open(_HISTORY_FILE, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
# 匹配所有 R 后跟数字的模式
|
||||
nums = [int(m) for m in re.findall(r'R(\d+)', content)]
|
||||
if not nums:
|
||||
return 1
|
||||
return max(nums) + 1
|
||||
|
||||
|
||||
def get_todo() -> str:
|
||||
"""返回 TODO.txt 的内容。若文件不存在返回提示。"""
|
||||
if not _TODO_FILE.exists(): return f"[autonomous_task] TODO.txt 不存在，路径: {_TODO_FILE}"
|
||||
with open(_TODO_FILE, "r", encoding="utf-8") as f: return f.read()
|
||||
|
||||
def get_history(n: int = 20) -> str:
|
||||
"""返回 history.txt 的前 n 行（最新在前）。"""
|
||||
if not _HISTORY_FILE.exists():
|
||||
return f"[autonomous_task] history.txt 不存在，路径: {_HISTORY_FILE}"
|
||||
with open(_HISTORY_FILE, "r", encoding="utf-8") as f:
|
||||
lines = f.readlines()
|
||||
return "".join(lines[:n])
|
||||
|
||||
|
||||
def set_todo(*args, **kwargs) -> str:
|
||||
"""返回 TODO.txt 的真实绝对路径，供 agent/子agent 自行读写。"""
|
||||
return f'路径: {str(_TODO_FILE)}'
|
||||
|
||||
|
||||
def complete_task(taskname: str, historyline: str, report_path: str) -> str:
|
||||
"""
|
||||
完成任务的原子操作：
|
||||
1. 移动 report_path → autonomous_reports/R{XX}_{taskname}.md（自动编号）
|
||||
2. prepend historyline 到 history.txt（校验必须单行）
|
||||
3. 返回字符串指示 agent 自己去改 TODO
|
||||
Args:
|
||||
taskname: 任务简短名称（用于报告文件名，如 "晨间简报"）
|
||||
historyline: 历史记录内容（必须单行，日期自动添加，如 "工程 | 晨间简报 | 完成7模块聚合"）
|
||||
report_path: agent 已写好的报告文件路径（绝对或相对于cwd）
|
||||
Returns:
|
||||
成功消息 + 改TODO指令，或错误消息
|
||||
"""
|
||||
errors = []
|
||||
|
||||
# ── 校验 ──
|
||||
if "\n" in historyline.strip():
|
||||
return "[ERROR] historyline 必须是单行，不能包含换行符"
|
||||
|
||||
report = Path(report_path).resolve()
|
||||
if not report.exists():
|
||||
return f"[ERROR] 报告文件不存在: {report_path}"
|
||||
|
||||
if not _REPORTS_DIR.exists():
|
||||
_REPORTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# ── 1. 移动报告 ──
|
||||
rnum = _next_report_number()
|
||||
# 清理 taskname 中的非法文件名字符
|
||||
safe_name = re.sub(r'[<>:"/\\|?*]', '_', taskname).strip()
|
||||
dest_name = f"R{rnum}_{safe_name}.md"
|
||||
dest_path = _REPORTS_DIR / dest_name
|
||||
|
||||
try:
|
||||
shutil.move(str(report), str(dest_path))
|
||||
except Exception as e:
|
||||
return f"[ERROR] 移动报告失败: {e}"
|
||||
|
||||
# ── 2. prepend history ──
|
||||
# 自动加编号 + 日期（剥离 agent 可能已写的编号/日期，统一重建）
|
||||
line = historyline.strip()
|
||||
line = re.sub(r'^R\d+\s*\|\s*', '', line) # 剥离 R 编号
|
||||
line = re.sub(r'^\d{4}-\d{2}-\d{2}\s*\|\s*', '', line) # 剥离日期
|
||||
today = datetime.now().strftime('%Y-%m-%d')
|
||||
line = f"R{rnum} | {today} | {line}"
|
||||
|
||||
try:
|
||||
existing = ""
|
||||
if _HISTORY_FILE.exists():
|
||||
with open(_HISTORY_FILE, "r", encoding="utf-8") as f:
|
||||
existing = f.read()
|
||||
with open(_HISTORY_FILE, "w", encoding="utf-8") as f:
|
||||
f.write(line + "\n" + existing)
|
||||
except Exception as e:
|
||||
# 回滚：把报告移回去
|
||||
try:
|
||||
shutil.move(str(dest_path), str(report))
|
||||
except:
|
||||
pass
|
||||
return f"[ERROR] 写入 history 失败: {e}（报告已回滚）"
|
||||
|
||||
# ── 3. 返回改 TODO 指令 ──
|
||||
return (
|
||||
f"✅ 完成！报告已保存: {dest_name}\n"
|
||||
f"历史已记录: {line}\n"
|
||||
f"👉 请在 {_TODO_FILE} 中将对应任务标记为 [x] R{rnum}，然后结束，**其他TODO下次再干**"
|
||||
)
|
||||
|
||||
|
||||
# ── 快速自检 ──
|
||||
if __name__ == "__main__":
|
||||
print(f"TEMP_DIR: {_TEMP_DIR}")
|
||||
print(f"REPORTS_DIR: {_REPORTS_DIR}")
|
||||
print(f"HISTORY: {_HISTORY_FILE}")
|
||||
print(f"TODO: {_TODO_FILE}")
|
||||
print(f"Next R#: R{_next_report_number()}")
|
||||
print(f"\n--- TODO ---\n{get_todo()[:200]}")
|
||||
print(f"\n--- History (5) ---\n{get_history(5)}")
|
||||
@@ -0,0 +1,41 @@
|
||||
# 任务规划模式
|
||||
|
||||
- **有TODO**：cwd下 `TODO.txt` 有待执行条目 → 直接跳到「执行流程」
|
||||
|
||||
价值公式：**「AI训练数据无法覆盖」×「对未来协作有持久收益」**。核心产出是记忆——有价值的发现整理为记忆更新提案纳入报告。
|
||||
|
||||
## 流程入口：
|
||||
- **无TODO → 进入任务规划模式**（本轮不执行任务，专注规划）：
|
||||
0. update_working_checkpoint: `规划模式：产出TODO后必须经过subagent评审，不允许自评审，禁止执行任何TODO，等待下次自主行动进入执行模式`
|
||||
1. ⚠️ **批判性读history.txt**：99%历史任务是**低价值的**，读取目的是**识别失败模式并避免**，而非寻找模仿对象
|
||||
- 识别低价值模式：浅层验证、无假设巡检、重复探索、泛采集、知名工具基础用法
|
||||
- 提炼高价值线索：未跟进的发现、待实测工具、可改进产出
|
||||
- 避免和历史重复和低价值边际效应
|
||||
2. 反思：为什么这些任务低价值？如何设计才能高价值？
|
||||
3. **批判性**盘点已有报告和记忆（ls autonomous_reports/ + ../memory），考虑如何发挥更大价值或优化
|
||||
4. 综合以上，产出5-7条TODO写入 `TODO.txt`，TODO已完成内容可压缩丢后面
|
||||
5. 每条格式：`[ ] 类型(产出/冲浪/环境) | 一句话目标 | 验收标准`
|
||||
6. **必须召唤subagent评审TODO**（保证可信三方）：input仅给TODO列表+"读记忆库和本sop自行判断，逐条评分1-10并简述理由"（不喂额外先验信息）
|
||||
7. **召唤失败无限重试**：TODO必须经过subagent评审，不允许自评审，未经评审的TODO不可执行
|
||||
8. 读subagent评分，低分项删除或替换
|
||||
9. 立刻**结束**，下次行动再执行
|
||||
|
||||
## 目标排序（按价值递减）：
|
||||
1. **实用产出与能力扩展**：写工具解决痛点，在已有能力上解锁新能力（能力树每多一个节点，可能性空间变大）
|
||||
2. **环境发现**：扫描已有但未利用的工具/库/数据源/配置
|
||||
3. **小众工具挖掘**：在GitHub/吾爱破解/果核剥壳**等**找冷门实用工具，实测AI常推荐但有坑的方案
|
||||
4. **了解用户与推荐**：分析老代码/PC文件/书签推断偏好，给出个性化推荐（游戏/视频/工具附理由）（低频）
|
||||
5. **自身演进**：思考框架不足，提出改进方案
|
||||
6. **记忆审查**：修正错误或过时记录
|
||||
|
||||
**大型任务**：允许设计**有价值**的大型任务，将其分解成若干个模块或步骤，写入TODO中，每次自主行动执行处理一个模块。
|
||||
|
||||
选择原则：个性化优先（只有探测这台PC才能获得的知识）→ 盲区优先（自身参数无法复现，有一定难度）→ 假设驱动（明确要验证什么，边探测边实验）→ 禁止低价值验证（不验证静态配置、不做无假设巡检、不做你轻易完成的工作）
|
||||
|
||||
## 探测策略（聚焦原则，非菜单）：
|
||||
- **能力树扩展**：优先能解锁新能力节点的工具/技能（一个节点带来多种可能性）
|
||||
- **个性化优先**：只有探测这台PC/这个用户才能获得的知识 > 通用知识
|
||||
- **线索驱动**：近期报告中提炼的后续任务
|
||||
- 冲浪规则：每次≤2话题，必须读正文提炼洞察，禁标题搬运；发现好工具→下轮TODO加实测任务
|
||||
|
||||
禁区：❌ Hacker News · 刷新闻头条 · 泛采集标题/无目标刷新闻 · 探索知名工具基础用法 · 调研弱于当前框架的AI工具 · 调研其他web自动化/computer use框架 · 读取自身代码库 · 使用im工具发送信息
|
||||
@@ -0,0 +1,102 @@
|
||||
# checklist_helper.py — CL(folder) 一站式任务清单（支持 checklist/mapreduce 两种模式）
|
||||
import json, time, subprocess, socket, sys
|
||||
from pathlib import Path
|
||||
_R = Path(__file__).resolve().parent.parent
|
||||
_BBS, _MAIN = _R/"assets/agent_bbs.py", _R/"agentmain.py"
|
||||
_W_RE, _M_RE = _R/"reflect/agent_team_worker.py", _R/"reflect/checklist_master.py"
|
||||
_PK = {"stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL}
|
||||
if sys.platform == "win32": _PK["creationflags"] = 0x200
|
||||
|
||||
class CL:
|
||||
def __init__(self, folder, goal="", workers=0):
|
||||
"""
|
||||
workers=0: checklist模式，master自己逐个执行，不启动BBS
|
||||
workers>0: mapreduce模式，启动BBS+N个worker并行
|
||||
"""
|
||||
self.folder = Path(folder); self.folder.mkdir(parents=True, exist_ok=True)
|
||||
self.path = self.folder / "state.json"
|
||||
self.workers = workers
|
||||
if self.path.exists(): self._d = json.loads(self.path.read_text("utf-8"))
|
||||
else:
|
||||
self._d = {"closed": False, "goal": goal, "bbs": None, "tasks": []}
|
||||
self._save()
|
||||
if workers > 0:
|
||||
self._ensure_bbs()
|
||||
self.start_worker(workers)
|
||||
|
||||
@property
|
||||
def tasks(self): return self._d["tasks"]
|
||||
@property
|
||||
def closed(self): return self._d.get("closed", False)
|
||||
@property
|
||||
def has_open(self): return any(t["result"] is None for t in self.tasks)
|
||||
@property
|
||||
def bbs_url(self): return self._d["bbs"]["url"] if self._d["bbs"] else None
|
||||
@property
|
||||
def bbs_key(self): return self._d["bbs"]["key"] if self._d["bbs"] else None
|
||||
@property
|
||||
def mode(self): return "mapreduce" if self._d["bbs"] else "checklist"
|
||||
def _save(self): self.path.write_text(json.dumps(self._d, ensure_ascii=False, indent=1), "utf-8")
|
||||
|
||||
def _ensure_bbs(self):
|
||||
if self._d["bbs"]: return
|
||||
with socket.socket() as s: s.bind(('',0)); port = s.getsockname()[1]
|
||||
key = f"cl_{int(time.time())%1000}"
|
||||
(self.folder/"bbs").mkdir(exist_ok=True)
|
||||
subprocess.Popen(["python", str(_BBS), "--cwd", str(self.folder/"bbs"),
|
||||
"--port", str(port), "--key", key], **_PK)
|
||||
time.sleep(1)
|
||||
self._d["bbs"] = {"url": f"http://127.0.0.1:{port}", "key": key}
|
||||
self._save()
|
||||
|
||||
def add(self, texts):
|
||||
nid = max((t["id"] for t in self.tasks), default=0) + 1
|
||||
ids = []
|
||||
for t in texts:
|
||||
self.tasks.append({"id": nid, "text": t, "result": None, "ts": int(time.time())})
|
||||
ids.append(nid); nid += 1
|
||||
self._save();
|
||||
print('task added, must reread checklist SOP before start executing ...');
|
||||
return ids
|
||||
|
||||
def mark(self, tid, result):
|
||||
for t in self.tasks:
|
||||
if t["id"] == tid: t["result"] = result; t["ts"] = int(time.time()); break
|
||||
self._save()
|
||||
|
||||
def look(self):
|
||||
done = sum(1 for t in self.tasks if t["result"] is not None)
|
||||
lines = [f"[{done}/{len(self.tasks)}] mode={self.mode}"]
|
||||
for t in self.tasks:
|
||||
l = f'{"✓" if t["result"] else "○"} #{t["id"]} {t["text"][:60]}'
|
||||
if t["result"]: l += f' → {t["result"][:60]}'
|
||||
lines.append(l)
|
||||
return "\n".join(lines)
|
||||
|
||||
def close(self):
|
||||
assert not self.has_open, "has open tasks"
|
||||
self._d["closed"] = True; self._save()
|
||||
|
||||
def start_worker(self, n=None):
|
||||
n = n or self.workers or 1
|
||||
if n <= 0: return
|
||||
for i in range(n):
|
||||
subprocess.Popen(["python", str(_MAIN), "--reflect", str(_W_RE),
|
||||
"--base_url", self.bbs_url, "--board_key", self.bbs_key, "--name", f"w{i+1}"], **_PK)
|
||||
if i < n - 1: time.sleep(5)
|
||||
|
||||
def _pid_alive(self, pid):
|
||||
if not pid: return False
|
||||
try:
|
||||
r = subprocess.run(["tasklist", "/FI", f"PID eq {pid}"], capture_output=True, text=True)
|
||||
return str(pid) in r.stdout
|
||||
except Exception: return False
|
||||
|
||||
def start_master(self):
|
||||
old_pid = self._d.get("master_pid")
|
||||
if old_pid and self._pid_alive(old_pid):
|
||||
print(f"[CL] master already running (PID {old_pid}), skip")
|
||||
return
|
||||
p = subprocess.Popen(["python", str(_MAIN), "--reflect", str(_M_RE),
|
||||
"--mr_folder", str(self.folder.resolve())], **_PK)
|
||||
self._d["master_pid"] = p.pid; self._save()
|
||||
@@ -0,0 +1,97 @@
|
||||
# Checklist SOP
|
||||
|
||||
## Booter（启动者/用户）
|
||||
|
||||
**Checklist 模式**（单人，master自己执行）：
|
||||
```python
|
||||
from checklist_helper import CL
|
||||
cl = CL("cl_xxx", goal="<用户要求任务，尽量原样>")
|
||||
cl.start_master() # Only for Booter，Master严禁调用
|
||||
```
|
||||
|
||||
**MapReduce 模式**（多人，master派发+worker执行）：
|
||||
```python
|
||||
from checklist_helper import CL
|
||||
cl = CL("cl_xxx", goal="<用户要求任务，尽量原样>", workers=2)
|
||||
cl.start_master() # Only for Booter，Master严禁调用
|
||||
```
|
||||
|
||||
goal 写法：只写「做什么 + 参考哪个SOP」，不写怎么做。Master 自己读 SOP 决定 plan。
|
||||
|
||||
## Master（reflect agent 使用）
|
||||
|
||||
```python
|
||||
from checklist_helper import CL
|
||||
cl = CL("cl_xxx") # 加载状态（BBS已在跑）
|
||||
cl.add(["任务1", "任务2"]) # 在你的笔记中记录TODO项
|
||||
cl.look() # 查进度
|
||||
cl.mark(id, "摘要") # 验收
|
||||
cl.close() # 全部完成后关闭
|
||||
```
|
||||
|
||||
## Master plan示例
|
||||
|
||||
目标可分解为多个**不相干、可并行**的子任务 → add 子任务。
|
||||
B 要等 A 的结果 → 不要硬拆，串行做。
|
||||
|
||||
任务使用短句，派发时再补充信息。
|
||||
|
||||
1. 下载网盘 /game 下所有文件
|
||||
→ 先 webscan 拿文件列表，再每个文件一条任务
|
||||
`cl.add(["下载A.exe", "下载B.zip", "下载C.zip"])`
|
||||
|
||||
2. 从语法、风格、格式角度检查 a.pdf
|
||||
→ 三个维度天然独立
|
||||
`cl.add(["检查语法", "检查风格", "检查格式"])`
|
||||
|
||||
3. 查所有 VPS 中版本 < 22 的，升级到 24
|
||||
→ 第一轮：每台一条查版本任务
|
||||
`cl.add(["查 node03 版本", "查 node09 版本", "查 Dell 版本"])`
|
||||
→ reduce：master 筛出 < 22 的
|
||||
→ 第二轮：每台需升级的一条任务
|
||||
`cl.add(["升级 node03 到 24", "升级 Dell 到 24"])`
|
||||
|
||||
## Master 循环
|
||||
|
||||
```
|
||||
cl.look()
|
||||
├─ 有未完成任务 → 去 BBS 派发（mapreduce模式）/ 自己干（无worker checklist模式）
|
||||
└─ 全部完成
|
||||
├─ 用户最终目标已达成 → close()
|
||||
└─ 最终目标未达成 → plan 下一步
|
||||
├─ 可解耦 → add() 新一批任务
|
||||
├─ 需串行前置 → 自己做一步，再回 look
|
||||
└─ 基本搞定 → 自己整合结果，交付最终报告
|
||||
```
|
||||
|
||||
master会被持续唤醒直到其显式成功调用close()。
|
||||
|
||||
## 派发任务（有workers模式下）
|
||||
|
||||
worker无法看到add的任务，只能看到BBS！
|
||||
每条任务 prompt 须**自包含**——worker 没有 master 的上下文。
|
||||
每次最多只派发3个任务，不要一次性把所有任务贴到bbs上。
|
||||
worker足够聪明，只允许写目标和需要的信息，不要干预
|
||||
**master不允许执行已经派发出去的任务，会导致重复执行！** 没事就sleep！
|
||||
|
||||
写 prompt 要点：
|
||||
1. **背景**：worker 需要的信息直接给（路径、数据、约定），不要假设 worker 知道
|
||||
2. **交付物**：明确产出什么、格式、写到哪里
|
||||
3. **不限手段**：说要什么结果，别规定怎么做
|
||||
4. **不干预 BBS 行为**：禁止教 worker 如何抢单/回帖/报告，那是 worker 自己的机制
|
||||
|
||||
交付规范（写进任务 prompt）：
|
||||
- 交付结果和报告信息必须分开。交付 = 纯成品；报告 = 过程/问题/备注
|
||||
- 交付文件禁止出现说明性废话
|
||||
- 长结果写文件，短结果直接回帖
|
||||
|
||||
## 验收
|
||||
|
||||
Master 收到 worker 回帖或自己完成子任务后：
|
||||
- 检查结果，语义判断 pass/fail → `cl.mark(id, "结果摘要")`
|
||||
- 交付物含过程废话 → 要求重写交付物
|
||||
- 失败 → 可重发、换 prompt、或自己补
|
||||
|
||||
## 注意
|
||||
|
||||
- 若子任务需要 web 工具，提醒并行 worker 新建 tab 并使用自己的 tab
|
||||
@@ -0,0 +1,42 @@
|
||||
# 什么是好的代码
|
||||
好的代码不是"能跑就行"，而是在长期演化中保持**压缩性、局部性、可组合性与可证伪性**。
|
||||
一句话判断：同样的功能，用最小必要的结构实现——概念少，覆盖广，变化不扩散。
|
||||
---
|
||||
## 一、模块边界清晰
|
||||
每个模块只做一件事，依赖方向稳定，指向抽象而非细节。改 A 不需要连带动 B、C、D，没有循环依赖，没有到处互相 import。
|
||||
## 二、局部可推理
|
||||
看一个文件、一个函数，就能判断它的行为、代价和失败模式。不需要全局搜索隐式约定、全局状态或魔法配置才能读懂。
|
||||
## 三、可组合
|
||||
小组件能自然地组合成大能力，接口一致、正交，不需要为组合写特例。"这个只能在那种情况下用"是坏信号。
|
||||
## 四、变化半径小
|
||||
改一个需求，改动集中、可预测，回归风险可控。而不是改动像地震，到处打补丁，回归测试覆盖不了心里的担忧。
|
||||
## 五、复杂度线性增长
|
||||
新增一个功能，新增代码量近似线性，重复少。而不是功能越多代码越膨胀，出现大量相似分支和 if-else 森林。
|
||||
## 六、约束写进代码
|
||||
关键不变量和约束写进类型、接口、校验、状态机，错误尽早暴露。而不是靠调用者"记得要先做 X 再做 Y"，靠注释和口口相传。
|
||||
## 七、可测试、可观测
|
||||
依赖可注入，单测容易写，日志和指标能定位因果链。而不是只能端到端测，一出问题就黑盒，难以复现。
|
||||
## 八、一致且不意外
|
||||
命名、错误处理、资源管理、并发模型全局一致，很少有反直觉的地方。而不是同一类问题三套解法，新人踩坑全靠运气。
|
||||
## 九、自解释，注释极简
|
||||
代码本身就是文档。命名、结构、流程足以说明意图，注释只出现在真正难以一眼看懂或容易误读的地方。如果一段代码需要大段注释才能读懂，说明代码本身该重写。
|
||||
## 十、代码极简，视觉均匀
|
||||
行数尽量少，不写多余的代码。每行长度大致平均，避免忽长忽短的锯齿感。简洁不是压缩，是没有废话。
|
||||
## 十一、函数式倾向，减少副作用
|
||||
优先纯函数，输入决定输出，减少隐藏的状态突变。但不教条——如果一个全局变量能显著降低整体复杂度，接受它。目标是整体简单，不是局部纯粹。
|
||||
## 十二、功能越多，代码应该越短
|
||||
好的抽象让新功能复用已有结构，而不是堆砌新代码。功能翻倍但代码量没怎么涨，说明抽象到位了。反过来，功能越加代码越膨胀，是架构在退化。
|
||||
## 十三、为未来的接入性设计
|
||||
写代码时设想：这段逻辑未来会被别的模块调用吗？会被外部系统接入吗？好代码天然留有干净的调用入口，而不是写死在某个特定场景里，等需要复用时才发现得大改。
|
||||
## 十四、Let it crash——按失败半径决定防御策略
|
||||
半径大的错误显式报错、快速中断；半径为零的静默放过。不分轻重地到处 try-catch，反而把真正需要暴露的问题吞掉了。
|
||||
## 十五、篇幅分布跟着功能分布走
|
||||
一个函数里，主功能占大部分代码，兜底和错误处理压到最短。如果防御性代码比正事还长，说明结构有问题。读代码应该一眼看出主线，而不是在 fallback 里找。
|
||||
---
|
||||
## 快速自检
|
||||
拿到一段代码，问自己四个问题：
|
||||
1. **我能不能不看全局，就安全地改一个局部？**
|
||||
2. **有没有一个清晰的核心抽象，让新功能主要是"加新实现"而不是"改旧逻辑"？**
|
||||
3. **变化点是收敛在边界上，还是散落在各处？**
|
||||
4. **出故障时，能快速定位到责任模块，还是全员背锅？**
|
||||
四个问题都能干脆地答"是"，就是好代码。
|
||||
@@ -0,0 +1,48 @@
|
||||
# computer_use
|
||||
|
||||
相关L3 memory: **ui_detect.py** ljqCtrl.py/ljqCtrlBg.py ljqCtrl_sop.md
|
||||
|
||||
## 0. GUI操作节奏建议
|
||||
进入新界面时，建议先只探测不操作：枚举窗口 + UIA + ljqCtrl截图 + ui_detect，读完实际输出再决定下一步
|
||||
明确一个操作后，可以在同一轮执行该动作，短暂等待，再立刻枚举窗口 + 截图/ui_detect 验证新状态；不要在未知状态下把多步决策写进大脚本
|
||||
尽量不要预测关键词筛候选，应看 detect 输出、坐标、层级和上下文判断
|
||||
若确定UIA可用则少用ui_detect/ljqCtrl；若UIA不可用，则后续不用UIA
|
||||
|
||||
## 1. 基础规则
|
||||
### 探测/定位四工具（按优先级降级，前者无效才用后者）
|
||||
| 工具 | 用法时机 | 能力/适用 | 限制 |
|
||||
|------|----------|-----------|------|
|
||||
| 0 win32gui 窗口枚举 | 始终先行，总是可用 | 枚举标题/类名/rect，确定目标窗口、前台状态、客户区原点 | 仅定位窗口，不进控件 |
|
||||
| 1 Python UIA（控件树）| 首选探测+操作 | 控件树可用时，探测与操作（含免坐标点击）都用 UIA | **游戏禁用**；对该窗口一旦无效则后续也不用 |
|
||||
| 2 ui_detect.py（配合ljqCtrl截图） | 1无效时才用 | 截图视觉检测控件，返回 bbox+OCR 文本 | bbox 是截图内坐标需转屏幕物理坐标 |
|
||||
| 3 vision(VLM) | 2仍不足时才用 | 仅语义理解、确认界面状态、辅助判断目标 | 不可信其坐标 |
|
||||
|
||||
Windows 下窗口截图和操作使用 ljqCtrl：严禁 pyautogui；记得先 Activate 到前台（除非用户明确要求后台操作或后台操作失效）
|
||||
ui_detect附送OCR，不要单独使用OCR
|
||||
用PIL传输图像，或者用统一1.png存储截图，不要创建大量截图文件
|
||||
|
||||
ui_detect 的 bbox 是截图内坐标，点击前必须用 `ClientToScreen(hwnd,(0,0))/dpi_scale + bbox中心` 转屏幕物理坐标
|
||||
坐标转换禁用 `GetWindowRect` 或 DWM 窗口矩形直接加截图坐标（含标题栏/边框/阴影会错位）
|
||||
ljqCtrl.Click 后会返回像素/前台变化，0% 或近 0% 变化立即停下诊断，禁止盲目重试。
|
||||
ljqCtrl 失效或目标为网络游戏时，必须使用硬件键鼠 Xbananakb / Arduino Leonardo（如有）
|
||||
网络游戏除非用户明确允许，严禁普通键鼠事件，必须硬件执行。
|
||||
|
||||
临时截图/可视化文件用后清理，或固定文件名覆盖，避免堆积。
|
||||
ui_detect 可跨端复用；手机端沿用本原则时，UIA 换成 ui dump/adb_ui，ljqCtrl 控制换成 adb
|
||||
|
||||
### 重要必坑
|
||||
坑1-遮盖/失焦：混乱时枚举窗口确认前台；
|
||||
坑2-DPI：必须先import ljqCtrl，之后一律使用物理坐标；
|
||||
|
||||
|
||||
## macOS 平台
|
||||
macOS 定位链与 §1 一致，工具映射如下：
|
||||
- 控制层：`import macljqCtrl as ljqCtrl`（替代 Windows ljqCtrl）
|
||||
- 窗口枚举：`ListWindows()` → 返回 id/app/title/bbox/pid（替代 win32gui）
|
||||
- 激活：`ActivateApp(pid)`（pid 来自 ListWindows，禁止用名字子串——同厂商 bundle 前缀会误伤）
|
||||
- UIA 层 = AX 辅助功能 API：`AXElements(pid)` 枚举控件树 → role/desc/title/id/物理坐标 xywh；`AXFind(pid, role=, desc=, title=)` 过滤；`AXPress(el)` 免坐标点击
|
||||
- 坐标：AX 返回逻辑点，库内自动 /dpi_scale 转物理像素，与 Click/Screenshot 统一；Retina 默认 scale=0.5
|
||||
- 截图：`GrabWindow(window_id)` 或 `ScreenCapAt(x, y, radius)`（物理坐标）
|
||||
- 权限：首次使用需授予「辅助功能」权限（系统设置 > 隐私与安全 > 辅助功能）；`AXIsProcessTrusted()` 检测
|
||||
- 依赖：`pip install pyobjc-framework-ApplicationServices`（AX 相关，软依赖——未装时键鼠/截图正常，仅 AX 函数不可用）
|
||||
- 节奏同 §2：先 ListWindows + AXElements 探测，确认控件后再操作；AXPress 优先，回退 Click(phys_cx, phys_cy)
|
||||
@@ -0,0 +1,49 @@
|
||||
# Goal Hive Master 工作 SOP
|
||||
|
||||
Master 是 Hive 的总体设计部：不亲自生产子任务产物，只负责**拆解子任务、判断、汇总**，靠调度 worker 把核心交付物在给定时间内稳定推向用户满意。Master 无权停止自己，不得设计自停条件。
|
||||
|
||||
本 SOP 按**第一性原理**从**工程控制论**推出：把交付当受控系统——**J\*＝用户真正要的价值（目标/价值函数，哲学层不变；变的只是你对它的形式化估计 Ĵ）**，**y＝当前产物**，**e＝J\*−y（偏差）**。每轮的活＝测 e、压 e，让 y 单调逼近 J\*；预算到点就交当前最好的 y。"失稳"＝系统跑偏/空转（见 §1）。
|
||||
Master也应基于**第一性原理**思考如何完成用户任务。
|
||||
|
||||
## 0. 怎么跑（每轮照做）
|
||||
|
||||
**三条铁律**
|
||||
1. Master 只做两件事：**拆**（把阶段目标切成互不重叠的独立子任务派给 worker）和**汇**（汇总产物、判断、排序）。绝不自己下场生产产物。
|
||||
2. 始终维护一个"**当前最优已验收版本**"作锚点；每轮只在锚点上**增量改**（在已有产物上找可优化点修改，能不重写就不重写）；**验收让 J 升才合入，变差就回退**——锚点只增不减。
|
||||
3. **循环到预算用尽才停**，交当前锚点版（不是"做完"才停）。任何时刻必须清楚自己在 `x.几`。
|
||||
|
||||
**一轮 = 探测 → 设计 → 执行 → 检查 →（重读本 SOP）→ 下一轮**。每阶段有自己的阶段目标，分**发散求全**（探测/检查：靠多 worker 并行、独立、去相关地铺开）和**收敛择优**（设计/执行：Master 判断、择一、忠实落地）两种。
|
||||
|
||||
### x.1 探测（阶段目标＝查得尽量全）
|
||||
- **查什么**：1 分析用户需求 2 探测环境现状 3 记忆中的重要信息/原则 4 调研可用的方案·材料·方法建议 5 上轮的结果·变化·检查报告。
|
||||
- **锁边界（先于动手，校准 Ĵ）**：钉死 J\* 范围——**要什么、明确不要什么**；需求模糊/有歧义处（如"接入"是只收还是收发）按"**最小必要 + 简单优雅**"收敛，或向用户澄清，**禁止臆测扩张 scope**。
|
||||
- **拆**：按上面四项切成独立调研子任务，**分头**派多个 worker。
|
||||
- **汇**：收齐 → 按对 J* 的重要性排序 → 写 `探测报告Tx.md`（只留重点和变化）。
|
||||
- 第 2 轮起只查"上轮变了/没查清"的，不重查。环境一变就重探变化的部分。
|
||||
|
||||
### x.2 设计（阶段目标＝收敛出最优那一个方案）
|
||||
- **Master 亲自做**（这阶段没什么可并行）：据探测报告 + 上轮检查报告，定这轮**改哪几处**、拆几个执行子任务、各自验收线。
|
||||
- 写 `执行方案Tx.md`，内含 **changelog ＝这轮要改的 P0/P1 清单**（来自上轮检查报告，逐条对准缺口，不在已饱和处精雕）。
|
||||
|
||||
### x.3 执行（阶段目标＝忠实落地选定方案）
|
||||
- **拆**：每个执行子任务＝独立接口（输入/输出/放哪里/合格线），派给 worker，能并行就并行。
|
||||
- **汇**：Master 不下场，只盯进度、收产物、按 changelog 增量改锚点。产物按需。
|
||||
|
||||
### x.4 检查（阶段目标＝挑出尽量多问题）
|
||||
- **拆**：从**多角度派独立**的挑刺/测试子任务给**不同** worker——① 用户视角试用 ② 攻击者/反面假设 ③ 边界与 corner case（测例尽量多、全、广）④ 第三方独立复核 ⑤ **回到需求质疑目标本身**：对照 J\* 看 Ĵ 是否做多/做偏/过度设计（如造了需求不要的能力）——校准 Ĵ，不只校准产物 y。独立才挑得出不同问题。
|
||||
- **验收线**：每个挑刺/测试子任务必须交**可复现的物理证据**，且**证据形态匹配交付形态**——代码→端到端跑通的命令+原始输出（不止单元桩测）；文稿→**全文通读**+按需渲染/视觉核对版式；数据→实跑校验。"声明已完成"不算验收。
|
||||
- **汇**：汇总所有问题 → 按对 J* 的伤害**排成 P0/P1** → 写 `检查报告Tx.md`。
|
||||
- **这份 P0/P1 报告就是下一轮 x.2 的 changelog**——偏差 `e = J*−y` 被具体化、带进下一轮增量修。
|
||||
|
||||
## 1. 失稳急刹（出现任一信号，立即按序处置）
|
||||
|
||||
信号：worker 忙但 J 不升 / 局部产物多但整体不可用 / 过程证明取代用户价值 / 多人改同一产物冲突 / Master 被细节牵走丢全局 / 额外产出污染核心交付。
|
||||
|
||||
处置：① 停止新派发 → ② 回读用户需求与 J* 重新对齐"现在最重要的一件事" → ③ 查接口是否未冻结 → ④ 砍掉与主目标最弱的在途任务 → ⑤ 某维度连续两轮 J 不升即判饱和，换离达标最远的维度 → 恢复闭环再派。
|
||||
|
||||
## 2. 底线
|
||||
|
||||
- 核心产物只放用户要用的成品（说人话、给成品、取舍随场景）；来源/验证/尝试记录另放，不污染成品。
|
||||
- 不确定性要么查证补全、要么删除，自己搞定，不留半成品、不推给用户。
|
||||
- 时间够就修到更优；预算到点仍未通过的项，必须**如实写入交付报告**。诚实记录写报告，不写进成品本身。
|
||||
- BBS_CWD 保持整洁：中间产物归子目录或带标注，核心交付物一眼可定位。
|
||||
@@ -0,0 +1,53 @@
|
||||
# Goal Hive Mode SOP
|
||||
|
||||
## 定义
|
||||
|
||||
Goal Hive = Goal Mode 的多 worker 协作协议
|
||||
Hive模式单独运行，不要和plan/supervisor/subagent混杂
|
||||
|
||||
## 启动
|
||||
|
||||
1. 选一个空闲端口 `PORT` 和本次协作 key `BOARD_KEY`。
|
||||
2. 创建本次 Hive 数据目录：`BBS_CWD=<CodeRoot>/temp/hive_<目标短名>`。
|
||||
3. 启动 BBS：`start /b python <CodeRoot>/assets/agent_bbs.py --cwd <BBS_CWD> --port <PORT> --key <BOARD_KEY>`。
|
||||
4. requests访问http://127.0.0.1:<PORT>/readme?key=<BOARD_KEY>。
|
||||
- 手动发帖/传文件 API：写请求带 header `X-API-Key: <BOARD_KEY>`；先 `POST /register` 得 `token`，再 `POST /post`；文件用 `POST /file/upload`。
|
||||
5. 在bbs发第一个帖子，按照以下“第一帖规范”
|
||||
6. 后台启动首个worker
|
||||
7. 询问用户时间预算，按`goal_mode_sop.md`后台启动hive master
|
||||
8. Hive master，workers都是与你不同的独立进程，你启动它们后应当报告用户并停止
|
||||
|
||||
### 第一帖规范
|
||||
|
||||
BBS 第一帖必须包含以下四项：
|
||||
1. 任务目标
|
||||
2. 下方「Hive Master 职责」全文4点（一字不改）
|
||||
3. 工作目录说明：优先使用 `<BBS_CWD>` 进行文件传输而非BBS文件功能
|
||||
4. 附加说明（一字不改）：`此为最终目标，worker不要接单，先等hive master拆分子任务。`
|
||||
|
||||
### Hive Master 职责
|
||||
1. master必须阅读记忆中goal_hive_master_duty.md，持续检查问题、寻找改进点
|
||||
2. 你**负责任务调度和团队组织**，只能干上述duty中提到的内容，不允许亲自干活导致 worker 空转
|
||||
3. 终极目标是要做到**完美的找不到任何问题的**任务交付结果，保证用户满意，围绕核心产出
|
||||
4. 如果子任务很多，worker做不过来，可以参照Goal Hive Mode SOP拉起更多worker
|
||||
|
||||
## Hive Master
|
||||
|
||||
### goal_state.json 规范
|
||||
|
||||
`objective` 必须包含以下几块，缺一不可：
|
||||
1. 用户目标（简明描述任务与交付物）
|
||||
2. BBS地址（用requests）：`http://127.0.0.1:<PORT>/readme?key=<BOARD_KEY>`
|
||||
3. 上方「Hive Master 职责」全文（一字不改）
|
||||
4. 阅读记忆中goal_hive_master_duty.md了解如何分派和管理工作
|
||||
|
||||
`done_prompt` 必须设置为以下固定文本（一字不改）：
|
||||
`关闭所有你拉起的worker，并在BBS发一条帖子宣告你管理的任务结束，worker除了明确追加任务外，不应再回应。`
|
||||
|
||||
启动 master 前必须回读 `goal_state.json`，逐项确认 objective 完整、done_prompt 原文匹配，否则不得启动。
|
||||
|
||||
## 拉起 worker
|
||||
|
||||
启动 worker：`start /b python <CodeRoot>/agentmain.py --reflect <CodeRoot>/reflect/agent_team_worker.py --base_url http://127.0.0.1:<PORT> --board_key <BOARD_KEY> --name hive-worker-1`。
|
||||
|
||||
后续 worker 由 Goal Master 按需要增加（不能超过5个，一般任务2-4个足够）。
|
||||
@@ -0,0 +1,50 @@
|
||||
# Goal Mode SOP
|
||||
|
||||
## 何时使用
|
||||
|
||||
用户给出开放目标 + 时间预算（如"花3小时持续优化X"、"没事也找事干"），且不是一次性闭环任务。
|
||||
|
||||
## 设置
|
||||
|
||||
写 `temp/goal_state.json`（或自定义路径）：
|
||||
|
||||
```json
|
||||
{
|
||||
"objective": "用户原话目标",
|
||||
"budget_seconds": 10800,
|
||||
"start_time": <time.time()>,
|
||||
"turns_used": 0,
|
||||
"max_turns": 200,
|
||||
"status": "running",
|
||||
"done_prompt": ""
|
||||
}
|
||||
```
|
||||
|
||||
- `budget_seconds`：最少 3 小时（10800），按用户要求调整
|
||||
- `max_turns`：防空转上限，一般 200 够用
|
||||
- `status`：必须为 `"running"`
|
||||
|
||||
## 启动
|
||||
|
||||
必须后台启动（长时间运行，不占前台终端）：
|
||||
|
||||
```bash
|
||||
# 默认路径 temp/goal_state.json
|
||||
start /b python agentmain.py --reflect reflect/goal_mode.py
|
||||
|
||||
# 自定义路径（多实例）
|
||||
set GOAL_STATE=temp/goal_xxx.json && start /b python agentmain.py --reflect reflect/goal_mode.py
|
||||
|
||||
# 用其他模型跑（--llm_no 选择已配置的第N个LLM，从0开始）
|
||||
set GOAL_STATE=temp/goal_xxx.json && start /b python agentmain.py --reflect reflect/goal_mode.py --llm_no 1
|
||||
```
|
||||
|
||||
## 停止
|
||||
|
||||
- 预算耗尽时自动进入收口轮，然后停止
|
||||
- 手动停：杀进程
|
||||
|
||||
## 观察进度
|
||||
|
||||
- 状态：读 goal_state.json 的 `turns_used` / `status`
|
||||
- 详情：看 `temp/model_responses/` 下最近修改的文件尾部
|
||||
@@ -0,0 +1,29 @@
|
||||
# Incubator SOP — 远程部署GA实例
|
||||
自我复制到任意节点的agent网络。每个节点有独立记忆，可以通过编辑memory干预它们的行为
|
||||
|
||||
## 文件清单
|
||||
```
|
||||
GARoot/*.py
|
||||
GARoot/assets/*.txt *.json
|
||||
mykey.py
|
||||
GARoot/memory/
|
||||
```
|
||||
memory下大部分文件不适合分发，不要复制 memory 下未被 gitignore 白名单的文件！
|
||||
不要复制 memory 下的L1/L2文件(global_mem(_insight).txt)，会自动初始化
|
||||
|
||||
打包红线：严格按上面4行清单执行，`*` 是 glob 全匹配，不得擅自改成“必要文件/可启动闭包”。
|
||||
- `GARoot/*.py` 必须包含根目录所有 `.py`。
|
||||
- `GARoot/assets/*.txt *.json` 必须包含 assets 顶层所有 `.txt`/`.json`。
|
||||
- `GARoot/memory/` 只取 `.gitignore` 白名单/已允许分发文件；排除 `global_mem.txt`、`global_mem_insight.txt`、`__pycache__/`、`*.pyc`。
|
||||
- 按当前清单实测压缩包约153KB/55文件；正常不应超过200KB，文件数不应超过60。
|
||||
|
||||
## 依赖
|
||||
requests beautifulsoup4
|
||||
尽量复用远端已有python/venv
|
||||
|
||||
## 通信
|
||||
1. **首选** 阅读 `assets/ga_httpapp.py`（HTTP API，~50行自解释）
|
||||
2. 备选：subagent.md 文件协议 或 reflect worker + bbs
|
||||
|
||||
## 干预记忆
|
||||
直接编辑远端 memory/ 下的文件（SOP/全局记忆）
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Keychain: save key to a file, then keys.set("name", file="path"); keys.name.use() to retrieve (use but no print)."""
|
||||
import json, os, hashlib, pathlib, getpass
|
||||
|
||||
_PATH = pathlib.Path.home() / "ga_keychain.enc"
|
||||
try: _user = os.getlogin()
|
||||
except OSError: _user = getpass.getuser()
|
||||
_MASK = hashlib.sha256(f"{_user}@ga_keychain".encode()).digest()
|
||||
|
||||
def _xor(data: bytes) -> bytes:
|
||||
return bytes(b ^ _MASK[i % len(_MASK)] for i, b in enumerate(data))
|
||||
|
||||
print('# SecretStr.use() to get raw, do not print raw value! | keys.ls() to list all keys')
|
||||
|
||||
class SecretStr:
|
||||
def __init__(self, name: str, val: str):
|
||||
self._name, self._val = name, val
|
||||
def use(self) -> str: return self._val
|
||||
def __repr__(self):
|
||||
n = len(self._val)
|
||||
if n <= 4: preview = '***'
|
||||
elif n <= 16: preview = f"{self._val[:3]}···{self._val[-3:]}"
|
||||
elif n <= 40: preview = f"{self._val[:6]}···{self._val[-6:]} len={n}"
|
||||
else: preview = f"{self._val[:10]}···{self._val[-6:]} len={n}"
|
||||
return f"SecretStr({self._name}={preview})"
|
||||
__str__ = __repr__
|
||||
|
||||
class _Keys:
|
||||
def __init__(self):
|
||||
self._d = {}
|
||||
if _PATH.exists():
|
||||
try:
|
||||
raw = json.loads(_xor(_PATH.read_bytes()))
|
||||
self._d = {k: SecretStr(k, v) for k, v in raw.items()}
|
||||
except Exception as e:
|
||||
print(f"[keychain] WARNING: failed to load {_PATH}: {e}")
|
||||
print(f"[keychain] Starting with empty keychain. Old file kept as .bak")
|
||||
_PATH.rename(_PATH.with_suffix('.enc.bak'))
|
||||
def _save(self):
|
||||
raw = {k: v.use() for k, v in self._d.items()}
|
||||
_PATH.write_bytes(_xor(json.dumps(raw).encode()))
|
||||
def __getattr__(self, k):
|
||||
if k.startswith('_'): raise AttributeError(k)
|
||||
if k not in self._d: raise KeyError(f"No secret: {k}")
|
||||
return self._d[k]
|
||||
def __repr__(self):
|
||||
return f"Keychain({len(self._d)} secrets: {', '.join(self._d.keys())})"
|
||||
def set(self, k, v=None, *, file=None):
|
||||
if file: v = pathlib.Path(file).read_text().strip()
|
||||
self._d[k] = SecretStr(k, v)
|
||||
self._save()
|
||||
def ls(self): return list(self._d.keys())
|
||||
|
||||
keys = _Keys()
|
||||
|
||||
def __getattr__(name): return getattr(keys, name)
|
||||
@@ -0,0 +1,177 @@
|
||||
"""
|
||||
CRITICAL: 严禁在此工具链中 import pyautogui (会污染 win32api 导致逻辑冲突)。
|
||||
ljqCtrl Quick Reference:
|
||||
- dpi_scale: float (Logical = Physical * dpi_scale)
|
||||
- Click(x, y, check=True): Use Physical Coordinates. check=True → 自动比前后像素变化，返回周边图像
|
||||
- SetCursorPos(z): Use Physical Coordinates z=(x, y)
|
||||
- Press(cmd, staytime=0): Keyboard shortcuts (e.g. 'ctrl+v')
|
||||
- FindBlock(fn, wrect=None, threshold=0.8) -> (obj_center_phys, is_found)
|
||||
- MouseDClick(staytime=0.05), MouseClick(staytime=0.05)
|
||||
- GrabWindow(hwnd) -> PIL Image: DPI-safe window screenshot (needs foreground)
|
||||
- GrabWindowBg(hwnd_or_name) -> PIL Image: WGC background capture (Win10+, pip install windows-capture)
|
||||
"""
|
||||
import os, sys, time, random, math, win32api, win32con, win32gui, ctypes
|
||||
import numpy as np
|
||||
|
||||
print('[TIPS] always use physical coordinates!')
|
||||
|
||||
dpi_scale = 1
|
||||
try:
|
||||
from PIL import ImageGrab, Image, ImageEnhance, ImageFilter, ImageDraw
|
||||
import cv2
|
||||
except: pass
|
||||
|
||||
ctypes.windll.user32.SetProcessDPIAware()
|
||||
|
||||
_hdc = ctypes.windll.user32.GetDC(0)
|
||||
swidth = ctypes.windll.gdi32.GetDeviceCaps(_hdc, 118) # DESKTOPHORZRES (物理)
|
||||
sheight = ctypes.windll.gdi32.GetDeviceCaps(_hdc, 117) # DESKTOPVERTRES
|
||||
ctypes.windll.user32.ReleaseDC(0, _hdc)
|
||||
cwidth = win32api.GetSystemMetrics(win32con.SM_CXSCREEN) # 逻辑
|
||||
cheight = win32api.GetSystemMetrics(win32con.SM_CYSCREEN)
|
||||
dpi_scale = cwidth / swidth
|
||||
print('Screen width & height:', swidth, sheight)
|
||||
print('dpi_scale:', dpi_scale)
|
||||
|
||||
def MouseDown(): win32api.mouse_event(win32con.MOUSEEVENTF_LEFTDOWN,0,0)
|
||||
def MouseUp(): win32api.mouse_event(win32con.MOUSEEVENTF_LEFTUP,0,0)
|
||||
|
||||
def MouseClick(staytime=0.05):
|
||||
MouseDown(); time.sleep(staytime)
|
||||
MouseUp(); time.sleep(0.05)
|
||||
|
||||
def MouseDClick(staytime=0.05):
|
||||
MouseDown(); MouseUp()
|
||||
MouseDown(); MouseUp()
|
||||
time.sleep(0.05)
|
||||
|
||||
def SetCursorPos(z):
|
||||
z = tuple(map(lambda v:int(v*dpi_scale), z))
|
||||
win32api.SetCursorPos(z)
|
||||
time.sleep(0.05)
|
||||
|
||||
def Click(x, y=None, check=True):
|
||||
if type(x) is type(tuple()): x, y = int(x[0]), int(x[1])
|
||||
if check: before, fg_before = ScreenCapAt(x, y), win32gui.GetForegroundWindow()
|
||||
SetCursorPos( (x, y) )
|
||||
MouseClick()
|
||||
if check:
|
||||
time.sleep(0.5)
|
||||
after = ScreenCapAt(x, y)
|
||||
b, a = np.array(before), np.array(after)
|
||||
diff = np.sum(np.any(b != a, axis=2))
|
||||
total = b.shape[0] * b.shape[1]
|
||||
fg_after = win32gui.GetForegroundWindow()
|
||||
fg_title = win32gui.GetWindowText(fg_after)
|
||||
fg_changed = fg_before != fg_after
|
||||
print(f'[Click check] {diff}/{total} px changed ({diff/total*100:.1f}%) | fg: "{fg_title}" {"⚠️CHANGED" if fg_changed else ""}')
|
||||
return after
|
||||
click = Click
|
||||
|
||||
def Press(cmd, staytime=0):
|
||||
if type(cmd) is list: cmds = [x.lower() for x in cmd]
|
||||
else: cmds = cmd.lower().split('+')
|
||||
for z in cmds:
|
||||
win32api.keybd_event(VK_CODE[z], 0, 0, 0)
|
||||
time.sleep(staytime)
|
||||
for z in reversed(cmds):
|
||||
time.sleep(staytime)
|
||||
win32api.keybd_event(VK_CODE[z], 0, win32con.KEYEVENTF_KEYUP, 0)
|
||||
press = Press
|
||||
|
||||
VK_CODE = {'backspace':0x08, 'tab':0x09, 'clear':0x0C, 'enter':0x0D, 'shift':0x10, 'ctrl':0x11, 'alt':0x12, 'pause':0x13, 'caps_lock':0x14, 'esc':0x1B, 'escape':0x1B, 'space':0x20, 'page_up':0x21, 'page_down':0x22, 'end':0x23, 'home':0x24, 'left_arrow':0x25, 'up_arrow':0x26, 'right_arrow':0x27, 'down_arrow':0x28, 'select':0x29, 'print':0x2A, 'execute':0x2B, 'print_screen':0x2C, 'ins':0x2D, 'del':0x2E, 'help':0x2F, '0':0x30, '1':0x31, '2':0x32, '3':0x33, '4':0x34, '5':0x35, '6':0x36, '7':0x37, '8':0x38, '9':0x39, 'a':0x41, 'b':0x42, 'c':0x43, 'd':0x44, 'e':0x45, 'f':0x46, 'g':0x47, 'h':0x48, 'i':0x49, 'j':0x4A, 'k':0x4B, 'l':0x4C, 'm':0x4D, 'n':0x4E, 'o':0x4F, 'p':0x50, 'q':0x51, 'r':0x52, 's':0x53, 't':0x54, 'u':0x55, 'v':0x56, 'w':0x57, 'x':0x58, 'y':0x59, 'z':0x5A, 'numpad_0':0x60, 'numpad_1':0x61, 'numpad_2':0x62, 'numpad_3':0x63, 'numpad_4':0x64, 'numpad_5':0x65, 'numpad_6':0x66, 'numpad_7':0x67, 'numpad_8':0x68, 'numpad_9':0x69, 'multiply_key':0x6A, 'add_key':0x6B, 'separator_key':0x6C, 'subtract_key':0x6D, 'decimal_key':0x6E, 'divide_key':0x6F, 'F1':0x70, 'F2':0x71, 'F3':0x72, 'F4':0x73, 'F5':0x74, 'F6':0x75, 'F7':0x76, 'F8':0x77, 'F9':0x78, 'F10':0x79, 'F11':0x7A, 'F12':0x7B, 'F13':0x7C, 'F14':0x7D, 'F15':0x7E, 'F16':0x7F, 'F17':0x80, 'F18':0x81, 'F19':0x82, 'F20':0x83, 'F21':0x84, 'F22':0x85, 'F23':0x86, 'F24':0x87, 'num_lock':0x90, 'scroll_lock':0x91, 'left_shift':0xA0, 'right_shift ':0xA1, 'left_control':0xA2, 'right_control':0xA3, 'left_menu':0xA4, 'right_menu':0xA5, 'browser_back':0xA6, 'browser_forward':0xA7, 'browser_refresh':0xA8, 'browser_stop':0xA9, 'browser_search':0xAA, 'browser_favorites':0xAB, 'browser_start_and_home':0xAC, 'volume_mute':0xAD, 'volume_Down':0xAE, 'volume_up':0xAF, 'next_track':0xB0, 'previous_track':0xB1, 'stop_media':0xB2, 'play/pause_media':0xB3, 'start_mail':0xB4, 'select_media':0xB5, 'start_application_1':0xB6, 'start_application_2':0xB7, 'attn_key':0xF6, 'crsel_key':0xF7, 'exsel_key':0xF8, 'play_key':0xFA, 'zoom_key':0xFB, 'clear_key':0xFE, '+':0xBB, ',':0xBC, '-':0xBD, '.':0xBE, '/':0xBF, '`':0xC0, ';':0xBA, '[':0xDB, '\\':0xDC, ']':0xDD, "'":0xDE}
|
||||
VK_CODE = {k.lower():v for k,v in VK_CODE.items()}
|
||||
|
||||
def Activate(hwnd):
|
||||
"""稳定切换前台窗口。绕过Windows前台锁限制。"""
|
||||
# 如果窗口最小化先恢复
|
||||
if ctypes.windll.user32.IsIconic(hwnd):
|
||||
ctypes.windll.user32.ShowWindow(hwnd, 9) # SW_RESTORE
|
||||
# 发假Alt-up骗过前台锁
|
||||
ctypes.windll.user32.keybd_event(0x12, 0, 2, 0) # VK_MENU up
|
||||
time.sleep(0.02)
|
||||
try:
|
||||
win32gui.SetForegroundWindow(hwnd)
|
||||
except Exception:
|
||||
# fallback: BringWindowToTop + SetFocus
|
||||
ctypes.windll.user32.BringWindowToTop(hwnd)
|
||||
ctypes.windll.user32.SetFocus(hwnd)
|
||||
time.sleep(0.15)
|
||||
activate = Activate
|
||||
|
||||
def GrabWindow(hwnd):
|
||||
if isinstance(hwnd, str): hwnd = win32gui.FindWindow(None, hwnd); assert hwnd, f'窗口未找到'
|
||||
Activate(hwnd); time.sleep(0.25)
|
||||
# 只截客户区(不含标题栏边框), 与GrabWindowBg一致 → 截图内坐标统一用ClientToScreen原点做偏移
|
||||
l, t = win32gui.ClientToScreen(hwnd, (0, 0))
|
||||
cr = win32gui.GetClientRect(hwnd) # (0,0,w,h)
|
||||
bbox = (l, t, l + cr[2], t + cr[3])
|
||||
bbox = tuple(int(v / dpi_scale) for v in bbox)
|
||||
return ImageGrab.grab(bbox)
|
||||
|
||||
def GrabWindowBg(hwnd_or_name, timeout=5):
|
||||
"""WGC后台截图(Win10+), 传hwnd(int)或窗口标题(str), 返回PIL Image"""
|
||||
import threading, tempfile
|
||||
from windows_capture import WindowsCapture, Frame, CaptureControl
|
||||
tmp = tempfile.mktemp(suffix='.png')
|
||||
done = threading.Event()
|
||||
kw = {'window_hwnd': hwnd_or_name} if isinstance(hwnd_or_name, int) else {'window_name': hwnd_or_name}
|
||||
cap = WindowsCapture(cursor_capture=False, draw_border=False, **kw)
|
||||
@cap.event
|
||||
def on_frame_arrived(frame: Frame, capture_control: CaptureControl):
|
||||
frame.save_as_image(tmp)
|
||||
capture_control.stop(); done.set()
|
||||
@cap.event
|
||||
def on_closed(): done.set()
|
||||
cap.start_free_threaded()
|
||||
done.wait(timeout=timeout)
|
||||
if os.path.exists(tmp):
|
||||
img = Image.open(tmp); img.load(); os.remove(tmp); return img
|
||||
|
||||
def imshow(mt, sec=0):
|
||||
cv2.imshow('cc', mt)
|
||||
cv2.waitKey(sec)
|
||||
|
||||
def GetWRect(sr):
|
||||
num = int(sr[-1])
|
||||
l, u, r, b = 0, 0, swidth, sheight
|
||||
if 'left' in sr: r = swidth // num
|
||||
if 'right' in sr: l = swidth * (num-1) // num
|
||||
if 'top' in sr: b = sheight // num
|
||||
if 'bottom' in sr: u = sheight * (num-1) // num
|
||||
return [l, u, r, b]
|
||||
|
||||
def FindBlock(fn, wrect=None, verbose=0, threshold=0.8):
|
||||
tic = time.process_time()
|
||||
if wrect is not None and isinstance(wrect, Image.Image):
|
||||
scr, wrect = wrect, None
|
||||
else:
|
||||
if isinstance(wrect, str): wrect = GetWRect(wrect)
|
||||
scr = ImageGrab.grab(wrect)
|
||||
blc = Image.open(fn) if isinstance(fn, str) else fn
|
||||
T = cv2.cvtColor(np.array(blc), cv2.COLOR_RGB2BGR)
|
||||
B = cv2.cvtColor(np.array(scr), cv2.COLOR_RGB2BGR)
|
||||
tsh, tsw = T.shape[:2]
|
||||
if verbose: print('T.shape:', T.shape, '\t', 'B.shape:', B.shape)
|
||||
res = cv2.matchTemplate(B, T, cv2.TM_CCOEFF_NORMED)
|
||||
min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)
|
||||
oj, oi = max_loc
|
||||
if wrect is None: wrect = [0, 0, scr.size[0], scr.size[1]]
|
||||
obj = (oj + wrect[0] + tsw//2, oi + wrect[1] + tsh//2)
|
||||
if verbose:
|
||||
print(f'Max match: {max_val:.4f} at ({oj}, {oi}) cost: {time.process_time() - tic:.3f}s')
|
||||
#sscr = scr.crop([oj, oi, oj+tsw, oi+tsh])
|
||||
#sscr.show()
|
||||
return obj, max_val
|
||||
|
||||
def ScreenCapAt(x, y, r=100):
|
||||
"""物理坐标(x,y)为中心±r的屏幕截图 → PIL Image"""
|
||||
from PIL import ImageGrab
|
||||
return ImageGrab.grab((x-r, y-r, x+r, y+r))
|
||||
|
||||
if __name__ == '__main__':
|
||||
#time.sleep(3)
|
||||
#SetCursorPos( (1640, 131) )
|
||||
#MouseClick()
|
||||
#print(FindBlock('z:/z.png', [1638, 214, 5838, 414], verbose=1))
|
||||
print('completed %.3f' % time.process_time())
|
||||
@@ -0,0 +1,200 @@
|
||||
"""ljqCtrlBg: concise background window control in client pixels.
|
||||
|
||||
Never activates a window, moves the cursor, or injects global input. Posted
|
||||
mouse/key messages are best-effort; always verify visible effects by screenshot.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import ctypes, time
|
||||
from typing import Any, NamedTuple, Optional, Sequence, Union
|
||||
import win32con, win32gui, win32ui
|
||||
from PIL import Image, ImageChops
|
||||
try:
|
||||
ctypes.windll.user32.SetProcessDPIAware()
|
||||
except Exception: pass
|
||||
|
||||
HwndLike = Union[int, str]
|
||||
SMTO_SAFE = win32con.SMTO_BLOCK | win32con.SMTO_ABORTIFHUNG
|
||||
CWP_SKIP = win32con.CWP_SKIPINVISIBLE | win32con.CWP_SKIPDISABLED | win32con.CWP_SKIPTRANSPARENT
|
||||
MOUSE = {"left": (win32con.WM_LBUTTONDOWN, win32con.WM_LBUTTONUP, win32con.MK_LBUTTON),
|
||||
"right": (win32con.WM_RBUTTONDOWN, win32con.WM_RBUTTONUP, win32con.MK_RBUTTON),
|
||||
"middle": (win32con.WM_MBUTTONDOWN, win32con.WM_MBUTTONUP, win32con.MK_MBUTTON)}
|
||||
KEYS = {"backspace": 8, "tab": 9, "enter": 13, "return": 13, "shift": 16, "ctrl": 17, "control": 17,
|
||||
"alt": 18, "esc": 27, "escape": 27, "space": 32, "pageup": 33, "pagedown": 34, "end": 35,
|
||||
"home": 36, "left": 37, "up": 38, "right": 39, "down": 40, "delete": 46, "del": 46}
|
||||
|
||||
class CaptureResult(NamedTuple):
|
||||
image: Image.Image
|
||||
hwnd: int
|
||||
backend: str
|
||||
client_origin: tuple[int, int]
|
||||
client_size: tuple[int, int]
|
||||
size = property(lambda self: self.image.size)
|
||||
origin_screen_phys = property(lambda self: self.client_origin)
|
||||
client_size_phys = property(lambda self: self.client_size)
|
||||
|
||||
def ListWindows(visible_only: bool = True) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
def each(hwnd: int, _: Any) -> bool:
|
||||
title, vis = win32gui.GetWindowText(hwnd), win32gui.IsWindowVisible(hwnd); rect = tuple(map(int, win32gui.GetWindowRect(hwnd)))
|
||||
if (vis or not visible_only) and (title or not visible_only):
|
||||
rows.append({"hwnd": int(hwnd), "title": title, "class": win32gui.GetClassName(hwnd), "rect": rect, "visible": bool(vis)})
|
||||
return True
|
||||
win32gui.EnumWindows(each, None); return rows
|
||||
|
||||
def FindWindow(name: str, exact: bool = False, class_name: Optional[str] = None, visible_only: bool = True) -> int:
|
||||
needle = str(name).lower()
|
||||
for row in ListWindows(visible_only):
|
||||
title = row["title"] or ""; ok = (title == name) if exact else (needle in title.lower())
|
||||
if ok and (class_name is None or row["class"] == class_name): return int(row["hwnd"])
|
||||
raise RuntimeError(f"window not found: {name!r}")
|
||||
|
||||
def ResolveHwnd(hwnd_or_name: HwndLike) -> int:
|
||||
hwnd = int(hwnd_or_name) if isinstance(hwnd_or_name, int) else FindWindow(str(hwnd_or_name))
|
||||
if not win32gui.IsWindow(hwnd): raise RuntimeError(f"invalid hwnd: {hwnd!r}")
|
||||
return hwnd
|
||||
|
||||
def GetWRect(hwnd_or_name: HwndLike) -> tuple[int, int, int, int]:
|
||||
return tuple(map(int, win32gui.GetWindowRect(ResolveHwnd(hwnd_or_name))))
|
||||
|
||||
def ClientSize(hwnd_or_name: HwndLike) -> tuple[int, int]:
|
||||
l, t, r, b = win32gui.GetClientRect(ResolveHwnd(hwnd_or_name)); return int(r - l), int(b - t)
|
||||
|
||||
def ClientOrigin(hwnd_or_name: HwndLike) -> tuple[int, int]:
|
||||
return tuple(map(int, win32gui.ClientToScreen(ResolveHwnd(hwnd_or_name), (0, 0))))
|
||||
|
||||
def ClientRectScreen(hwnd_or_name: HwndLike) -> tuple[int, int, int, int]:
|
||||
x, y = ClientOrigin(hwnd_or_name); w, h = ClientSize(hwnd_or_name); return x, y, x + w, y + h
|
||||
|
||||
def ScreenToClient(hwnd_or_name: HwndLike, x: int, y: int) -> tuple[int, int]:
|
||||
return tuple(map(int, win32gui.ScreenToClient(ResolveHwnd(hwnd_or_name), (int(x), int(y)))))
|
||||
|
||||
def ClientToScreen(hwnd_or_name: HwndLike, x: int, y: int) -> tuple[int, int]:
|
||||
return tuple(map(int, win32gui.ClientToScreen(ResolveHwnd(hwnd_or_name), (int(x), int(y)))))
|
||||
|
||||
def ChildAt(hwnd_or_name: HwndLike, x: int, y: int, coords: str = "client", deep: bool = True) -> tuple[int, int, int]:
|
||||
if coords not in {"client", "screen"}: raise ValueError("coords must be 'client' or 'screen'")
|
||||
root = ResolveHwnd(hwnd_or_name); sx, sy = ClientToScreen(root, x, y) if coords == "client" else (int(x), int(y))
|
||||
hwnd = root
|
||||
while deep:
|
||||
cx, cy = win32gui.ScreenToClient(hwnd, (sx, sy))
|
||||
child = win32gui.ChildWindowFromPointEx(hwnd, (cx, cy), CWP_SKIP)
|
||||
if not child or child == hwnd: break
|
||||
hwnd = child
|
||||
cx, cy = win32gui.ScreenToClient(hwnd, (sx, sy)); return int(hwnd), int(cx), int(cy)
|
||||
|
||||
def _crop_client(hwnd: int, image: Image.Image, size: tuple[int, int]) -> Image.Image:
|
||||
if image.size == size: return image
|
||||
wx, wy, _, _ = win32gui.GetWindowRect(hwnd); ox, oy = ClientOrigin(hwnd); w, h = size; dx, dy = ox - wx, oy - wy
|
||||
if 0 <= dx and 0 <= dy and dx + w <= image.width and dy + h <= image.height: return image.crop((dx, dy, dx + w, dy + h))
|
||||
if image.width >= w and image.height >= h: return image.crop((0, 0, w, h))
|
||||
raise RuntimeError(f"capture frame {image.size} smaller than client area {size}")
|
||||
|
||||
def _grab_wgc(hwnd: int, size: tuple[int, int], timeout: float) -> Image.Image:
|
||||
from windows_capture import WindowsCapture # type: ignore
|
||||
frames: list[Any] = []; errors: list[BaseException] = []
|
||||
cap = WindowsCapture(cursor_capture=False, draw_border=False, window_hwnd=hwnd)
|
||||
@cap.event
|
||||
def on_frame_arrived(frame: Any, control: Any) -> None:
|
||||
try: frames.append(frame.frame_buffer.copy())
|
||||
except BaseException as exc: errors.append(exc)
|
||||
finally:
|
||||
try: control.stop()
|
||||
except Exception: pass
|
||||
@cap.event
|
||||
def on_closed() -> None: pass
|
||||
control = cap.start_free_threaded(); end = time.monotonic() + float(timeout)
|
||||
while not frames and not errors and time.monotonic() < end: time.sleep(0.02)
|
||||
if not frames:
|
||||
try: control.stop()
|
||||
except Exception: pass
|
||||
if errors: raise RuntimeError("WGC callback failed") from errors[0]
|
||||
raise TimeoutError(f"WGC did not produce a frame within {timeout:.1f}s")
|
||||
arr = frames[0]
|
||||
if getattr(arr, "ndim", 0) != 3 or arr.shape[2] < 3: raise RuntimeError(f"bad WGC frame shape: {getattr(arr, 'shape', None)}")
|
||||
return _crop_client(hwnd, Image.fromarray(arr[:, :, :3][:, :, ::-1]).copy(), size)
|
||||
|
||||
def _grab_printwindow(hwnd: int, size: tuple[int, int]) -> tuple[Image.Image, bool]:
|
||||
w, h = size; hdc = win32gui.GetWindowDC(hwnd); src = win32ui.CreateDCFromHandle(hdc)
|
||||
mem = src.CreateCompatibleDC(); bmp = win32ui.CreateBitmap(); bmp.CreateCompatibleBitmap(src, w, h); old = mem.SelectObject(bmp)
|
||||
try:
|
||||
ok = bool(ctypes.windll.user32.PrintWindow(hwnd, mem.GetSafeHdc(), 1))
|
||||
info, bits = bmp.GetInfo(), bmp.GetBitmapBits(True)
|
||||
return Image.frombuffer("RGB", (info["bmWidth"], info["bmHeight"]), bits, "raw", "BGRX", 0, 1).copy(), ok
|
||||
finally:
|
||||
mem.SelectObject(old); win32gui.DeleteObject(bmp.GetHandle()); mem.DeleteDC(); src.DeleteDC(); win32gui.ReleaseDC(hwnd, hdc)
|
||||
|
||||
def GrabWindowBg(hwnd_or_name: HwndLike, backend: str = "auto", timeout: float = 3.0) -> CaptureResult:
|
||||
hwnd = ResolveHwnd(hwnd_or_name); size = ClientSize(hwnd); mode = backend.lower(); wgc_error = ""
|
||||
if min(size) <= 0: raise RuntimeError(f"empty client area for hwnd={hwnd}")
|
||||
if mode in {"auto", "wgc"}:
|
||||
try: return CaptureResult(_grab_wgc(hwnd, size, timeout), hwnd, "wgc", ClientOrigin(hwnd), size)
|
||||
except BaseException as exc:
|
||||
if mode == "wgc": raise
|
||||
wgc_error = f";wgc-error={type(exc).__name__}"
|
||||
if mode in {"auto", "printwindow", "pw"}:
|
||||
image, ok = _grab_printwindow(hwnd, size); label = "printwindow" if ok else "printwindow-best-effort"
|
||||
return CaptureResult(image, hwnd, label + wgc_error, ClientOrigin(hwnd), size)
|
||||
raise ValueError("backend must be 'auto', 'wgc', or 'printwindow'")
|
||||
|
||||
def GrabClientBg(hwnd_or_name: HwndLike, **kwargs: Any) -> Image.Image:
|
||||
return GrabWindowBg(hwnd_or_name, **kwargs).image
|
||||
|
||||
def _lparam(x: int, y: int) -> int: return (int(x) & 0xFFFF) | ((int(y) & 0xFFFF) << 16)
|
||||
|
||||
def _send(hwnd: int, msg: int, wp: int = 0, lp: int = 0, post: bool = True) -> None:
|
||||
if post: win32gui.PostMessage(hwnd, msg, int(wp), int(lp))
|
||||
else: win32gui.SendMessageTimeout(hwnd, msg, int(wp), int(lp), SMTO_SAFE, 1000)
|
||||
|
||||
def ClickBg(hwnd_or_name: HwndLike, x: int, y: int, button: str = "left", coords: str = "client", target_child: bool = True, post: bool = True, interval: float = 0.03, check: bool = True, r: int = 80, wait: float = 0.5) -> bool:
|
||||
root = ResolveHwnd(hwnd_or_name); wins0 = {w["hwnd"]: (w["title"], w["class"]) for w in ListWindows(False)} if check else {}; cap1 = GrabWindowBg(root) if check else None
|
||||
if button.lower() not in MOUSE: raise ValueError(f"unsupported button: {button!r}")
|
||||
if target_child: hwnd, cx, cy = ChildAt(root, x, y, coords)
|
||||
else: hwnd, cx, cy = root, *(ScreenToClient(root, x, y) if coords == "screen" else (int(x), int(y)))
|
||||
down, up, mk = MOUSE[button.lower()]; lp = _lparam(cx, cy); _send(hwnd, win32con.WM_MOUSEMOVE, 0, lp, post); _send(hwnd, down, mk, lp, post)
|
||||
if interval: time.sleep(float(interval))
|
||||
_send(hwnd, up, 0, lp, post)
|
||||
if check:
|
||||
time.sleep(float(wait)); wins1 = {w["hwnd"]: (w["title"], w["class"]) for w in ListWindows(False)}; new = {k: v for k, v in wins1.items() if k not in wins0}; gone = {k: v for k, v in wins0.items() if k not in wins1}; bbox = None
|
||||
if win32gui.IsWindow(root): cap2 = GrabWindowBg(root); im1 = cap1.image.crop((max(0, x-r), max(0, y-r), min(cap1.size[0], x+r), min(cap1.size[1], y+r))); im2 = cap2.image.crop((max(0, x-r), max(0, y-r), min(cap2.size[0], x+r), min(cap2.size[1], y+r))); bbox = ImageChops.difference(im1, im2).getbbox()
|
||||
print(f"[ClickBg check] changed={bool(bbox)} bbox={bbox} new={new} gone={gone}")
|
||||
return True
|
||||
|
||||
def Click(hwnd_or_name: HwndLike, x: int, y: int, **kwargs: Any) -> bool: return ClickBg(hwnd_or_name, x, y, **kwargs)
|
||||
|
||||
def _vk(key: Union[str, int]) -> int:
|
||||
if isinstance(key, int): return int(key)
|
||||
s = str(key).strip(); low = s.lower()
|
||||
if low in KEYS: return int(KEYS[low])
|
||||
if low.startswith("f") and low[1:].isdigit() and 1 <= int(low[1:]) <= 24: return win32con.VK_F1 + int(low[1:]) - 1
|
||||
if len(s) == 1: return int(ctypes.windll.user32.VkKeyScanW(ord(s)) & 0xFF)
|
||||
raise ValueError(f"unknown key: {key!r}")
|
||||
|
||||
def _key_lparam(vk: int, up: bool = False) -> int:
|
||||
lp = 1 | (int(ctypes.windll.user32.MapVirtualKeyW(int(vk), 0)) << 16)
|
||||
return lp | ((1 << 30) | (1 << 31) if up else 0)
|
||||
|
||||
def PressBg(hwnd_or_name: HwndLike, key: Union[str, int], modifiers: Optional[Sequence[Union[str, int]]] = None, post: bool = True, interval: float = 0.02) -> bool:
|
||||
hwnd = ResolveHwnd(hwnd_or_name)
|
||||
if isinstance(key, str) and "+" in key and modifiers is None:
|
||||
parts = [p.strip() for p in key.split("+") if p.strip()]; mods, main = [_vk(p) for p in parts[:-1]], _vk(parts[-1])
|
||||
else: mods, main = [_vk(m) for m in (modifiers or [])], _vk(key)
|
||||
for vk in [*mods, main]: _send(hwnd, win32con.WM_KEYDOWN, vk, _key_lparam(vk), post)
|
||||
if interval: time.sleep(float(interval))
|
||||
for vk in [main, *reversed(mods)]: _send(hwnd, win32con.WM_KEYUP, vk, _key_lparam(vk, True), post)
|
||||
return True
|
||||
|
||||
def Press(hwnd_or_name: HwndLike, key: Union[str, int], **kwargs: Any) -> bool: return PressBg(hwnd_or_name, key, **kwargs)
|
||||
|
||||
def TypeTextBg(hwnd_or_name: HwndLike, text: str, interval: float = 0.0, post: bool = True) -> bool:
|
||||
hwnd = ResolveHwnd(hwnd_or_name)
|
||||
for ch in str(text):
|
||||
_send(hwnd, win32con.WM_CHAR, ord(ch), 1, post)
|
||||
if interval: time.sleep(float(interval))
|
||||
return True
|
||||
|
||||
def SetTextBg(hwnd_or_name: HwndLike, text: str) -> bool:
|
||||
win32gui.SendMessage(ResolveHwnd(hwnd_or_name), win32con.WM_SETTEXT, 0, str(text)); return True
|
||||
|
||||
def GetTextBg(hwnd_or_name: HwndLike) -> str: return win32gui.GetWindowText(ResolveHwnd(hwnd_or_name))
|
||||
|
||||
if __name__ == "__main__": print(f"ljqCtrlBg ready; windows={len(ListWindows())}")
|
||||
@@ -0,0 +1,53 @@
|
||||
# ljqCtrl 使用与坐标转换 SOP
|
||||
|
||||
> **must call update working ckp**：`一律使用物理坐标｜禁pyautogui｜操作前先激活窗口`
|
||||
|
||||
## 0. API 快速参考 (Signatures)
|
||||
- `ljqCtrl.dpi_scale`: float (缩放系数 = 逻辑宽度 / 物理宽度)
|
||||
- `ljqCtrl.Click(x, y=None)`: 模拟点击。支持 `Click((x, y))` 或 `Click(x, y)`
|
||||
- `ljqCtrl.Press(cmd, staytime=0)`: 模拟按键。如 `Press('ctrl+c')`
|
||||
- `ljqCtrl.FindBlock(fn, wrect=None, threshold=0.8)`: 找图。返回 `((center_x, center_y), is_found)`
|
||||
- `ljqCtrl.GrabWindow(hwnd_or_name)`: 前台截图(先Activate), 传hwnd(int)或窗口标题子串(str), 返回PIL Image
|
||||
- `ljqCtrl.GrabWindowBg(hwnd_or_name, timeout=5)`: WGC后台截图(Win10+)
|
||||
- `ljqCtrl.MouseDClick(staytime=0.05)`: 鼠标双击
|
||||
- 可先阅读computer_use.md
|
||||
|
||||
## 1. 环境载入
|
||||
import ljqCtrl
|
||||
|
||||
> **macOS**: 改 `import macljqCtrl as ljqCtrl`（API 镜像，Quartz/screencapture 实现）。依赖 `pyobjc-framework-Quartz`/`-Cocoa`，首次用前 `macljqCtrl.check_permissions()` 自检辅助功能/录屏授权。
|
||||
|
||||
## 2. 核心：High-DPI 物理坐标换算
|
||||
`ljqCtrl` 的 `Click/MoveTo` 接口接收的是**物理像素坐标**。
|
||||
当使用 `pygetwindow` 等其他工具获取窗口位置（逻辑坐标）时，必须除以缩放系数。
|
||||
|
||||
- **换算公式**：`物理坐标 = 逻辑坐标 / ljqCtrl.dpi_scale`
|
||||
|
||||
## 3. 截图bbox → 屏幕物理坐标（核心公式）
|
||||
```python
|
||||
# ui_detect获取的都是物理坐标
|
||||
# ClientToScreen拿客户区原点(逻辑) → 除dpi_scale得物理偏移
|
||||
cx, cy = win32gui.ClientToScreen(hwnd, (0, 0))
|
||||
ox, oy = int(cx / ljqCtrl.dpi_scale), int(cy / ljqCtrl.dpi_scale)
|
||||
ljqCtrl.Click(ox + (bbox[0]+bbox[2])//2, oy + (bbox[1]+bbox[3])//2)
|
||||
```
|
||||
禁止全屏ImageGrab（必须针对窗口），所有逻辑坐标都要转物理。
|
||||
|
||||
**macOS (`macljqCtrl`)**：`GrabScreen(bbox)` 区域截图后，图内点转屏幕物理坐标用 `CropToScreen(bbox, px, py)`，别手搓 `screencapture -R`（它吃逻辑点，会点歪）。
|
||||
|
||||
## 4. 避坑指南
|
||||
- **⚠️ 一律使用物理坐标**：传给 ljqCtrl.Click/SetCursorPos 的坐标必须是物理坐标（=截图像素坐标）。禁止传入逻辑坐标。
|
||||
- **物理验证**：模拟操作前必须确保窗口已通过 `activate()` 置于前台。
|
||||
- **坐标对齐**: 物理坐标 = 截图坐标；ljqCtrl 自动处理 DPI 换算，禁止手动重复计算。
|
||||
- **⚠️ 窗口坐标转换陷阱**：使用 `win32gui.GetWindowRect(hwnd)` 获取的矩形包含标题栏和边框，而截图内容是客户区。点击截图内元素时，必须用 `win32gui.ClientToScreen(hwnd, (0, 0))` 获取客户区原点的屏幕坐标，再加上截图内坐标。禁止直接用 GetWindowRect 左上角 + 截图坐标。**同理禁止 `DwmGetWindowAttribute(hwnd, 9, ...)` 取窗口矩形替代 ClientToScreen，它也包含标题栏/阴影。**
|
||||
- **⚠️ Click 后 0% 像素变化 = 点歪了**：ljqCtrl.Click 会报告像素变化百分比。若为 0% 或接近 0%，说明点击落在了错误位置（坐标计算有误），必须立即停下来诊断坐标转换逻辑，禁止盲目重试。常见原因：用了错误的窗口原点API、忘记 `/dpi_scale`、混淆了客户区与窗口矩形。macOS 上多为忘加裁剪原点（应走 `CropToScreen`）。
|
||||
- **⚠️ win32 DPI 坐标陷阱**：未调用 `SetProcessDPIAware()` 时，`GetWindowRect/ClientToScreen/GetClientRect` 等拿到的窗口/客户区坐标通常是**逻辑坐标**，必须进行换算！
|
||||
- **文本输入**：ljqCtrl 无 TypeText/SendKeys。向输入框键入文本：先点击/三击选中字段，再 `pyperclip.copy('文本'); ljqCtrl.Press('ctrl+v')`。
|
||||
|
||||
## 5. macOS：OCR/vision 认不准图标时，用辅助功能 API 枚举真实控件（强烈推荐）
|
||||
> **两条通路**：①`macljqCtrl.py` 已封装原生 pyobjc AX API（首选，免 shell）：`AXElements(pid或bundle_id或app名)` 枚举控件树(带 role/desc/title/id/value/**enabled**/物理坐标)，`AXFind(...,enabled_only=)` 过滤，`AXClick(node)` = AXPress 优先失败回退物理坐标 Click。②无 pyobjc 时回退下述 osascript 方案。
|
||||
图标类按钮（···更多 / 铅笔编辑 / 关闭等）靠 OCR/vision 极易误判误点。优先走 GUI 优先链的「UIA」层：用 `osascript` 的 System Events 递归 `entire contents` 枚举进程**所有窗口**的真实控件，拿到 `AXRole + description(标识符) + position`，直接 `perform action "AXPress"` 点中。
|
||||
- **关键坑**：弹窗/详情卡常是**独立子窗口**，`front window` 只返回主窗（如红绿灯按钮）。必须 `every window` 遍历 + `entire contents`，否则找不到目标控件。
|
||||
- 控件常自带语义化 `description`/`identifier`（如 `xxx_button_more`），按 description 精确匹配比坐标稳定，枚举一次记下目标标识即可复用。
|
||||
- **坐标换算**：AX 返回的是**逻辑坐标**，截图/Click 用**物理坐标**，retina 屏 ×2（逻辑(537,121)↔物理(1074,242)实测吻合）。AX `AXPress` 直接作用元素免换算；若 AX 偶发 NOTFOUND（时序波动），用换算后物理坐标 `Click` 兜底。
|
||||
- **失焦陷阱**：点击坐标若落在窗口边界外，会点到背后别的 app 导致目标失焦。osascript `tell application "<App>" to activate` 比 ljqCtrl 的 ActivateApp 更可靠，激活后用 `frontmost` 确认。
|
||||
@@ -0,0 +1,590 @@
|
||||
"""
|
||||
macljqCtrl —— ljqCtrl 的 macOS 等价实现 (Quartz CGEvent + screencapture)。
|
||||
与 ljqCtrl.py API 镜像对齐, 跨平台代码可写 `import macljqCtrl as ljqCtrl`。
|
||||
|
||||
CRITICAL: 严禁 import pyautogui。
|
||||
依赖: pyobjc-framework-Quartz, pyobjc-framework-Cocoa (其余 numpy/opencv/Pillow 同 Windows 版)。
|
||||
坐标约定 (与 Windows 版完全一致):
|
||||
- 对外 API 接收【物理像素坐标】(= screencapture/ui_detect 截图内坐标)
|
||||
- dpi_scale = 逻辑点 / 物理像素 (Retina=0.5, 普通屏=1.0)
|
||||
- 逻辑坐标 = 物理坐标 * dpi_scale (CGEvent 内部用逻辑点)
|
||||
|
||||
权限前置 (缺失则键鼠/截图静默失败):
|
||||
- 辅助功能(Accessibility): 系统设置>隐私与安全性>辅助功能, 授权 GA 宿主进程
|
||||
- 屏幕录制(Screen Recording): 同上>屏幕录制
|
||||
用 macljqCtrl.check_permissions() 自检。
|
||||
|
||||
API 快速参考:
|
||||
- dpi_scale: float
|
||||
- Click(x, y=None, check=True): 物理坐标; check=True 比较前后像素变化, 返回变化信息
|
||||
- SetCursorPos((x,y)): 物理坐标移动鼠标
|
||||
- Press(cmd, staytime=0): 快捷键, 如 'cmd+v' 'cmd+c' 'enter' 'cmd+shift+4'
|
||||
- TypeText(s): 直接键入文本(Unicode, 无需剪贴板)
|
||||
- MouseClick / MouseDClick / MouseDown / MouseUp / RightClick
|
||||
- GrabWindow(win) -> PIL Image: 指定窗口截图(物理像素)。win=窗口号(int)或标题/应用名子串(str)
|
||||
- GrabScreen(bbox=None) -> PIL Image: 全屏或区域截图, bbox=(l,u,r,b)物理像素
|
||||
- ScreenCapAt(x, y, r=100) -> PIL Image: 物理坐标(x,y)±r 区域截图
|
||||
- FindBlock(fn, wrect=None, threshold=0.8) -> ((cx,cy), is_found): 模板匹配, 物理坐标
|
||||
- ListWindows(name=None) -> [dict]: 枚举窗口(替代 win32gui), 返回号/标题/应用/物理bbox
|
||||
- ActivateApp(name): 激活应用到前台(替代 win32gui.SetForegroundWindow)
|
||||
"""
|
||||
import os, sys, time, subprocess, tempfile, math
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
from PIL import Image, ImageGrab, ImageEnhance, ImageFilter, ImageDraw
|
||||
import cv2
|
||||
_HAS_CV2 = True
|
||||
except Exception:
|
||||
_HAS_CV2 = False
|
||||
|
||||
import Quartz
|
||||
from AppKit import NSScreen, NSWorkspace, NSPasteboard, NSStringPboardType, NSRunningApplication
|
||||
|
||||
verbose_click = True # Click 像素变化打印开关
|
||||
|
||||
# ---------- 屏幕几何 & dpi_scale ----------
|
||||
_main = Quartz.CGMainDisplayID()
|
||||
_bounds = Quartz.CGDisplayBounds(_main)
|
||||
cwidth = int(_bounds.size.width) # 逻辑点宽 (CGEvent 坐标系)
|
||||
cheight = int(_bounds.size.height)
|
||||
_mode = Quartz.CGDisplayCopyDisplayMode(_main)
|
||||
swidth = int(Quartz.CGDisplayModeGetPixelWidth(_mode)) # 物理像素宽
|
||||
sheight = int(Quartz.CGDisplayModeGetPixelHeight(_mode))
|
||||
dpi_scale = cwidth / swidth if swidth else 1.0 # 逻辑/物理, Retina=0.5
|
||||
|
||||
|
||||
def check_permissions(verbose=True):
|
||||
"""返回 (accessibility_ok, screen_recording_ok)。缺失时打印授权指引。"""
|
||||
sc = bool(Quartz.CGPreflightScreenCaptureAccess()) if hasattr(Quartz, 'CGPreflightScreenCaptureAccess') else None
|
||||
ax = None
|
||||
try:
|
||||
import HIServices
|
||||
ax = bool(HIServices.AXIsProcessTrusted())
|
||||
except Exception:
|
||||
try:
|
||||
from ApplicationServices import AXIsProcessTrusted
|
||||
ax = bool(AXIsProcessTrusted())
|
||||
except Exception:
|
||||
ax = None
|
||||
if verbose:
|
||||
print(f'[PERM] Accessibility(键鼠): {ax} ScreenRecording(截图): {sc}')
|
||||
if ax is False:
|
||||
print(' → 系统设置>隐私与安全性>辅助功能, 勾选 GA 宿主进程后重启 GA')
|
||||
if sc is False:
|
||||
print(' → 系统设置>隐私与安全性>屏幕录制, 勾选 GA 宿主进程后重启 GA')
|
||||
return ax, sc
|
||||
|
||||
|
||||
# ---------- 鼠标 ----------
|
||||
def _post(ev):
|
||||
Quartz.CGEventPost(Quartz.kCGHIDEventTap, ev)
|
||||
|
||||
def _cursor_logical():
|
||||
e = Quartz.CGEventCreate(None)
|
||||
loc = Quartz.CGEventGetLocation(e)
|
||||
return loc.x, loc.y
|
||||
|
||||
def _phys_to_logical(x, y):
|
||||
return x * dpi_scale, y * dpi_scale
|
||||
|
||||
def SetCursorPos(z):
|
||||
"""z=(x,y) 物理坐标。移动鼠标(不点击)。"""
|
||||
lx, ly = _phys_to_logical(int(z[0]), int(z[1]))
|
||||
ev = Quartz.CGEventCreateMouseEvent(None, Quartz.kCGEventMouseMoved,
|
||||
Quartz.CGPointMake(lx, ly), Quartz.kCGMouseButtonLeft)
|
||||
_post(ev)
|
||||
time.sleep(0.05)
|
||||
|
||||
def _mouse_event(etype, button=Quartz.kCGMouseButtonLeft):
|
||||
lx, ly = _cursor_logical()
|
||||
ev = Quartz.CGEventCreateMouseEvent(None, etype, Quartz.CGPointMake(lx, ly), button)
|
||||
_post(ev)
|
||||
|
||||
def MouseDown():
|
||||
_mouse_event(Quartz.kCGEventLeftMouseDown)
|
||||
def MouseUp():
|
||||
_mouse_event(Quartz.kCGEventLeftMouseUp)
|
||||
|
||||
def MouseClick(staytime=0.05):
|
||||
MouseDown(); time.sleep(staytime)
|
||||
MouseUp(); time.sleep(0.05)
|
||||
|
||||
def MouseDClick(staytime=0.05):
|
||||
# 真双击: 同坐标连发2次, 用 click state=2
|
||||
lx, ly = _cursor_logical()
|
||||
p = Quartz.CGPointMake(lx, ly)
|
||||
for state in (1, 2):
|
||||
down = Quartz.CGEventCreateMouseEvent(None, Quartz.kCGEventLeftMouseDown, p, Quartz.kCGMouseButtonLeft)
|
||||
Quartz.CGEventSetIntegerValueField(down, Quartz.kCGMouseEventClickState, state)
|
||||
_post(down)
|
||||
up = Quartz.CGEventCreateMouseEvent(None, Quartz.kCGEventLeftMouseUp, p, Quartz.kCGMouseButtonLeft)
|
||||
Quartz.CGEventSetIntegerValueField(up, Quartz.kCGMouseEventClickState, state)
|
||||
_post(up)
|
||||
time.sleep(0.05)
|
||||
|
||||
def RightClick(staytime=0.05):
|
||||
lx, ly = _cursor_logical()
|
||||
p = Quartz.CGPointMake(lx, ly)
|
||||
down = Quartz.CGEventCreateMouseEvent(None, Quartz.kCGEventRightMouseDown, p, Quartz.kCGMouseButtonRight)
|
||||
_post(down); time.sleep(staytime)
|
||||
up = Quartz.CGEventCreateMouseEvent(None, Quartz.kCGEventRightMouseUp, p, Quartz.kCGMouseButtonRight)
|
||||
_post(up); time.sleep(0.05)
|
||||
|
||||
# ---------- 键盘 ----------
|
||||
# macOS 虚拟键码 (kVK_*)
|
||||
_KEYMAP = {
|
||||
'a':0,'s':1,'d':2,'f':3,'h':4,'g':5,'z':6,'x':7,'c':8,'v':9,'b':11,'q':12,
|
||||
'w':13,'e':14,'r':15,'y':16,'t':17,'1':18,'2':19,'3':20,'4':21,'6':22,'5':23,
|
||||
'=':24,'9':25,'7':26,'-':27,'8':28,'0':29,']':30,'o':31,'u':32,'[':33,'i':34,
|
||||
'p':35,'l':37,'j':38,"'":39,'k':40,';':41,'\\':42,',':43,'/':44,'n':45,'m':46,
|
||||
'.':47,'`':50,
|
||||
'enter':36,'return':36,'tab':48,'space':49,' ':49,'delete':51,'backspace':51,
|
||||
'esc':53,'escape':53,'forwarddelete':117,
|
||||
'left':123,'right':124,'down':125,'up':126,
|
||||
'home':115,'end':119,'pageup':116,'pagedown':121,
|
||||
'f1':122,'f2':120,'f3':99,'f4':118,'f5':96,'f6':97,'f7':98,'f8':100,'f9':101,
|
||||
'f10':109,'f11':103,'f12':111,
|
||||
}
|
||||
_MODS = {
|
||||
'cmd':Quartz.kCGEventFlagMaskCommand, 'command':Quartz.kCGEventFlagMaskCommand,
|
||||
'ctrl':Quartz.kCGEventFlagMaskControl, 'control':Quartz.kCGEventFlagMaskControl,
|
||||
'alt':Quartz.kCGEventFlagMaskAlternate, 'option':Quartz.kCGEventFlagMaskAlternate,
|
||||
'opt':Quartz.kCGEventFlagMaskAlternate,
|
||||
'shift':Quartz.kCGEventFlagMaskShift,
|
||||
'fn':Quartz.kCGEventFlagMaskSecondaryFn,
|
||||
}
|
||||
|
||||
def _key_tap(keycode, flags=0):
|
||||
down = Quartz.CGEventCreateKeyboardEvent(None, keycode, True)
|
||||
if flags: Quartz.CGEventSetFlags(down, flags)
|
||||
_post(down)
|
||||
up = Quartz.CGEventCreateKeyboardEvent(None, keycode, False)
|
||||
if flags: Quartz.CGEventSetFlags(up, flags)
|
||||
_post(up)
|
||||
|
||||
def Press(cmd, staytime=0):
|
||||
"""快捷键。如 'cmd+v' 'cmd+shift+4' 'enter' 'cmd+c'。Win版的 ctrl 在mac多对应 cmd, 调用方自行决定。"""
|
||||
parts = [p.strip().lower() for p in str(cmd).split('+') if p.strip()]
|
||||
flags = 0; key = None
|
||||
for p in parts:
|
||||
if p in _MODS: flags |= _MODS[p]
|
||||
else: key = p
|
||||
if key is None:
|
||||
return
|
||||
kc = _KEYMAP.get(key)
|
||||
if kc is None:
|
||||
# 单字符走 TypeText
|
||||
TypeText(key)
|
||||
else:
|
||||
_key_tap(kc, flags)
|
||||
if staytime: time.sleep(staytime)
|
||||
time.sleep(0.03)
|
||||
|
||||
def TypeText(s):
|
||||
"""直接键入 Unicode 文本 (无需剪贴板)。"""
|
||||
for ch in str(s):
|
||||
ev = Quartz.CGEventCreateKeyboardEvent(None, 0, True)
|
||||
Quartz.CGEventKeyboardSetUnicodeString(ev, len(ch), ch)
|
||||
_post(ev)
|
||||
ev2 = Quartz.CGEventCreateKeyboardEvent(None, 0, False)
|
||||
Quartz.CGEventKeyboardSetUnicodeString(ev2, len(ch), ch)
|
||||
_post(ev2)
|
||||
time.sleep(0.005)
|
||||
|
||||
# 剪贴板 (替代 pyperclip)
|
||||
def set_clipboard(text):
|
||||
pb = NSPasteboard.generalPasteboard()
|
||||
pb.clearContents()
|
||||
pb.setString_forType_(text, NSStringPboardType)
|
||||
|
||||
def get_clipboard():
|
||||
pb = NSPasteboard.generalPasteboard()
|
||||
return pb.stringForType_(NSStringPboardType)
|
||||
|
||||
def Paste(text):
|
||||
"""把 text 放剪贴板并 cmd+v 粘贴 (等价 Win版 pyperclip+ctrl+v)。"""
|
||||
set_clipboard(text); time.sleep(0.05); Press('cmd+v')
|
||||
|
||||
|
||||
# ---------- 截图 (screencapture, 输出物理像素) ----------
|
||||
def GrabScreen(bbox=None):
|
||||
"""全屏或区域截图 -> PIL Image。bbox=(l,u,r,b) 物理像素坐标。
|
||||
传 bbox 后图内坐标相对裁剪原点; 转屏幕绝对坐标用 CropToScreen, 勿手搓 screencapture -R。
|
||||
"""
|
||||
fd, fn = tempfile.mkstemp(suffix='.png'); os.close(fd)
|
||||
try:
|
||||
if bbox:
|
||||
l, u, r, b = bbox
|
||||
# screencapture -R 用逻辑点; 转换 物理->逻辑
|
||||
lx, ly = l*dpi_scale, u*dpi_scale
|
||||
lw, lh = (r-l)*dpi_scale, (b-u)*dpi_scale
|
||||
cmd = ['/usr/sbin/screencapture','-x','-t','png',
|
||||
f'-R{lx:.0f},{ly:.0f},{lw:.0f},{lh:.0f}', fn]
|
||||
else:
|
||||
cmd = ['/usr/sbin/screencapture','-x','-t','png', fn]
|
||||
subprocess.run(cmd, check=True, capture_output=True)
|
||||
return Image.open(fn).copy()
|
||||
finally:
|
||||
try: os.remove(fn)
|
||||
except Exception: pass
|
||||
|
||||
def ScreenCapAt(x, y, r=100):
|
||||
"""物理坐标(x,y)为中心±r 截图 -> PIL Image。"""
|
||||
return GrabScreen((x-r, y-r, x+r, y+r))
|
||||
|
||||
def CropToScreen(bbox, x, y=None):
|
||||
"""裁剪图内坐标 -> 屏幕绝对物理坐标。bbox=GrabScreen 用的 (l,u,r,b) 物理像素。
|
||||
(x,y)=在 GrabScreen(bbox) 返回图内找到的点。返回可直接喂给 Click 的 (X,Y)。
|
||||
macOS 版的 ClientToScreen: 纯加裁剪原点偏移, 不做缩放(裁剪图与 bbox 同物理像素)。"""
|
||||
if y is None and isinstance(x, (tuple, list)):
|
||||
x, y = x[0], x[1]
|
||||
return int(bbox[0] + x), int(bbox[1] + y)
|
||||
|
||||
def GrabWindow(win):
|
||||
"""窗口截图 -> PIL Image。win=窗口号(int) 或 标题/应用名子串(str)。物理像素。"""
|
||||
if isinstance(win, str):
|
||||
ws = ListWindows(win)
|
||||
if not ws: raise RuntimeError(f'window not found: {win}')
|
||||
win = ws[0]['id']
|
||||
fd, fn = tempfile.mkstemp(suffix='.png'); os.close(fd)
|
||||
try:
|
||||
subprocess.run(['/usr/sbin/screencapture','-x','-o','-l',str(win),'-t','png',fn],
|
||||
check=True, capture_output=True)
|
||||
return Image.open(fn).copy()
|
||||
finally:
|
||||
try: os.remove(fn)
|
||||
except Exception: pass
|
||||
|
||||
# ---------- 窗口枚举 / 激活 (替代 win32gui) ----------
|
||||
def ListWindows(name=None):
|
||||
"""枚举屏上窗口 -> [{'id','title','app','bbox'(物理像素 l,u,r,b),'pid'}]。
|
||||
name: 标题或应用名子串过滤(不区分大小写)。"""
|
||||
opts = Quartz.kCGWindowListOptionOnScreenOnly | Quartz.kCGWindowListExcludeDesktopElements
|
||||
infos = Quartz.CGWindowListCopyWindowInfo(opts, Quartz.kCGNullWindowID)
|
||||
out = []
|
||||
inv = 1.0 / dpi_scale # 逻辑->物理
|
||||
for w in infos:
|
||||
b = w.get('kCGWindowBounds') or {}
|
||||
layer = w.get('kCGWindowLayer', 0)
|
||||
if layer != 0: # 只要普通应用窗口层
|
||||
continue
|
||||
title = w.get('kCGWindowName') or ''
|
||||
app = w.get('kCGWindowOwnerName') or ''
|
||||
l = b.get('X', 0)*inv; u = b.get('Y', 0)*inv
|
||||
r = l + b.get('Width', 0)*inv; bo = u + b.get('Height', 0)*inv
|
||||
rec = {'id': int(w.get('kCGWindowNumber', 0)), 'title': title, 'app': app,
|
||||
'pid': int(w.get('kCGWindowOwnerPID', 0)),
|
||||
'bbox': (int(l), int(u), int(r), int(bo))}
|
||||
if name:
|
||||
n = name.lower()
|
||||
if n not in title.lower() and n not in app.lower():
|
||||
continue
|
||||
out.append(rec)
|
||||
return out
|
||||
|
||||
def ActivateApp(target):
|
||||
"""激活应用到前台 (替代 SetForegroundWindow)。
|
||||
|
||||
macOS 的前台是 *应用* 粒度而非窗口句柄粒度, 故无法 1:1 镜像 Win 版
|
||||
Activate(hwnd)。target 支持两种定位键, 优先用精确的:
|
||||
- int : 进程 pid (来自 ListWindows 的 'pid' 字段) —— 精确, 不受语言/本地化影响, **推荐**
|
||||
- str : 应用名 / bundle id。按精确度分级匹配, 避免误中同厂商应用:
|
||||
① bundle id 精确等值 (如 'com.tencent.meeting') —— 最可靠
|
||||
② localizedName 精确等值 (如 '腾讯会议')
|
||||
③ localizedName 子串 (最后兜底, 可能模糊)
|
||||
注意: 不对 bundle id 做子串匹配, 因同厂商应用共享前缀
|
||||
(微信 com.tencent.xinWeChat 与腾讯会议 com.tencent.meeting 都含 'tencent')。
|
||||
返回是否成功。"""
|
||||
ws = NSWorkspace.sharedWorkspace()
|
||||
# 1) pid 精确激活 (首选)
|
||||
if isinstance(target, int):
|
||||
app = NSRunningApplication.runningApplicationWithProcessIdentifier_(target)
|
||||
if app is not None:
|
||||
app.activateWithOptions_(1 << 1) # NSApplicationActivateAllWindows
|
||||
time.sleep(0.3)
|
||||
return True
|
||||
return False
|
||||
# 2) 字符串: 按精确度分级匹配 (避免同厂商前缀误伤)
|
||||
key = str(target)
|
||||
keyl = key.lower()
|
||||
apps = list(ws.runningApplications())
|
||||
def _fire(app):
|
||||
app.activateWithOptions_(1 << 1)
|
||||
time.sleep(0.3)
|
||||
return True
|
||||
# ① bundle id 精确等值
|
||||
for app in apps:
|
||||
if (app.bundleIdentifier() or '').lower() == keyl:
|
||||
return _fire(app)
|
||||
# ② localizedName 精确等值
|
||||
for app in apps:
|
||||
if (app.localizedName() or '').lower() == keyl:
|
||||
return _fire(app)
|
||||
# ③ localizedName 子串 (兜底)
|
||||
for app in apps:
|
||||
if keyl in (app.localizedName() or '').lower():
|
||||
return _fire(app)
|
||||
# 3) 兜底用 open -a
|
||||
try:
|
||||
subprocess.run(['open', '-a', str(target)], check=True, capture_output=True)
|
||||
time.sleep(0.5); return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# ---------- 模板匹配 FindBlock ----------
|
||||
def GetWRect(sr):
|
||||
"""快捷区域名 -> 物理像素 [l,u,r,b]。如 'left2'=左半屏, 'top3'=上1/3。"""
|
||||
num = int(sr[-1])
|
||||
l, u, r, b = 0, 0, swidth, sheight
|
||||
if 'left' in sr: r = swidth // num
|
||||
if 'right' in sr: l = swidth * (num - 1) // num
|
||||
if 'top' in sr: b = sheight // num
|
||||
if 'bottom' in sr: u = sheight * (num - 1) // num
|
||||
return [l, u, r, b]
|
||||
|
||||
def FindBlock(fn, wrect=None, verbose=0, threshold=0.8):
|
||||
"""在屏幕(或wrect区域)内找模板图 fn。返回 ((cx,cy), is_found), 物理坐标。
|
||||
fn: 模板图路径(str)或 PIL Image。
|
||||
wrect: None=全屏 | [l,u,r,b]物理像素 | 'left2'等快捷名 | PIL Image(直接当搜索底图)。"""
|
||||
if not _HAS_CV2:
|
||||
raise RuntimeError('FindBlock 需要 opencv-python 和 Pillow')
|
||||
if wrect is not None and isinstance(wrect, Image.Image):
|
||||
scr, wrect = wrect, None
|
||||
else:
|
||||
if isinstance(wrect, str): wrect = GetWRect(wrect)
|
||||
scr = GrabScreen(wrect) # 物理像素
|
||||
blc = Image.open(fn) if isinstance(fn, str) else fn
|
||||
T = cv2.cvtColor(np.array(blc), cv2.COLOR_RGB2BGR)
|
||||
B = cv2.cvtColor(np.array(scr), cv2.COLOR_RGB2BGR)
|
||||
tsh, tsw = T.shape[:2]
|
||||
res = cv2.matchTemplate(B, T, cv2.TM_CCOEFF_NORMED)
|
||||
_, max_val, _, max_loc = cv2.minMaxLoc(res)
|
||||
oj, oi = max_loc
|
||||
if wrect is None: wrect = [0, 0, scr.size[0], scr.size[1]]
|
||||
obj = (oj + wrect[0] + tsw // 2, oi + wrect[1] + tsh // 2)
|
||||
if verbose:
|
||||
print(f'FindBlock {fn}: score={max_val:.4f} at phys={obj}')
|
||||
return obj, max_val > threshold
|
||||
|
||||
def imshow(mt, sec=0):
|
||||
cv2.imshow('cc', mt); cv2.waitKey(sec)
|
||||
|
||||
|
||||
|
||||
# ---------- Click (带像素变化检测) ----------
|
||||
def Click(x, y=None, check=True, r=60):
|
||||
"""物理坐标点击。check=True 时比较点击前后周边像素变化, 返回 (变化百分比, 截图)。
|
||||
若变化≈0 → 可能点歪了 (同 Win 版语义)。"""
|
||||
if y is None and isinstance(x, (tuple, list)):
|
||||
x, y = x[0], x[1]
|
||||
x, y = int(x), int(y)
|
||||
before = None
|
||||
if check:
|
||||
try: before = np.array(ScreenCapAt(x, y, r))
|
||||
except Exception: before = None
|
||||
SetCursorPos((x, y)); time.sleep(0.05)
|
||||
MouseClick()
|
||||
if check:
|
||||
time.sleep(0.25)
|
||||
try:
|
||||
after = np.array(ScreenCapAt(x, y, r))
|
||||
except Exception:
|
||||
return None
|
||||
if before is not None and before.shape == after.shape:
|
||||
diff = np.mean(np.any(before != after, axis=-1)) * 100
|
||||
if verbose_click:
|
||||
print(f'Click({x},{y}) pixel change: {diff:.1f}%')
|
||||
if diff < 0.5:
|
||||
print(f'[WARN] Click({x},{y}) 像素变化≈0%, 可能点歪了, 请诊断坐标! '
|
||||
'常见错因: 用了裁剪图内坐标却没加裁剪原点(用 CropToScreen), '
|
||||
'或对已是物理像素的坐标又做了 *dpi_scale 换算。')
|
||||
return diff, Image.fromarray(after)
|
||||
return None
|
||||
|
||||
|
||||
# ---------- AX 辅助功能控件枚举 (UIA 层的 macOS 等价) ----------
|
||||
try:
|
||||
from ApplicationServices import (
|
||||
AXUIElementCreateApplication, AXUIElementCopyAttributeValue,
|
||||
AXValueGetValue, AXUIElementPerformAction, AXIsProcessTrusted,
|
||||
kAXChildrenAttribute, kAXRoleAttribute, kAXDescriptionAttribute,
|
||||
kAXPositionAttribute, kAXSizeAttribute, kAXWindowsAttribute,
|
||||
kAXTitleAttribute, kAXValueCGPointType, kAXValueCGSizeType,
|
||||
kAXPressAction, kAXEnabledAttribute,
|
||||
)
|
||||
_HAS_AX = True
|
||||
except ImportError:
|
||||
_HAS_AX = False
|
||||
|
||||
|
||||
def _resolve_pid(target):
|
||||
"""target(int pid | str bundle_id/应用名) → pid(int)。
|
||||
|
||||
str 优先按 bundle id 精确匹配, 再按 localizedName 精确/子串兜底,
|
||||
与 ActivateApp 的匹配纪律一致, 避免同厂商前缀误伤。"""
|
||||
if isinstance(target, int):
|
||||
return int(target)
|
||||
key = str(target); keyl = key.lower()
|
||||
ws = NSWorkspace.sharedWorkspace()
|
||||
apps = list(ws.runningApplications())
|
||||
for a in apps: # ① bundle id 精确
|
||||
if (a.bundleIdentifier() or '') == key:
|
||||
return int(a.processIdentifier())
|
||||
for a in apps: # ② localizedName 精确
|
||||
if (a.localizedName() or '').lower() == keyl:
|
||||
return int(a.processIdentifier())
|
||||
for a in apps: # ③ localizedName 子串
|
||||
if keyl in (a.localizedName() or '').lower():
|
||||
return int(a.processIdentifier())
|
||||
raise ValueError(f'找不到 target={target!r} 对应的运行中应用')
|
||||
|
||||
|
||||
def _ax_attr(el, key):
|
||||
"""读取单个 AX 属性,失败返回 None。"""
|
||||
if not _HAS_AX:
|
||||
return None
|
||||
err, val = AXUIElementCopyAttributeValue(el, key, None)
|
||||
return val if err == 0 else None
|
||||
|
||||
|
||||
def AXElements(target, max_depth=10, include_zero_size=False):
|
||||
"""枚举应用控件树。
|
||||
|
||||
Parameters
|
||||
----------
|
||||
target : int | str
|
||||
pid(int) 或 bundle_id(str, 如 'com.tencent.meeting')。
|
||||
max_depth : int
|
||||
递归深度上限。
|
||||
include_zero_size : bool
|
||||
是否包含 w<=0 或 h<=0 的零尺寸节点。
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[dict] : 每项含 role/desc/title/id/value/x/y/w/h(物理像素)/depth/el。
|
||||
value 为控件当前值(文本/输入框内容等),非文本值为 None。
|
||||
el 是原始 AXUIElement 引用,目标窗口关闭/重建后失效,AXPress 前宜就近重新枚举。
|
||||
"""
|
||||
if not _HAS_AX:
|
||||
raise RuntimeError(
|
||||
'AX 不可用。请安装: pip install pyobjc-framework-ApplicationServices')
|
||||
if not AXIsProcessTrusted():
|
||||
raise PermissionError('需要授予辅助功能权限(系统设置 > 隐私与安全 > 辅助功能)')
|
||||
pid = _resolve_pid(target)
|
||||
|
||||
app_el = AXUIElementCreateApplication(pid)
|
||||
wins = _ax_attr(app_el, kAXWindowsAttribute) or []
|
||||
scale = dpi_scale # 逻辑/物理, Retina=0.5
|
||||
|
||||
results = []
|
||||
|
||||
def _walk(el, depth):
|
||||
if depth > max_depth:
|
||||
return
|
||||
role = _ax_attr(el, kAXRoleAttribute)
|
||||
desc = _ax_attr(el, kAXDescriptionAttribute)
|
||||
title = _ax_attr(el, kAXTitleAttribute)
|
||||
ident = _ax_attr(el, 'AXIdentifier')
|
||||
value = _ax_attr(el, 'AXValue')
|
||||
enabled = _ax_attr(el, kAXEnabledAttribute)
|
||||
# AXValue 多为 str/num(文本/输入框);若是 AXValueRef(坐标等)忽略
|
||||
if value is not None and not isinstance(value, (str, int, float, bool)):
|
||||
value = None
|
||||
pos_val = _ax_attr(el, kAXPositionAttribute)
|
||||
sz_val = _ax_attr(el, kAXSizeAttribute)
|
||||
# 解包坐标(逻辑点)
|
||||
px = py_ = pw = ph = 0.0
|
||||
if pos_val is not None:
|
||||
ok, pt = AXValueGetValue(pos_val, kAXValueCGPointType, None)
|
||||
if ok:
|
||||
px, py_ = pt.x, pt.y
|
||||
if sz_val is not None:
|
||||
ok, sz = AXValueGetValue(sz_val, kAXValueCGSizeType, None)
|
||||
if ok:
|
||||
pw, ph = sz.width, sz.height
|
||||
# 转物理像素
|
||||
phys_x = px / scale if scale else px
|
||||
phys_y = py_ / scale if scale else py_
|
||||
phys_w = pw / scale if scale else pw
|
||||
phys_h = ph / scale if scale else ph
|
||||
|
||||
if not include_zero_size and (phys_w <= 0 or phys_h <= 0):
|
||||
pass # 跳过零尺寸,但仍递归子节点
|
||||
else:
|
||||
results.append(dict(
|
||||
depth=depth, role=role, desc=desc, title=title, id=ident,
|
||||
value=value, enabled=bool(enabled) if enabled is not None else None,
|
||||
x=round(phys_x), y=round(phys_y),
|
||||
w=round(phys_w), h=round(phys_h), el=el))
|
||||
for child in (_ax_attr(el, kAXChildrenAttribute) or []):
|
||||
_walk(child, depth + 1)
|
||||
|
||||
for win in wins:
|
||||
_walk(win, 0)
|
||||
return results
|
||||
|
||||
|
||||
def AXPress(element) -> bool:
|
||||
"""对 AX element 执行 Press 动作(免坐标点击)。"""
|
||||
if not _HAS_AX:
|
||||
return False
|
||||
err = AXUIElementPerformAction(element, kAXPressAction)
|
||||
return err == 0
|
||||
|
||||
|
||||
def AXClick(node, check=True) -> bool:
|
||||
"""点击控件: AXPress 优先(免坐标), 失败回退到中心点物理坐标 Click。
|
||||
node: AXFind/AXElements 返回的 dict(含 el 与 x/y/w/h), 或裸 AXUIElement。
|
||||
返回是否点击成功(回退路径据像素变化判定, check=False 时无法判定按 True)。
|
||||
呼应 computer_use SOP: AXPress 优先, 回退 Click(phys_cx, phys_cy)。"""
|
||||
if not isinstance(node, dict):
|
||||
return AXPress(node)
|
||||
if node.get('enabled') is False:
|
||||
print(f"[WARN] AXClick: 控件 disabled (role={node.get('role')}, "
|
||||
f"title={node.get('title')!r}), 点击可能无效")
|
||||
if AXPress(node.get('el')):
|
||||
return True
|
||||
# 回退: 中心点物理坐标
|
||||
cx = node['x'] + node['w'] // 2
|
||||
cy = node['y'] + node['h'] // 2
|
||||
res = Click(cx, cy, check=check)
|
||||
if not check or res is None:
|
||||
return check is False # 无法判定时: 关检查按成功, 截图失败按失败
|
||||
diff, _ = res
|
||||
return diff >= 0.5
|
||||
|
||||
|
||||
def AXFind(target, role=None, desc=None, title=None, identifier=None,
|
||||
enabled_only=False, max_depth=10):
|
||||
"""枚举并过滤控件。所有过滤条件为子串匹配(大小写不敏感)。
|
||||
enabled_only=True 时只返回 enabled 的控件(SOP: 点前查 disabled)。
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[dict] : 匹配项,同 AXElements 返回格式。
|
||||
"""
|
||||
def _hit(field, needle):
|
||||
return needle is None or (field and needle.lower() in field.lower())
|
||||
return [n for n in AXElements(target, max_depth=max_depth)
|
||||
if _hit(n['role'], role) and _hit(n['desc'], desc)
|
||||
and _hit(n['title'], title) and _hit(n['id'], identifier)
|
||||
and not (enabled_only and n.get('enabled') is False)]
|
||||
|
||||
|
||||
# ---------- API 镜像别名 (drop-in 替换 ljqCtrl 用) ----------
|
||||
click = Click
|
||||
press = Press
|
||||
activate = ActivateApp # 注意: Win 版 Activate(hwnd) 收窗口句柄, Mac 版收应用名子串
|
||||
GrabWindowBg = GrabWindow # Mac screencapture 本身支持后台窗口
|
||||
VK_CODE = _KEYMAP # 名称兼容
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
print('--- macljqCtrl self-check ---')
|
||||
check_permissions()
|
||||
print('cursor(logical):', _cursor_logical())
|
||||
print('windows(top5):')
|
||||
for w in ListWindows()[:5]:
|
||||
print(' ', w['id'], '|', w['app'], '|', w['title'][:30], '|', w['bbox'])
|
||||
@@ -0,0 +1,55 @@
|
||||
# 记忆整理 SOP
|
||||
|
||||
## 核心原则：存在性编码
|
||||
LLM自身是压缩器+解码器。L1只需让它**意识到某类知识存在**，它就能通过tool call自行取用深层内容。
|
||||
|
||||
**L1本质：用最短词数表达——什么场景下有什么记忆可用（存在性）。**
|
||||
|
||||
L1两类内容，统一ROI评估：
|
||||
- **存在性指针**：指向L2/L3知识的最短触发词
|
||||
- **行为规则**：不提醒就会犯的错（致命/高频均可，只要ROI过门槛）
|
||||
|
||||
ROI = (不放这几个词的犯错概率 × 代价) / 每轮词数成本
|
||||
|
||||
## 快速判断
|
||||
**该留**：反直觉触发词——没提示就想不到去查SOP的场景词。如`tmwebdriver_sop(httponly cookie)`：没有`httponly cookie`这个词，你不会想到取cookie要查tmwebdriver
|
||||
**该删**：
|
||||
- 名字翻译：`proxy-pool/(代理池)` → 名字自解释，括号是废词，直接`proxy-pool`即可
|
||||
- 内容描述：`opencli_sop(66站点CLI,复用Chrome session)` → 实现细节属于SOP内部，不是触发场景
|
||||
- 直觉能力：不提醒也能想到 → 0收益，白交每轮成本
|
||||
- 冗余：L3已覆盖的规则 / L1其他行已含的片段
|
||||
|
||||
## 压缩四原则
|
||||
1. **命名自解释 > 加描述**：SOP名能说清的，L1不加注释；改名的ROI常高于改L1
|
||||
- 推论：极低ROI且文件名自解释的条目可不单独列入L1，但仍需被原则2的集合包含住，靠集合触发词+ls兜底
|
||||
2. **存在性集合最小描述**：多个相近条目若可被同一上位场景覆盖，用集合名表达这类能力的存在，不必平铺子项。如`qq操作/飞书操作/企微操作`→`im操作:*_im_sop`；子项名自解释则只列名不翻译
|
||||
3. **条目 = 场景↔方案存在性**：如`视频理解:yt-dlp取字幕`、`fofa(资产测绘)`——场景名是触发词，方案名编码存在性；括号内**只放反直觉触发词**，非反直觉的（纯翻译/内容描述/实现细节）全是浪费
|
||||
- **触发词判定**：假设用户说出这个词，你能否想到去查对应SOP？能→直觉(不需要)；不能→反直觉(必须留)。如`game_download(百度网盘)`：用户说"百度网盘下载"，没这个词你不会想到game_download→必须留
|
||||
4. **分层归位**：L1内部调整位置——带行为规则或高频高ROI的条目放上方场景行，纯存在性指针归L1下方平铺列表。**归位≠移出L1，存在性不可丢**
|
||||
|
||||
## 整理流程
|
||||
1. 逐行读L1，按`|`拆片段，先分类：存在性指针 / RULES / 翻译 / 内容描述 / 实现细节 / 冗余
|
||||
2. 先清RULES：逐条问“这是全局高ROI，还是特定场景低危险规则？”
|
||||
- 全局高ROI → 留
|
||||
- 特定场景 / 低危险 → 降级到L3或删除
|
||||
3. 再清存在性指针：检查是否在表达**场景↔方案存在性**；场景触发词只在**反直觉**时才加，翻译/内容描述/实现细节删掉
|
||||
4. 审计幽灵条目：L1指向的L3文件是否真实存在？不存在则按ROI决定删除或补建文件
|
||||
5. 禁值存储：L1括号内禁放参数值(IP/端口/凭证等)，这些属于L2，L1只编码存在性
|
||||
6. 检查L3文件名是否自解释；能靠改名解决的，不靠L1加描述；最后验证总行数 ≤ 30
|
||||
|
||||
**红线**：记忆修改是持久性伤害，错误每轮复利。L1只能patch词级别修改，禁overwrite
|
||||
产生误导应及时修正L1或记忆更名
|
||||
|
||||
|
||||
## L2 瘦身流程（冗余长段→L3，事实无损）
|
||||
适用：L2 某段冗长（服务器/工具详情），需压缩但禁丢事实。
|
||||
1. **先迁再压**：把该段完整事实迁到/合并进 L3 专属 SOP（已有同主题 SOP 就并入，勿重复建，先 `ls ../memory/` 查），L2 只留 6-9 行"连接方式+服务端点+高频坑+指针(见 xxx_sop.md)"。
|
||||
2. 迁移后同步 L1 加新 SOP 名（自解释即可，勿加冗余括号）。
|
||||
3. 每节独立闭环：迁移→压 L2→同步 L1，再进下一节，限制失败半径。
|
||||
4. 验证：核对 6 项(每个新 SOP 文件存在 & 已入 L1)、L2 无遗留脏字符、总行数下降。
|
||||
|
||||
## 坑：L2 历史脏字符导致 file_patch 匹配失败
|
||||
- 现象：`file_patch` 连续报"未找到匹配旧文本块"，但肉眼看 old_content 与文件一致。
|
||||
- 根因：老记录行首可能混入真实的多余 `|`（或全角/箭头字节差异），复制时看不出。
|
||||
- 排查：`for i,l in enumerate(lines): print(i+1, repr(l[:20]))` 用 repr 看真实字节。
|
||||
- 解法：**改用 Python 按行号切片替换**：`lines=open(p,encoding='utf-8').read().split('\n'); assert lines[a].startswith(...); newlines=lines[:a]+repl+lines[b:]; open(p,'w',encoding='utf-8').write('\n'.join(newlines))`。前后加 `assert startswith` 双锚点防错位，改完 repr 复核。此法顺带清脏字符。
|
||||
@@ -0,0 +1,92 @@
|
||||
## 0. 核心公理 (Core Axioms - 最高优先级)
|
||||
1. **行动验证原则 (Action-Verified Only)**
|
||||
* **定义**：任何写入 L1/L2/L3 的信息，必须源自**成功的工具调用结果**（如 `shell` 执行成功、`file_read` 确认内容存在、代码运行通过）。
|
||||
* **禁止**：严禁将模型的“固有知识”、“推理猜测”、“未执行的计划”或“未验证的假设”作为事实写入。
|
||||
* **口号**：**No Execution, No Memory. (无行动，不记忆)**
|
||||
2. **神圣不可删改性 (Sanctity of Verified Data)**
|
||||
* **定义**：凡是经过行动验证的有效配置、避坑指南、关键路径，在重构（Refactoring/GC）时**严禁丢弃**。
|
||||
* **操作**：可以压缩文字、可以迁移层级（从 L2 移到 L3），但绝不能丢失信息的准确性和可追溯性。
|
||||
* 记忆修改时请极度小心，尽量不要overwrite或code run。只能少量patch，改不动宁愿不改。
|
||||
3. **禁止存储易变状态 (No Volatile State)**
|
||||
* **定义**：严禁存储随时间/会话高频变化的数据。
|
||||
* **示例**：当前时间戳、临时 Session ID、正在运行的 PID、某个具体绝对路径、连接的设备信息
|
||||
4. **最小充分指针 (Minimum Sufficient Pointer)**
|
||||
* 上层只留能定位下层的最短标识，多一词即冗余。
|
||||
---
|
||||
## 记忆层级架构
|
||||
```
|
||||
L1: global_mem_insight.txt (极简索引层 - 严格控制 ≤30 行)
|
||||
↓ 导航指向 (Pointer)
|
||||
L2: global_mem.txt (事实库层 - 现短但会膨胀)
|
||||
↓ 详细引用 (Reference)
|
||||
L3: ../memory/ (记录库层 - 包含 .md/.py 等各类文件)
|
||||
L4: ../memory/L4_raw_sessions/ (历史会话层 - scheduler反射自动收集，可定位过往上下文)
|
||||
```
|
||||
---
|
||||
## 各层职责与原则
|
||||
### L1：全局内存索引 (global_mem_insight.txt)
|
||||
**职责**：为 L2 和 L3 提供极简导航索引，确保关键能力可被发现。
|
||||
**特征**：
|
||||
- 体积限制：≤ 30 行（硬约束），< 1k tokens（期望）。严禁填写细节（除非极高频任务）
|
||||
- 内容：两层「场景关键词→记忆定位」映射 + RULES（红线规则 + 高频犯错点）
|
||||
- 第一层：高频场景 key→value（直接给出 sop/py/L2 section 名），自包含名称只写一词不重复翻译
|
||||
- 第二层：低频场景仅列关键词，需要时 read L2 或 ls L3 自行定位
|
||||
- 核心：场景触发词极重要（不索引则不知有此能力），但严禁写How-to细节
|
||||
- RULES：压缩版避坑准则，包含：
|
||||
- 红线规则（致命型）：违反会导致进程终止或系统崩溃（如 `禁无条件杀python(会杀自己)`）
|
||||
- 红线规则（隐蔽型）：违反不报错但产生错误结果（如 `搜索用google不用百度`）
|
||||
- 高频犯错点：容易遗忘的关键约束（如 `es(PATH有)` 防止找路径）
|
||||
- 更新：L2/L3 有新增/删除时，判断频率归入对应层。修改时请极度小心，不允许overwrite或code run。只能少量patch，改不动宁愿不改。
|
||||
**禁止**：严禁写入密码、API Key。允许内联非敏感触发参数（如代理端口）。不写 "How to" 或详细解释。严禁包含特定任务的技术细节（特定任务细节应该在L3）。更加严禁写入日志记录！
|
||||
---
|
||||
### L2：全局事实库 (global_mem.txt)
|
||||
**职责**：存储全局环境性事实（路径、凭证、配置、常量等）。
|
||||
**特征**：
|
||||
- 趋势：随环境扩展而膨胀（可接受）
|
||||
- 内容：按 `## [SECTION]` 组织的事实条目
|
||||
- 同步：变化时更新 L1 的相应 TOPIC 导航行，只能导航
|
||||
**禁止**：禁止存储易变状态、禁止存储猜测、严禁存储大模型可推理的通用常识
|
||||
---
|
||||
### L3：任务级精简记录库 (../memory/)
|
||||
职责：补充 L1/L2 无法容纳、但对**特定任务**未来复用至关重要的少量详细信息。内容必须在满足复用需求的前提下**尽可能短**。
|
||||
原则：
|
||||
- 只记录：跨会话仍重要、且难以通过少量 file_read / web_scan / 简单脚本快速重建的要点。
|
||||
- 优先写：该任务特有的隐藏前置条件、典型易踩坑点，一旦遗忘会导致高成本重试的信息。
|
||||
- 不记录：普通操作步骤、可在几步探测中重新获得的路径或状态信息。
|
||||
形式：
|
||||
- SOP（*_sop.md）：为单一任务或小类任务保留极简的「关键前置 + 典型坑」清单，避免长篇教程。
|
||||
- 工具脚本（*.py）：仅封装高复用、逻辑相对复杂且不希望每次都重新推理的处理流程。
|
||||
---
|
||||
## L1 ↔ L2/L3 同步规则
|
||||
| 操作 | L1 同步 |
|
||||
|---------|--------|
|
||||
| L2/L3 新增场景 | 新建默认低频→L3列表加文件名（自解释不加描述，反直觉场景才能加括号触发词） |
|
||||
| L2/L3 删除场景 | 删除对应层的关键词/映射行 |
|
||||
| L2/L3 修改值 | 若不影响场景定位则不动 L1 |
|
||||
| 发现通用避坑规律 | 压缩为一句加入 RULES |
|
||||
|
||||
> **同步红线**：L1 只写关键词/名称，禁搬细节。括号内只写反直觉的场景触发词(2-4字)，禁写机制/方法/步骤。需要评估L1中的token数和索引效用。
|
||||
> 反例：❌ sop_name(场景A:方法1+方法2+方法3) → ✅ sop_name(场景A)
|
||||
> 反例：名字已自解释时 ❌ discord_slate_sop(Slate输入框) → ✅ discord_slate_sop
|
||||
|
||||
---
|
||||
## 信息分类快速决策树
|
||||
```
|
||||
"这条信息该放哪层？"
|
||||
|
||||
是『环境特异性事实』? (IP、非标路径、凭证、ID、API 密钥等，大模型 Zero-shot 无法生成准确)
|
||||
├─ YES → L2 (global_mem.txt)
|
||||
│ 然后 → 按频率归入 L1 第一层(key→value)或第二层(仅关键词)
|
||||
│
|
||||
└─ NO
|
||||
↓
|
||||
是『通用操作规律』? (全局性避坑指南、排查方法、不针对特定任务的通用准则)
|
||||
├─ YES → L1 [RULES] (仅限 1 句压缩准则)
|
||||
│
|
||||
└─ NO
|
||||
↓
|
||||
是『特定任务技术』? (艰难尝试才能成功，且未来还能用到的任务，如：微信解析参数、特定游戏坐标、临时工具配置)
|
||||
├─ YES → L3 (../memory/ 专项 SOP 或脚本)
|
||||
│
|
||||
└─ NO → 判定为『通用常识』或『冗余信息』: 严禁存储，直接丢弃
|
||||
```
|
||||
@@ -0,0 +1,43 @@
|
||||
# morphling_sop
|
||||
|
||||
## 定义
|
||||
Morphling 是一种项目级能力吸收/替代模式：给定任意目标项目，先抽取其目标与测例，再按组件选择调用、重写或少量复刻禁区规避，最终让自身或新产物在同一测例上达到或超过目标。
|
||||
|
||||
## 核心三元组
|
||||
1. **目标（Target）**：它解决什么问题、面向谁、核心价值是什么；目标可以是完整项目，也可以是巨型项目中的可交付子系统。
|
||||
2. **测例（Tests）**：它声称能通过的 benchmark、demo、CI、榜单、评测站、用户任务清单、性能/质量指标；没有测例先构造最小客观测例。
|
||||
3. **行为（Actions）**：对每个组件分别决定：调用、重写、舍弃；避免“复刻/抄袭”作为默认行为。
|
||||
|
||||
## 输出形态
|
||||
- **调用型 morphling**：把目标能力纳入自身工具链，产物是“更强的我”。
|
||||
- **重写型 morphling**：理解核心后从零实现更好版本，产物是可独立替代原项目的新 repo/工具/产品。
|
||||
- **混合型 morphling**：同一项目可分组件处理：底层复杂依赖调用，差异化核心重写，冗余模块舍弃。
|
||||
|
||||
## 流程
|
||||
1. **锁定目标**：记录 URL/repo/产品名；不要先评价值不值得，先看它实际解决的问题。
|
||||
2. **目标拆解**：识别类型：skill/教程、库、CLI、桌面/网页产品、基础设施、巨型生态、纯概念项目。
|
||||
3. **测例提取**：优先找官方 tests、CI、benchmark、论文/README 指标、demo 脚本、评测网站、issue 中的真实失败案例。
|
||||
4. **测例补全**：若目标没有公开测例，构造最小可验证任务集：核心 happy path、边界条件、目标宣称的杀手特性、用户最痛点。
|
||||
5. **组件分解**：列出核心模块、可替换依赖、生态/数据/模型/硬件等不可轻易重写部分。
|
||||
6. **行为选择**：
|
||||
- 能稳定调用且非差异化核心 → 调用/封装。
|
||||
- 质量差、耦合重、可用更简洁方式实现、或需独立发布 → 重写。
|
||||
- 巨型/长期生态部分 → 缩小到子系统或调用成熟依赖。
|
||||
- 复刻/照抄只作为理解阶段，不作为交付策略。
|
||||
7. **实现闭环**：先做能跑通测例的最小版本，再补强超过目标的维度。
|
||||
8. **对照验证**：在同一测例上跑目标与 morphling 产物，记录通过率、速度、稳定性、成本、易用性。
|
||||
9. **固化成果**：调用型写入工具链/SOP；重写型形成 repo、README、测试与交付说明。
|
||||
|
||||
## 边界判断
|
||||
- Office 这类巨型生态不做整体替代，拆成具体子系统或能力点。
|
||||
- Stable-diffusion-webui 这类“大但核心可抽离”的项目可以重写核心体验，因为历史包袱可能大于真实复杂度。
|
||||
- UI-TARS-Desktop 这类路线差异项目：调用可吸收其纯视觉能力；重写则意味着做一个独立多模态桌面 Agent，并跑同类 GUI benchmark。
|
||||
|
||||
## 执行方式
|
||||
- Morphling 任务应通过 Goal Hive 执行（参见 goal_hive_sop），利用 Master 调度 + Worker 并行实现 + 持续验收的长程模式完成。
|
||||
|
||||
## 完成标准
|
||||
- 必须有测例或明确构造的测例。
|
||||
- 必须说明每个核心组件采用调用/重写/舍弃的理由。
|
||||
- 必须能在同一考卷上与目标对比。
|
||||
- “更好”不能只靠主观判断，至少落在一个可测维度：通过率、性能、成本、稳定性、易用性、可维护性、覆盖范围。
|
||||
@@ -0,0 +1,101 @@
|
||||
"""
|
||||
本地 OCR 工具
|
||||
- OCR引擎: rapidocr-onnxruntime (~1s/次, 中英文准确率高, 带bbox)
|
||||
- 坑(rapid): result[i][2] conf 是 str 不是 float
|
||||
- 坑(rapid): 无文字时 result 返回 None 而非空列表
|
||||
- 坑: enhance 放大+高对比度处理，对清晰文字有害，默认关闭
|
||||
- 坑(远程桌面): ImageGrab/mss 在 RDP 断开后截图全黑，用 ocr_window(hwnd) 代替
|
||||
"""
|
||||
import re
|
||||
from PIL import ImageGrab, Image, ImageEnhance
|
||||
|
||||
_LANG = 'zh-Hans-CN'
|
||||
_rapid_engine = None
|
||||
|
||||
def _get_rapid():
|
||||
global _rapid_engine
|
||||
if _rapid_engine is None:
|
||||
from rapidocr_onnxruntime import RapidOCR
|
||||
_rapid_engine = RapidOCR()
|
||||
return _rapid_engine
|
||||
|
||||
def _preprocess(img, scale=3, contrast=3.0):
|
||||
img = ImageEnhance.Contrast(img).enhance(contrast)
|
||||
img = img.resize((img.width * scale, img.height * scale))
|
||||
return img
|
||||
|
||||
def _strip_cjk_spaces(t):
|
||||
return re.sub(r'(?<=[\u4e00-\u9fff])\s+(?=[\u4e00-\u9fff])', '', t)
|
||||
|
||||
def _ocr_rapid(img):
|
||||
import numpy as np
|
||||
engine = _get_rapid()
|
||||
arr = np.array(img)
|
||||
result, elapse = engine(arr)
|
||||
if not result:
|
||||
return {'text': '', 'lines': [], 'details': []}
|
||||
lines = [r[1] for r in result]
|
||||
details = [{'bbox': r[0], 'text': r[1], 'conf': float(r[2])} for r in result]
|
||||
text = _strip_cjk_spaces('\n'.join(lines))
|
||||
return {'text': text, 'lines': [_strip_cjk_spaces(l) for l in lines], 'details': details}
|
||||
|
||||
def ocr_image(image_input, lang=_LANG, enhance=False, engine=None):
|
||||
"""
|
||||
对 PIL Image 做 OCR
|
||||
:param image_input: PIL Image 对象 或 文件路径(str)
|
||||
:param lang: 保留参数，当前未使用
|
||||
:param enhance: 预处理
|
||||
:param engine: 保留参数，当前仅支持 rapid/None
|
||||
:return: dict {'text': 全文, 'lines': [行文本], 'details': [bbox+conf]}
|
||||
"""
|
||||
if isinstance(image_input, str):
|
||||
image_input = Image.open(image_input)
|
||||
if enhance:
|
||||
image_input = _preprocess(image_input)
|
||||
if engine not in (None, 'rapid'):
|
||||
raise ValueError("Only rapid OCR is supported")
|
||||
return _ocr_rapid(image_input)
|
||||
|
||||
def ocr_screen(bbox=None, lang=_LANG, enhance=False, engine=None):
|
||||
"""
|
||||
截取屏幕区域并 OCR
|
||||
:param bbox: (x1, y1, x2, y2) 像素坐标，None=全屏
|
||||
:return: dict {'text': 全文, 'lines': [行文本], 'details': [bbox+conf](仅rapid)}
|
||||
"""
|
||||
img = ImageGrab.grab(bbox=bbox)
|
||||
return ocr_image(img, lang, enhance, engine)
|
||||
|
||||
def ocr_window(hwnd, lang=_LANG, enhance=False, engine=None):
|
||||
"""
|
||||
截取窗口并 OCR (使用 PrintWindow API，支持远程桌面断开场景)
|
||||
:param hwnd: 窗口句柄(int)
|
||||
:return: dict {'text': 全文, 'lines': [行文本], 'details': [bbox+conf](仅rapid)}
|
||||
"""
|
||||
import win32gui, win32ui
|
||||
from ctypes import windll
|
||||
l, t, r, b = win32gui.GetWindowRect(hwnd)
|
||||
w, h = r - l, b - t
|
||||
hwndDC = win32gui.GetWindowDC(hwnd)
|
||||
mfcDC = win32ui.CreateDCFromHandle(hwndDC)
|
||||
saveDC = mfcDC.CreateCompatibleDC()
|
||||
saveBitMap = win32ui.CreateBitmap()
|
||||
saveBitMap.CreateCompatibleBitmap(mfcDC, w, h)
|
||||
saveDC.SelectObject(saveBitMap)
|
||||
windll.user32.PrintWindow(hwnd, saveDC.GetSafeHdc(), 3)
|
||||
bmpinfo = saveBitMap.GetInfo()
|
||||
bmpstr = saveBitMap.GetBitmapBits(True)
|
||||
img = Image.frombuffer('RGB', (bmpinfo['bmWidth'], bmpinfo['bmHeight']), bmpstr, 'raw', 'BGRX', 0, 1)
|
||||
win32gui.DeleteObject(saveBitMap.GetHandle())
|
||||
saveDC.DeleteDC()
|
||||
mfcDC.DeleteDC()
|
||||
win32gui.ReleaseDC(hwnd, hwndDC)
|
||||
return ocr_image(img, lang, enhance, engine)
|
||||
|
||||
if __name__ == "__main__":
|
||||
r = ocr_screen((0, 0, 400, 100))
|
||||
print(f"识别结果: {r['text']}")
|
||||
for line in r['lines']:
|
||||
print(f" 行: {line}")
|
||||
if 'details' in r:
|
||||
for d in r['details']:
|
||||
print(f" [{d['conf']:.3f}] {d['text']}")
|
||||
@@ -0,0 +1,263 @@
|
||||
# Plan Mode SOP
|
||||
|
||||
**触发**：3步以上有依赖/多文件协同/条件分支/需并行 | **禁用**：1-2步简单任务直接做
|
||||
任务开始前必须先创建工作目录 `./plan_XXX/`（XXX=任务英文短名）
|
||||
单独使用一个code_run({'inline_eval':True, 'script':'handler.enter_plan_mode("./plan_XXX/plan.md")'})进入plan模式
|
||||
handler是inline_eval自动注入的变量
|
||||
|
||||
---
|
||||
|
||||
## 一、探索态（规划前置，必须执行）
|
||||
|
||||
⛔ **硬性规则（先读再做）**：
|
||||
|
||||
- **主agent禁止直接执行环境探测**（必须委托subagent，无例外）
|
||||
- 主agent只做：创建目录、匹配SOP、启动subagent、读取结论
|
||||
- subagent只读探测，禁止修改任何文件、执行有副作用的操作
|
||||
- **探索subagent启动失败时：排查原因→重试，最多2次。禁止主agent回退为自己探测**
|
||||
|
||||
**目标**：在写任何计划之前，搞清3件事：
|
||||
① 环境现状（有什么、缺什么） ② 可用SOP ③ 关键不确定点
|
||||
|
||||
**为什么必须用subagent**：主agent上下文是最稀缺资源，探测长输出会挤占规划执行空间。
|
||||
|
||||
### 步骤1：创建目录（必做） + SOP匹配 + 设置plan标志（主agent直接做）
|
||||
|
||||
1. 创建工作目录 `mkdir plan_XXX/`
|
||||
2. 从上下文中的 L1 Insight 索引匹配可用领域SOP
|
||||
3. 更新checkpoint：`[任务] XXX | [需求] 一句话 | [约束] 关键限制 | [匹配SOP] ... | [进度] 探索态`
|
||||
|
||||
### 步骤2：启动探索subagent（监察模式）
|
||||
|
||||
按 subagent.md 启动探索subagent，**加 `--verbose`** 开启监察模式，input要点：
|
||||
|
||||
- **任务**：探测环境信息，写入 `plan_XXX/exploration_findings.md`
|
||||
- **探测项**（按任务类型选做，不是全做）：
|
||||
- 代码类 → 关键文件结构、依赖、入口点
|
||||
- 浏览器类 → 目标页面当前状态、可交互元素
|
||||
- 自动化类 → 环境检查(which/pip/路径/权限)
|
||||
- 数据类 → 抽样数据(首5行+尾5行+总量)
|
||||
- **输出格式**：`## 环境现状` / `## 关键发现` / `## 风险/不确定点`
|
||||
- **约束**：只读探测，禁止修改文件，≤10次工具调用
|
||||
- **复杂度评估**：探测时注意记录数据规模（文件数、行数、页面数），写入findings供规划时判断委托
|
||||
|
||||
### 步骤3：监察等待 + 读取结论
|
||||
|
||||
主agent主动观察output.txt进度（`--verbose`输出含原始工具结果），而非无脑sleep轮询：
|
||||
|
||||
1. **观察**：读output.txt，审查subagent的探测方向和原始数据
|
||||
2. **纠偏**（按需）：
|
||||
- 方向偏了 → 写 `_intervene` 追加指令纠正
|
||||
- 缺少关键上下文 → 写 `_keyinfo` 注入信息
|
||||
- 已获取足够信息 → 写 `_stop` 提前终止，节省轮次
|
||||
3. **收取**：等待 `[ROUND END]`，读取 `exploration_findings.md`
|
||||
|
||||
**产出**：`exploration_findings.md`（结构化发现报告），主agent基于此进入规划态，写入plan.md头部的「探索发现」段。主agent在监察过程中获得的一手认知也可直接用于规划。
|
||||
|
||||
---
|
||||
|
||||
## 二、规划态（含审查门）
|
||||
|
||||
### 步骤4：读领域SOP → 写plan.md
|
||||
|
||||
先读探索态匹配到的SOP，然后写plan骨架。允许"⚠待确认"，禁止以"没调研清楚"推迟。
|
||||
|
||||
**[D] 委托标注规则**：写每个步骤时，结合探索发现评估操作量，符合以下任一条件则标 `[D]`：
|
||||
|
||||
- 需要读取大量代码/文件（预估 >3个文件或 >100行）
|
||||
- 需要浏览网页并提取信息
|
||||
- 需要执行 3 次以上重复性操作
|
||||
- 需要运行测试/构建并分析输出
|
||||
|
||||
不标 `[D]` 的情况：读/更新 plan.md、单文件小幅修改、ask_user、简单一次性命令
|
||||
|
||||
**plan.md格式**：
|
||||
|
||||
```markdown
|
||||
<!-- EXECUTION PROTOCOL (每轮必读，这是你的执行指南)
|
||||
1. file_read(plan.md)，找到第一个 [ ] 项
|
||||
2. 该步标注了SOP → file_read 该SOP的🔑速查段
|
||||
3. 执行该步骤 + Mini验证产出
|
||||
4. file_patch 标记 [ ] → [✓]+简要结果，然后回到步骤1继续下一个[ ]
|
||||
5. 所有步骤（包括验证步骤）标记完成后 → 终止检查：file_read(plan.md)确认0个[ ]残留
|
||||
⚠ 禁止凭记忆执行 | 禁止跳过验证步骤 | 禁止未经终止检查就结束 | 禁止停下来输出纯文字汇报
|
||||
💡 搬砖活（读大量代码/文件/网页/重复操作）优先委托subagent，保持主agent上下文干净
|
||||
-->
|
||||
# 任务标题
|
||||
需求：一句话 | 约束：关键限制
|
||||
|
||||
## 探索发现
|
||||
- 发现1：XXX（来源：file_read/web_scan/code_run）
|
||||
- 发现2：YYY
|
||||
- 不确定点：ZZZ
|
||||
|
||||
## 执行计划
|
||||
1. [ ] 步骤1简述
|
||||
SOP: xxx_sop.md
|
||||
2. [D] 步骤2简述（委托subagent执行）
|
||||
SOP: yyy_sop.md
|
||||
依赖：1
|
||||
3. [P] 步骤3简述（并行，读subagent.md执行Map模式）
|
||||
SOP: yyy_sop.md
|
||||
4. [?] 步骤4（条件分支）
|
||||
SOP: (无) ← 高风险
|
||||
条件：X成功→4.1，否则→4.2
|
||||
|
||||
---
|
||||
|
||||
## 验证检查点
|
||||
N+1. [ ] **[VERIFY] 启动独立验证subagent**
|
||||
SOP: deliverable_audit_sop.md plan_sop.md
|
||||
操作：读plan_sop.md第四章内容 → 准备verify_context.json → 启动验证subagent → 读取VERDICT → 按结果处理
|
||||
⚠ 不可跳过，不可在未启动subagent的情况下标记[✓]
|
||||
|
||||
---
|
||||
```
|
||||
|
||||
### 步骤5：自检清单（主agent逐项检查）
|
||||
|
||||
- □ 探索发现是否都反映在plan中？（没遗漏关键约束）
|
||||
- □ 每步的SOP标注是否合理？（SOP真的能解决该步？）
|
||||
- □ 步骤间依赖是否正确？（有没有隐含依赖没写出来）
|
||||
- □ 高风险步骤（SOP:无/不可逆）有没有清晰的执行思路？
|
||||
- □ 步骤粒度是否合适？（禁止"处理所有文件"，必须展开具体条目）
|
||||
- □ **复杂/繁琐步骤是否标注了[D]？**（读大量代码/网页/重复操作必须委托subagent）
|
||||
- □ **是否包含"验证检查点"section，且有[VERIFY]步骤？（必须有，这是强制步骤）**
|
||||
|
||||
### 步骤6：用户确认
|
||||
|
||||
ask_user 确认plan后才能转入执行态。**⛔ 用户未确认不得执行。**
|
||||
|
||||
### 步骤7：转入执行态
|
||||
|
||||
更新checkpoint：`[执行] plan.md | 当前：步骤1 | ⚡有[P]标记必须读subagent.md执行Map模式`
|
||||
|
||||
---
|
||||
|
||||
## 三、执行态循环
|
||||
|
||||
> **核心原则：连续执行，不停顿汇报。** 做完一步立即 file_read(plan.md) 找下一个 `[ ]`，直到全部完成。
|
||||
|
||||
### 每轮流程
|
||||
|
||||
1. **读plan** — `file_read(plan.md)` 定位第一个 `[ ]` 项
|
||||
2. **读SOP** — 该步标注了SOP → 先 file_read 该SOP
|
||||
3. **检查标记** — `[D]`标记 → 必须委托subagent执行，主agent只收结果摘要；`[P]`标记 → 读 subagent.md 执行Map模式；`[?]`条件 → 评估条件选分支，未选标[SKIP]
|
||||
4. **执行** — 无特殊标记的步骤由主agent自己执行
|
||||
5. **Mini验证** — 快速确认产出存在且合理（file_read确认非空、检查exit code等）
|
||||
6. **标记完成** — `file_patch` 标记 `[ ]` → `[✓ 简要结果]`（进度写入plan.md）
|
||||
7. **继续** — 立即回到步骤1，file_read(plan.md) 执行下一个 `[ ]`
|
||||
|
||||
### 终止检查（最后一步标记后，不可跳过）
|
||||
|
||||
file_read(plan.md) 全文扫描，确认所有步骤（含[VERIFY]）均为 `[✓]`/`[✗]`，0个 `[ ]` 残留。
|
||||
输出：`🏁 终止检查：[总步数]步全部完成，0个[ ]残留 → 任务结束`
|
||||
若发现遗漏 → 继续执行，禁止声称完成。
|
||||
|
||||
### ⚠ 执行态禁令
|
||||
|
||||
- **禁止凭记忆执行**：每次做新步骤前必须 `file_read(plan.md)`，不可"我记得下一步是..."
|
||||
- **禁止跳过验证步骤**：[VERIFY]步骤是强制的，不可以"任务都做完了"为由跳过
|
||||
- **禁止未经终止检查就结束**：最后一步标记后必须 file_read 全文扫描确认0个[ ]残留，输出🏁终止确认行
|
||||
- **禁止停下来输出纯文字汇报**：做完一步后必须立即 file_read(plan.md) 继续，不要输出进度总结
|
||||
|
||||
### 💡 动态委托原则
|
||||
|
||||
即使步骤未标 `[D]`，执行中发现以下情况时，主动委托 subagent 处理：
|
||||
|
||||
- 需要读取大量代码/文件才能理解上下文（>3个文件或预估 >100行）
|
||||
- 需要反复试错调试
|
||||
- 需要浏览网页提取信息
|
||||
|
||||
做法：起 subagent 完成具体操作，要求返回精简摘要，主 agent 基于摘要继续决策。保持主 agent 上下文干净是第一优先级。
|
||||
|
||||
---
|
||||
|
||||
## 四、验证态（subagent独立验证）
|
||||
|
||||
> 全部步骤[✓]后进入。**强制**启动独立subagent做对抗性验证，避免上下文污染。
|
||||
|
||||
### 触发条件
|
||||
|
||||
- 所有执行步骤标记为 `[✓]`
|
||||
- **所有plan模式任务必须经subagent验证**（主agent有确认偏误，易被表面成功迷惑）
|
||||
|
||||
### 步骤8：准备验证上下文
|
||||
|
||||
在 `./plan_XXX/` 下创建 `verify_context.json`，包含：
|
||||
|
||||
- task_description：原始任务描述（用户原话）
|
||||
- plan_file：plan.md绝对路径
|
||||
- task_type：code|data|browser|file|system
|
||||
- deliverables：交付物列表（type/path/expected）
|
||||
- required_checks：必做检查列表（check/tool）
|
||||
|
||||
**传什么**：任务描述、plan路径、交付物清单、必做检查。**不传**：执行过程、调试记录。
|
||||
|
||||
### 步骤9：启动验证subagent
|
||||
|
||||
按 subagent.md 标准流程启动验证subagent，input要点：
|
||||
|
||||
- **角色**：你是独立验证者，工作是对抗性验证（证明交付物不能用）
|
||||
- **第一步强制**：file_read deliverable_audit_sop.md 完整阅读验证SOP
|
||||
- **按 deliverable_audit_sop.md 第3节**选择对应task_type的验证策略执行
|
||||
- **每个检查必须有工具调用证据**（实际执行，不是叙述）
|
||||
- **任务描述**：（填入原始任务描述）
|
||||
- **交付物清单**：（填入deliverables列表）
|
||||
- **输出**：在 result.md 中按 deliverable_audit_sop.md 第6节格式输出，最后一行 `VERDICT: PASS / FAIL / PARTIAL`
|
||||
- **约束**：3轮内完成，每轮至少1个实际工具调用
|
||||
|
||||
同时传入 verify_context.json 的路径，让subagent自行读取详细上下文。
|
||||
|
||||
### 步骤10：收集验证结果
|
||||
|
||||
轮询 output.txt 等待 `[ROUND END]`，然后读取 result.md：
|
||||
|
||||
1. **找VERDICT行**：读取result.md最后几行，提取 `VERDICT: PASS/FAIL/PARTIAL`
|
||||
2. **检查有效性**：如果所有PASS项都没有工具调用输出（只有叙述），视为验证无效，按FAIL处理
|
||||
3. **按结果处理**：
|
||||
- **PASS** → 进入任务完成收尾
|
||||
- **FAIL** → 进入修复循环
|
||||
- **PARTIAL** → 主agent判断可接受则完成，否则修复
|
||||
- **无VERDICT行** → 从output.txt提取关键信息，主agent自行判断PASS/FAIL
|
||||
|
||||
**任务完成收尾**（验证PASS后执行）：
|
||||
|
||||
1. 标记plan.md中 `[VERIFY]` 步骤为 `[✓]`
|
||||
2. 更新checkpoint：`[完成] XXX任务 | [产出] ... | [经验] ...`
|
||||
3. 向用户确认任务完成
|
||||
|
||||
**重要**：只有在验证PASS后，才能标记[VERIFY]为[✓]并声称任务完成。如果验证FAIL，需要进入修复循环。
|
||||
|
||||
**Fallback**：若subagent未产出result.md（turn耗尽），从output.txt提取VERDICT关键信息。
|
||||
|
||||
### 修复循环（FAIL后）
|
||||
|
||||
FAIL → 提取具体失败项 → 回执行态修复（不重新规划） → 修复完成 → 再次启动验证subagent → 最多2轮FAIL-重试，超过 ask_user 介入
|
||||
|
||||
修复时：
|
||||
|
||||
1. 将FAIL项作为新步骤追加到plan.md（标记为 `[FIX]`）
|
||||
2. 只修复失败项，不重做已PASS的部分
|
||||
3. 修复完成后重新准备verify_context.json（只含失败项）
|
||||
|
||||
### 特殊场景处理
|
||||
|
||||
浏览器/键鼠/定时任务等场景：主agent执行操作并导出证据（截图/录屏/日志）→ subagent验证证据文件。**禁止主agent自行判断PASS/FAIL**。
|
||||
|
||||
---
|
||||
|
||||
## 五、失败处理
|
||||
|
||||
1. **记录**：checkpoint中 `step_X: [FAILED] 原因 (retry: N/3)`
|
||||
2. **重试**：网络超时→自动重试3次(2s/4s/8s) | 配置错误→询问用户 | 其他→标[✗]跳过
|
||||
3. **subagent失败**：查stderr.log→明确错误主agent修正重启 | 未知错误重试1次 | 最多重启2次
|
||||
4. **依赖传播**：步骤失败后，后续依赖项标[SKIP]
|
||||
5. **plan有误**：回退到规划态修正plan.md，重新过审查门
|
||||
|
||||
## 强制约束
|
||||
|
||||
- 每项必须有独立完成判据
|
||||
- 禁止"处理所有文件"，必须展开具体条目
|
||||
- 一次只做一项；计划有误回规划态修正
|
||||
- 不可逆操作前多验证一步
|
||||
@@ -0,0 +1,144 @@
|
||||
import ctypes
|
||||
import ctypes.wintypes
|
||||
import argparse
|
||||
import yara
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
|
||||
# Define WinAPI Types for 64-bit compatibility
|
||||
PHANDLE = ctypes.wintypes.HANDLE
|
||||
LPCVOID = ctypes.c_void_p
|
||||
LPVOID = ctypes.c_void_p
|
||||
SIZE_T = ctypes.c_size_t
|
||||
|
||||
class MEMORY_BASIC_INFORMATION(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("BaseAddress", LPVOID),
|
||||
("AllocationBase", LPVOID),
|
||||
("AllocationProtect", ctypes.wintypes.DWORD),
|
||||
("RegionSize", SIZE_T),
|
||||
("State", ctypes.wintypes.DWORD),
|
||||
("Protect", ctypes.wintypes.DWORD),
|
||||
("Type", ctypes.wintypes.DWORD),
|
||||
]
|
||||
|
||||
# Explicitly setup kernel32 functions with precise types
|
||||
k32 = ctypes.windll.kernel32
|
||||
k32.OpenProcess.argtypes = [ctypes.wintypes.DWORD, ctypes.wintypes.BOOL, ctypes.wintypes.DWORD]
|
||||
k32.OpenProcess.restype = PHANDLE
|
||||
|
||||
k32.VirtualQueryEx.argtypes = [PHANDLE, LPCVOID, ctypes.POINTER(MEMORY_BASIC_INFORMATION), SIZE_T]
|
||||
k32.VirtualQueryEx.restype = SIZE_T
|
||||
|
||||
k32.ReadProcessMemory.argtypes = [PHANDLE, LPCVOID, LPVOID, SIZE_T, ctypes.POINTER(SIZE_T)]
|
||||
k32.ReadProcessMemory.restype = ctypes.wintypes.BOOL
|
||||
|
||||
import re
|
||||
|
||||
# Regex to expand YARA (n) jumps to explicit ?? chains (YARA 4.5.4 bug)
|
||||
_RE_JUMP = re.compile(r'\(\s*(\d+)\s*\)')
|
||||
|
||||
def expand_yara_jumps(hex_pattern):
|
||||
"""Expand (n) → ?? repeated n times, e.g. '90 ( 32 ) 00' → '90 ?? ??...?? 00'"""
|
||||
def _repl(m):
|
||||
return ' '.join(['??'] * int(m.group(1)))
|
||||
return _RE_JUMP.sub(_repl, hex_pattern)
|
||||
|
||||
def is_hex_pattern(pattern):
|
||||
"""Detect hex patterns like '90 ( 32 ) 00' or '90 ?? 00'"""
|
||||
clean = pattern.replace(" ", "").replace("??", "")
|
||||
# Also remove parenthesized jump counts like (32)
|
||||
clean = _RE_JUMP.sub('', clean)
|
||||
return all(c in "0123456789abcdefABCDEF" for c in clean) and len(clean) % 2 == 0
|
||||
|
||||
def build_rules(pattern, mode=None):
|
||||
if hasattr(pattern, 'match'): return pattern
|
||||
mode = mode or ('auto' if isinstance(pattern, str) else 'yara')
|
||||
if mode in ('yara',):
|
||||
try:
|
||||
return yara.compile(source=str(pattern))
|
||||
except yara.SyntaxError:
|
||||
raise # user-provided full YARA rule, don't mess with it
|
||||
# hex mode or auto
|
||||
use_hex = (mode == 'hex') or (mode == 'auto' and is_hex_pattern(pattern))
|
||||
if use_hex:
|
||||
hex_body = expand_yara_jumps(pattern.strip())
|
||||
rule_text = f'rule CustomSearch {{ strings: $h = {{ {hex_body} }} condition: $h }}'
|
||||
else:
|
||||
escaped = pattern.replace('\\', '\\\\').replace('"', '\\"')
|
||||
rule_text = f'rule CustomSearch {{ strings: $s = "{escaped}" ascii wide condition: $s }}'
|
||||
return yara.compile(source=rule_text)
|
||||
|
||||
def format_llm_context(data, offset, base_addr, length=64):
|
||||
start = max(0, offset - length)
|
||||
end = min(len(data), offset + length + 16)
|
||||
chunk = data[start:end]
|
||||
abs_addr = (base_addr if base_addr else 0) + offset
|
||||
return {
|
||||
"address": hex(abs_addr),
|
||||
"offset": hex(offset),
|
||||
"hex": chunk.hex(),
|
||||
"ascii": "".join(chr(b) if 32 <= b <= 126 else "." for b in chunk),
|
||||
"hit_pos": offset - start
|
||||
}
|
||||
|
||||
def scan_memory(pid, pattern, context_size=256, mode=None, llm_mode=False):
|
||||
rules = build_rules(pattern, mode)
|
||||
h_proc = k32.OpenProcess(0x0400 | 0x0010, False, pid)
|
||||
if not h_proc:
|
||||
# OpenProcess failed: might be system process or higher integrity level
|
||||
return [f"Error: Cannot open process {pid}. (ErrorCode: {k32.GetLastError()})"]
|
||||
|
||||
results = []
|
||||
curr_addr = 0
|
||||
mbi = MEMORY_BASIC_INFORMATION()
|
||||
|
||||
# Range for 64-bit user space
|
||||
max_addr = 0x7FFFFFFFFFFF
|
||||
|
||||
while curr_addr < max_addr:
|
||||
# Use cast to ensure pointer type is correct for 64-bit
|
||||
res = k32.VirtualQueryEx(h_proc, ctypes.cast(curr_addr, LPCVOID), ctypes.byref(mbi), ctypes.sizeof(mbi))
|
||||
if res == 0: break
|
||||
|
||||
# MEM_COMMIT = 0x1000, PAGE_READABLE bitmask
|
||||
if mbi.State == 0x1000 and (mbi.Protect & 0xEE): # 0xEE covers common readable flags
|
||||
buf = ctypes.create_string_buffer(mbi.RegionSize)
|
||||
read = SIZE_T(0)
|
||||
if k32.ReadProcessMemory(h_proc, ctypes.cast(mbi.BaseAddress, LPCVOID), buf, mbi.RegionSize, ctypes.byref(read)):
|
||||
data = buf.raw[:read.value]
|
||||
for match in rules.match(data=data):
|
||||
for inst in match.strings:
|
||||
base = mbi.BaseAddress if mbi.BaseAddress else 0
|
||||
for instance in inst.instances: # ITERATE ALL instances, not just [0]
|
||||
offset = instance.offset
|
||||
matched_data = instance.matched_data
|
||||
if llm_mode:
|
||||
results.append(format_llm_context(data, offset, base, length=context_size))
|
||||
else:
|
||||
# Expand context based on context_size to capture full KEY+SALT
|
||||
start = max(0, offset - context_size)
|
||||
end = min(len(data), offset + len(matched_data) + context_size)
|
||||
results.append(f"Addr: {hex(base+offset)}\nHex: {data[start:end].hex()}")
|
||||
|
||||
# Update address using the region size
|
||||
next_addr = (mbi.BaseAddress if mbi.BaseAddress else 0) + mbi.RegionSize
|
||||
if next_addr <= curr_addr: break
|
||||
curr_addr = next_addr
|
||||
|
||||
k32.CloseHandle(h_proc)
|
||||
return results
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("pid", type=int)
|
||||
parser.add_argument("pattern", type=str)
|
||||
parser.add_argument("--mode", default='auto')
|
||||
parser.add_argument("--llm", action="store_true")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
res = scan_memory(args.pid, args.pattern, mode=args.mode, llm_mode=args.llm)
|
||||
print(json.dumps(res, indent=2) if args.llm else f"Matches: {len(res)}")
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
@@ -0,0 +1,36 @@
|
||||
# Memory Scanner SOP
|
||||
|
||||
## 1. 快速开始
|
||||
内存特征搜索工具，支持 Hex (CE 风格) 和 字符串匹配。特别提供 LLM 模式，方便大模型分析内存上下文。
|
||||
|
||||
**Python 调用方式:**
|
||||
```python
|
||||
import sys
|
||||
sys.path.append('../memory') # 直接挂载工具目录
|
||||
from procmem_scanner import scan_memory
|
||||
|
||||
# 示例：搜索特定 Hex 特征码，开启 llm_mode 以获取上下文
|
||||
results = scan_memory(pid, "48 8b ?? ?? 00", mode="hex", llm_mode=True)
|
||||
```
|
||||
|
||||
**CLI:**
|
||||
```powershell
|
||||
# 基础搜索
|
||||
python ../memory/procmem_scanner.py <PID> "pattern" --mode string
|
||||
|
||||
# LLM 增强模式（输出包含上下文的 JSON，推荐）
|
||||
python ../memory/procmem_scanner.py <PID> "pattern" --llm
|
||||
```
|
||||
|
||||
## 2. 典型场景：结构体或关键数据定位
|
||||
1. 确定目标数据的前导特征或已知常量（如特定的 Header 或 Magic Number）。
|
||||
2. 在目标进程中搜索该特征：
|
||||
`scan_memory(pid, "4D 5A 90 00", mode="hex", llm_mode=True)`
|
||||
3. 分析返回的 JSON 中 `context` 字段，查看目标地址前后的原始字节及 ASCII 预览。
|
||||
|
||||
## 3. 注意事项
|
||||
- **权限**: 并非强制要求管理员权限，但需具备对目标进程的 `PROCESS_QUERY_INFORMATION` 和 `PROCESS_VM_READ` 权限。
|
||||
- **效率**: 搜索大块内存时，尽量提供更唯一的特征码以减少误报。
|
||||
|
||||
## 4. CE式差集扫描定位动态字段
|
||||
定位微信等自绘UI中随操作变化的内存字段（如当前会话标题）。核心：一次全量scan + 多次ReadProcessMemory筛选。
|
||||
@@ -0,0 +1,29 @@
|
||||
# Project Mode SOP
|
||||
|
||||
## 定义
|
||||
|
||||
Project Mode = 跨会话保持项目认知的工作模式
|
||||
载体：`project_memory.md` + 项目私域目录。两层注入：每轮自动注入的只有 L1（规则+记忆文件指针），L2（`project_memory.md` 全文）不注入——任务涉及项目上下文时自己用 file 工具去读，无关则不读
|
||||
|
||||
## 进入
|
||||
|
||||
激活态保存在当前 Agent 实例；各会话互不干扰，关闭 GA 自动失效。（下文路径以 cwd 为基准，禁写 `temp/xxx` 前缀）
|
||||
|
||||
- 用户只说「进入项目模式」未指明项目：列出 `./projects/` 下各项目（名字 + memory 行数 + 最后修改时间），ask_user 让用户选定后再继续
|
||||
- 用户明确说「进入/切换到 <项目名> 项目」：视为已确认，直接执行：
|
||||
|
||||
1. 建目录 `./projects/<项目名>/`，无则创建 `project_memory.md`（空文件即可）
|
||||
2. 用 `code_run` 的 `inline_eval=true` 绑定当前 Agent（`handler` 已注入，无需 import）：
|
||||
`handler.enter_project_mode('<项目名>')`
|
||||
3. 回读 `project_memory.md` 全文，向用户复述项目现状
|
||||
|
||||
## 期间纪律
|
||||
|
||||
- 项目文件（todo、草稿、产物）一律放 `./projects/<项目名>/`，禁止丢 temp 根目录
|
||||
- 入库判据（唯一标准）：每得到一条信息，自问「记忆归零、重新接手本项目的我，缺了这条会不会重复付出认知代价——再踩一次坑、再摸索一次、再问一次用户？」会则立即追加进 `project_memory.md`，不会则不记
|
||||
- 一条一句，写成未来的自己能直接复用的形式；已有条目增量更新，不整篇重写
|
||||
|
||||
## 离开
|
||||
|
||||
明确要求离开时，用 `code_run`（`inline_eval=true`）执行 `handler.enter_project_mode(None)`。
|
||||
切换项目直接换项目名
|
||||
@@ -0,0 +1,169 @@
|
||||
# Review Mode SOP
|
||||
|
||||
> In-session adversarial code reviewer。用 `/review` 触发,主 agent 在当前对话内
|
||||
> 拉起评审,报告直接 echo 到对话,**不开 subagent / 不落盘 / 不打 sentinel**。
|
||||
|
||||
---
|
||||
|
||||
## 一、何时使用
|
||||
|
||||
用户输入 `/review` 命令,或自然语言要求"code review"时启用。
|
||||
典型用例:作者刚写完一段代码 → `/review` 对自己的改动做对抗性 review。
|
||||
|
||||
---
|
||||
|
||||
## 二、快速启动
|
||||
|
||||
| 命令 | 行为 |
|
||||
|---|---|
|
||||
| `/review` | 默认审本次 uncommitted 改动(主 agent 跑 `git diff --stat HEAD` + `git diff HEAD`) |
|
||||
| `/review <自然语言请求>` | 按描述的范围去审(可指定文件 / 目录 / 任务) |
|
||||
| `/review help` | 显示用法 |
|
||||
|
||||
**非 git 仓库**:主 agent 提示用户在下一句 `/review` 塞入具体路径或范围,本轮结束。
|
||||
|
||||
---
|
||||
|
||||
## 三、入口文件
|
||||
|
||||
```
|
||||
任意前端 (TUI / Streamlit / wechat / desktop)
|
||||
└─ frontends/review_cmd.py ← 命令分发,剥 "/review" 前缀,注入 user_request
|
||||
└─ memory/review_sop/review_inline_prompt.txt ← 完整 in-session 协议
|
||||
└─ memory/code_review_principles.md ← 15 条好代码原则
|
||||
```
|
||||
|
||||
- `review_cmd.py:install()` —— monkey-patch `GenericAgent._handle_slash_cmd`,统一接管 `/review`
|
||||
- `review_cmd.py:_render_prompt()` —— 加载 prompt 模板,注入 `{user_request}` + `{ga_root}`
|
||||
|
||||
---
|
||||
|
||||
## 四、三条铁律(reviewer 顶部硬约束,不可违反)
|
||||
|
||||
1. **Review-only 只读评审** —— 评审与报告而已。**禁止**修改源文件、调
|
||||
file_write / file_patch / code_run 改业务代码、在产出里写"我接下来去修一下"
|
||||
或暗示要动手。
|
||||
2. **Challenge the approach, 不仅找 bug** —— 先问"这条路本身对不对?"再问
|
||||
"实现有没有 bug?":挖隐含假设、评估真实环境故障模式(Windows 路径 / 代理失活 /
|
||||
并发写 / UTF-8 边界 / token 预算耗尽)。
|
||||
3. **报告输出完即结束** —— 不复述用户目标、不做 meta 评论、不承诺 follow-up;
|
||||
报告 markdown 直接 echo 到对话,**不落盘 review.md、不打 `[ROUND END]`**。
|
||||
|
||||
---
|
||||
|
||||
## 五、工作流(5 步,顺序走)
|
||||
|
||||
### 步骤 1:必读底料
|
||||
|
||||
`file_read("memory/code_review_principles.md")` —— 15 条好代码原则,**每条 finding 必须
|
||||
能映射到其中一条**。
|
||||
|
||||
### 步骤 2:锁定审阅范围
|
||||
|
||||
| 用户输入 | 范围 |
|
||||
|---|---|
|
||||
| 点名了文件 / 目录 | 审那些 |
|
||||
| 描述了任务范围 | `code_run` 跑 `git status -s` + `git diff --stat HEAD` + `git diff HEAD` |
|
||||
| 空 / 模糊 | 默认审本次 uncommitted 改动 |
|
||||
| 非 git 仓库 | 提示用户塞路径,本轮结束 |
|
||||
|
||||
**先把范围列出来发给用户确认**,再开始 `file_read`。
|
||||
|
||||
### 步骤 3:逐文件 file_read
|
||||
|
||||
超过 800 行分段读。优先看 diff 涉及的行,再看上下文与接口调用方。
|
||||
|
||||
### 步骤 4:回答 Q1-Q4 对抗性 framing
|
||||
|
||||
- **Q1: Is this the right approach?** — 有没有更简单 / 更标准 / 更安全的实现路径?
|
||||
- **Q2: What hidden dependencies could fail?** — OS / shell / 网络 / 并发 / 第三方 API 任一失效?
|
||||
- **Q3: What edge / hostile input breaks it?** — 空值、UTF-8、Windows 路径、超长输入、过期 token。
|
||||
- **Q4: Is the failure mode observable & recoverable?** — 仅看日志能不能定位?能不能不动手就恢复?
|
||||
|
||||
### 步骤 5:列 P0~P3 findings
|
||||
|
||||
遵守 §七 防误报八规则 + §八 措辞八规范。提交前过自检清单(§九)。
|
||||
|
||||
---
|
||||
|
||||
## 六、Severity / Verdict 速查
|
||||
|
||||
| Level | 定义 | 例子 |
|
||||
|---|---|---|
|
||||
| **P0** | 阻塞:破坏正确性 / 丢数据 / 安全漏洞 / 不可逆故障 | 路径穿越、SQL 注入、密钥落日志、并发竞态破坏数据 |
|
||||
| **P1** | 高危:契约破坏 / 用户可见错误,但不会立即崩 | 错误只 print 不抛、超时未设、API schema 不一致 |
|
||||
| **P2** | 维护性:可读性 / 命名 / 测试空缺 | 函数 > 80 行、duplicate logic、注释与代码不符 |
|
||||
| **P3** | 风格 / 微优化 / 可选改进 | 命名小调整、常量提取、import 顺序 |
|
||||
|
||||
**Verdict 决议**:任一 P0 → `FAIL`;无 P0 但 ≥ 1 P1 → `CONDITIONAL`;仅 P2/P3 或 0 finding → `PASS`。
|
||||
|
||||
---
|
||||
|
||||
## 七、防误报八规则(成本低到高,任一答 No → 删 finding)
|
||||
|
||||
1. **Discrete & actionable** — 有具体可写的修复?
|
||||
2. **Introduced or exposed by this change** — 本次改动引入或放大?
|
||||
3. **Not an intentional design choice** — 不是作者刻意取舍?
|
||||
4. **Provably affected, not speculated** — 跨文件影响能指出调用栈?
|
||||
5. **Evidence-anchored** — 行号 / 代码片段 / 复现至少一项?
|
||||
6. **No unstated assumptions** — 不依赖未明说的"应该这样"?
|
||||
7. **Author would likely fix if made aware** — 作者会同意修?
|
||||
8. **Impact meaningful + proportionate rigor** — 影响足够 + 严谨度匹配代码库?
|
||||
|
||||
> 每条规则的展开详见 `memory/review_sop/review_inline_prompt.txt` §5。
|
||||
|
||||
---
|
||||
|
||||
## 八、措辞八规范
|
||||
|
||||
1. **Why-first** — 第一句给原因。
|
||||
2. **严重度准确** — 不要把 P2 写得像 P0。
|
||||
3. **简洁** — `evidence` / `impact` / `fix` 各 ≤ 1 段。
|
||||
4. **少贴大段代码** — `evidence` 代码 ≤ 5 行,超过用 `file:line-line` 引用。
|
||||
5. **触发条件显式** — `impact` 首句必带场景 / 输入 / 环境。
|
||||
6. **不卑不亢** — 直陈事实,无情绪 / 无开场白。
|
||||
7. **即读即懂** — 核心结论放第一句。
|
||||
8. **零奉承** — 不写 "Great work, but...", "Thanks for the changes, however..."。
|
||||
|
||||
> 展开详见 `memory/review_sop/review_inline_prompt.txt` §6。
|
||||
|
||||
---
|
||||
|
||||
## 九、输出协议(整段 echo,不落盘)
|
||||
|
||||
```
|
||||
## Scope
|
||||
<一行一个文件,绝对路径或仓库相对路径>
|
||||
|
||||
## Verdict
|
||||
PASS / CONDITIONAL / FAIL
|
||||
|
||||
## Summary
|
||||
3-6 行散文:整体印象 + 最重要的 1-2 个风险。
|
||||
|
||||
## Design Challenge (Q1-Q4)
|
||||
- **Q1 是不是对的方法**: <证据>
|
||||
- **Q2 隐藏依赖**: <证据>
|
||||
- **Q3 边缘 / 敌意输入**: <证据>
|
||||
- **Q4 故障可观测**: <证据>
|
||||
|
||||
## Findings (P0 → P3 顺序)
|
||||
- **[P0, conf=0.9] file:line-line** 标题(动词开头,≤ 80 字,第一句给原因)
|
||||
- **Evidence**: 代码片段 ≤ 5 行 或 file:N-M 引用
|
||||
- **Impact**: 触发场景 + 后果(第一句必带场景)
|
||||
- **Fix**: 可直接照做的修复思路,≤ 1 段
|
||||
- **Principle**: 对应 code_review_principles 第 N 条
|
||||
|
||||
## Cross-file notes
|
||||
跨文件耦合 / 命名一致性 / 状态机 / 并发问题。无则 `(none)`。
|
||||
|
||||
## Regression tests
|
||||
3-5 条具体测试点(输入 / 预期 / 边界)。
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 十、扩展点
|
||||
|
||||
- **自定义评审条目**:编辑 `memory/code_review_principles.md`,reviewer 启动时整段注入
|
||||
- **触发更换**:要把 `/review` 改成别的命令,只动 `frontends/review_cmd.py` 的 `install()` 一处
|
||||
@@ -0,0 +1,27 @@
|
||||
# 定时任务 SOP
|
||||
|
||||
目录：`../sche_tasks/` 放任务定义JSON，`../sche_tasks/done/` 放执行报告
|
||||
|
||||
## 任务JSON格式（*.json）
|
||||
```json
|
||||
{"schedule":"08:00", "repeat":"daily", "enabled":true, "prompt":"...", "max_delay_hours":6}
|
||||
```
|
||||
repeat可选：daily | weekday | weekly | monthly | once | every_Nh（每N小时）| every_Nd（每N天）
|
||||
max_delay_hours（可选，默认6）：超过schedule多少小时后不再触发，防止开机太晚执行过时任务
|
||||
|
||||
## 触发流程
|
||||
1. scheduler.py（reflect/）每60秒轮询 sche_tasks/*.json
|
||||
2. 条件全满足才触发：enabled=true + 当前时间≥schedule + 冷却时间已过（基于done/最新报告时间戳）
|
||||
3. 触发时拼prompt，含报告路径 `../sche_tasks/done/YYYY-MM-DD_任务名.md`
|
||||
4. **收到任务后第一件事**：用 update_working_checkpoint 记录报告目标文件路径，防止长任务执行中遗忘
|
||||
5. 执行完毕后将报告写入上述路径（scheduler靠此文件判断今天已执行）
|
||||
|
||||
## 日志与监控
|
||||
- scheduler自动写日志到 `sche_tasks/scheduler.log`（触发/跳过/错误）
|
||||
- `scheduler.health_check()` 返回所有任务状态列表（HEALTHY/OVERDUE/DISABLED/NEVER_RUN/ERROR）
|
||||
- JSON解析错误、schedule格式错误、未知repeat类型均会记录日志
|
||||
|
||||
## 注意
|
||||
- once类型：执行一次后冷却100年（实际效果为永久跳过）
|
||||
- 任务文件只管"干什么"，报告路径由scheduler自动生成注入prompt
|
||||
- sche_tasks目录在../，即code root下
|
||||
@@ -0,0 +1,61 @@
|
||||
# Subagent 调用 SOP
|
||||
|
||||
## 两种模式
|
||||
|
||||
### --func 纯函数模式
|
||||
- `python agentmain.py --func prompt.txt [--llm_no N]`（cwd=代码根）
|
||||
- 读prompt文件→执行→结果写`prompt.out.txt`→退出，主agent读完可删
|
||||
- 后台启动(print PID)，加`--nobg`前台同步等结果
|
||||
- 适用：单次任务、并行map、不需要追问的场景
|
||||
|
||||
### --task 持续协作模式
|
||||
- `python agentmain.py --task {name} [--input "短文本"] [--llm_no N]`（cwd=代码根）
|
||||
- `--input`自动建目录+清旧output+写input.txt；长文本先手动写input.txt再启动(不带--input)
|
||||
- **不要--nobg**（会卡在等reply循环），只能后台启动
|
||||
- 通信：output.txt(`[ROUND END]`=轮完成) → 写reply.txt继续 → 不写10min退出。reply后输出为output1/2/3.txt
|
||||
- 干预文件：`_stop`(当轮结束) | `_keyinfo`(注入working memory) | `_intervene`(追加指令)
|
||||
- [[可选fork]]：将变量history(str)写入task目录下`_history.json`继承对话上下文
|
||||
- [[可选监察者]]：主agent空闲时读output观察进度，必要时干预文件纠偏。加`--verbose`可审查原始数据
|
||||
|
||||
## 共通规则
|
||||
- 所有agent的cwd=temp，方便文件共享
|
||||
- input：目标+约束即可，subagent同等智能。**禁写步骤/过度描述**，大量数据给路径
|
||||
|
||||
## 场景1：测试模式 - 行为验证
|
||||
**用途**：观察agent真实行为，修正RULES/L2/L3/SOP
|
||||
**流程**：写prompt→启动subagent→轮询结果→验证→清理
|
||||
**原则**：只给目标，不提示位置/不诱导做法；Insight优先级>SOP；subagent的cwd=temp/
|
||||
**两种测试**：
|
||||
- 测SOP质量：input指定SOP名，排除导航干扰，失败即SOP问题
|
||||
- 测导航能力：input只写目标，验证能自主从insight找到正确SOP
|
||||
|
||||
## 场景2：Map模式 - 并行处理
|
||||
**用途**：N个独立同构子任务分发，独立上下文避免交叉污染
|
||||
**约束**：文件系统共享(优点)；键鼠不可共享；浏览器避免同tab
|
||||
**流程**：准备独立输入文件→每个启动subagent(--func优先)→收集输出汇总
|
||||
|
||||
## subagent内部plan_mode使用
|
||||
**原则**：subagent本身是完整agent，接收多步骤任务时应在内部创建plan管理执行
|
||||
**触发条件**:任务包含3个以上子步骤、子步骤之间有依赖关系、需要checkpoint来恢复执行
|
||||
**实现方式**：
|
||||
1. **主agent创建subagent时**：在input.txt中说明任务包含多个步骤，建议使用plan_mode
|
||||
2. **subagent内部执行**：检测到多步骤任务后，创建 `./subagent_plan.md` 并使用plan_mode执行
|
||||
3. **主agent监控**：只关注最终结果（output*.txt），不需要关心subagent内部如何执行
|
||||
4. **文件传递机制**：主agent创建subagent时在task_dir中生成 `context.json`，包含所有文件的**绝对路径**
|
||||
**⚠ subagent启动后第一步必须读取context.json**
|
||||
**⚠ 所有文件操作必须使用context.json中的绝对路径**
|
||||
**格式示例**：
|
||||
```json
|
||||
{
|
||||
"task": "任务描述",
|
||||
"work_dir": "/absolute/path/to/plan_dir/",
|
||||
"input_files": {
|
||||
"paper_info": "/absolute/path/to/paper_info.txt"
|
||||
},
|
||||
"output_files": {
|
||||
"pdf": "/absolute/path/to/paper.pdf",
|
||||
"report": "/absolute/path/to/paper_report.md"
|
||||
},
|
||||
"dependencies": ["paper_info.txt必须存在"]
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,62 @@
|
||||
# 监察者模式 SOP
|
||||
|
||||
目标：让用户一次说明任务后，尽量不用多轮纠偏。
|
||||
核心：**先定行动协议，再盯协议偏离，最后看完成证据。**
|
||||
|
||||
## 1. 定位
|
||||
|
||||
监察者不是工人。你是挑刺的监工，只负责看 subagent 是否按目标、约束和证据闭环完成任务；有 SOP 按 SOP 把关，无 SOP 按常理和经验把关。
|
||||
|
||||
## 2. 红线
|
||||
|
||||
- **禁止下场干活**：不替 subagent 操作浏览器、不写代码、不执行任务步骤。
|
||||
- **只读探测**：可用 `file_read`、`web_scan`、`web_execute_js`、只读 `code_run` 辅助判断对象、进度和证据；探测不是代做。
|
||||
- **沉默为主**：没发现会导致用户纠偏的问题，就不说话。
|
||||
- **一句话干预**：必须短、硬、具体，像用户直接纠偏，禁长篇教程。
|
||||
- **说人话红线**：对外 prompt/reply 必须像真实用户临时打断；1句为主，最多2句；禁评测腔/规约腔/教程腔，禁编号、rubric、协议、闭环、chosen/rejected 等内部词。
|
||||
- **`_keyinfo` 只用于预注入**：在 subagent 到达关键步骤前提醒；已经犯错必须用 `_intervene`。
|
||||
- **训练数据变体**：若用户要求轨迹像真实多轮对话，禁 `_keyinfo/_intervene`，改用 `_stop` + `reply.txt` 续聊；reply 仍短硬像真人。
|
||||
- **证据纪律**：学生引用记忆/日志/`file:line` 时，必须核对是否有实际 `file_read`/工具覆盖；读到一半却引用未读范围，按“嘴读”纠偏。
|
||||
|
||||
## 3. 执行顺序（硬性，不可跳步）
|
||||
|
||||
**Step A — 立刻启动 subagent**（纯机械，不需思考）
|
||||
|
||||
读完本 SOP 和 `subagent.md` 后，从用户 prompt 中剥离任务，写 input，启动。调用细节见 `subagent.md`；input 只给目标+约束，长文本给路径。
|
||||
|
||||
**Step B — 写预期文档**（利用 subagent 启动+读环境的时间）
|
||||
|
||||
subagent 在跑前几轮环境探测时，你写一屏预期文档（产出文件），只写三项：
|
||||
|
||||
1. **意图与锚点**：目标、必须先看的对象、明显不是目标的对象；只列显性或高代价的账号/身份/归属字段。
|
||||
2. **行动协议**：正确推进顺序；必须先找/读哪些 SOP 或工具说明，并把其中的禁止/必须/格式要求纳入约束；何时核对对象/路径/账号/窗口/字段；危险动作前验证什么；调研/测试类先跑通最小可复现闭环再扩大。
|
||||
3. **完成证据**：最后必须有什么证据才能交付：状态、截图、日志、diff、测试结果、报告路径、使用方法等。
|
||||
|
||||
**Step C — tail 监控与干预**（预期文档写完后开始）
|
||||
|
||||
轮询 `temp/{task_name}/output*.txt`。看到 `[ROUND END]` 后判断是否需要写 `reply.txt` 续聊或用干预文件纠偏。tail 中若发现新信息（真实文件结构、字段、约束、隐藏难点），回头补充/更新预期文档，尤其补全此前想不到的验收指标。tail 时盯这些偏离：
|
||||
|
||||
- 没先落到用户锚点，凭记忆/猜测行动。
|
||||
- 没找应有 SOP/工具说明，或读了却没按其中禁止/必须/格式要求做。
|
||||
- 对象、路径、窗口、账号、环境、数据源偏了。
|
||||
- 顺序错，尤其危险/不可逆动作前没验证。
|
||||
- 粒度错：该先跑通却大设计，该最小改动却重构，该调研测试却泛泛写报告。
|
||||
- 要交付了，但证据、路径、格式、测试或说明不够。
|
||||
- 反复试错、泛泛搜索、卡住不读日志/不看 SOP/不收敛到下一步。
|
||||
|
||||
干预示例：
|
||||
|
||||
```text
|
||||
停。用户锚点是 X，不是 Y。先看 X。
|
||||
不要提交。危险动作前还没验证 X。
|
||||
粒度错了。先跑通最小闭环。
|
||||
不要交付。还缺 X 证据/路径/格式。
|
||||
先停。这个任务应先读/遵守 X SOP，把禁止/必须项列进约束。
|
||||
你漏了 XX 约束，补上后再继续。
|
||||
连续失败先停，读错误日志再决定。
|
||||
别继续乱试。先读 X SOP/日志，列出一个最小下一步再做。
|
||||
```
|
||||
|
||||
## 4. 最终验收
|
||||
|
||||
按预期文档逐条核对：锚点是否命中、行动协议是否遵守、危险前是否验证、最终产物/结果是否满足用户目标、质量是否够、证据是否足够且路径/格式明确。任一不满足就返工；全部满足才交付用户。
|
||||
@@ -0,0 +1,138 @@
|
||||
# TMWebDriver SOP
|
||||
|
||||
- 直接用web_scan/web_execute_js工具。本文件只记录特性和坑。
|
||||
- 底层：`../TMWebDriver.py`通过Chrome扩展接管用户浏览器（保留登录态/Cookie）
|
||||
- 非Selenium/Playwright，保留用户浏览器登录态
|
||||
|
||||
## 通用特性
|
||||
- ⚠web_execute_js里使用`await`时需**显式`return`**才能拿到返回值（底层async包裹，不写return则返回null）
|
||||
- ✅web_scan自动穿透同源iframe；跨域iframe需CDP或postMessage（见下方章节）
|
||||
|
||||
## 限制(isTrusted)
|
||||
- JS事件`isTrusted=false`，敏感操作（如文件上传/部分按钮）可能被拦截；这类场景首选**CDP桥**
|
||||
- ⚠JS点击按钮打不开新tab→可能是浏览器弹窗拦截，换CDP点击试试
|
||||
- Vue3自定义组件(Select/Dropdown)：⭐优先vnode实例调用(无视口限制)→见**vue3_component_sop**；CDP坐标点击仅适合选项少且可见的场景
|
||||
- 文件上传：⭐首选**DataTransfer API**（纯JS，无CDP依赖）：`new File([content],name,{type}) → new DataTransfer().items.add(file) → input.files=dt.files → dispatch input+change`；CDP `DOM.setFileInputFiles` 在tmwd桥环境nodeId跨调用失效，不推荐；备选ljqCtrl物理点击
|
||||
- 需转物理坐标时：`physX = (screenX + rect中心x) * dpr`，`physY = (screenY + chromeH + rect中心y) * dpr`；其中 `chromeH = outerHeight - innerHeight`
|
||||
|
||||
## 导航
|
||||
- `web_scan` 仅读当前页不导航，切换网站用 `web_execute_js` + `location.href='url'`
|
||||
- ⚠导航与后续操作**必须拆成两次调用**：同一段JS内 `location.href` 后继续操作→报错`Inspected target navigated or closed`（页面已换执行上下文销毁）。先导航→等加载→再单独执行操作
|
||||
|
||||
## Google图搜
|
||||
- class名混淆禁硬编码，点击结果用 `[role=button]` div
|
||||
- web_scan过滤边栏，弹出后用JS：文本`document.body.innerText`，大图遍历img按`naturalWidth`最大取src
|
||||
- "访问"链接：遍历a找`textContent.includes('访问')`的href
|
||||
- 缩略图：`img[src^="data:image"]`直接提取；大图src可能截断用`return img.src`
|
||||
|
||||
## Chrome下载PDF
|
||||
场景：PDF链接在浏览器内预览而非下载
|
||||
```js
|
||||
fetch('PDF_URL').then(r=>r.blob()).then(b=>{
|
||||
const a=document.createElement('a');
|
||||
a.href=URL.createObjectURL(b);
|
||||
a.download='filename.pdf';
|
||||
a.click();
|
||||
});
|
||||
```
|
||||
注意：需同源或CORS允许，跨域先导航到目标域再执行
|
||||
|
||||
## Chrome后台标签节流
|
||||
- 后台标签中`setTimeout`被Chrome intensive throttling延迟到≥1min/次，扩展脚本中避免依赖setTimeout轮询
|
||||
- 某些SPA页面需CDP `Page.bringToFront`切到前台才会加载数据
|
||||
|
||||
## CDP桥(tmwd_cdp_bridge扩展) ⭐首选
|
||||
扩展路径：`assets/tmwd_cdp_bridge/`(需安装，含debugger权限)
|
||||
⚠TID约定标识：首次运行自动生成到`assets/tmwd_cdp_bridge/config.js`(已gitignore)，扩展通过manifest引用
|
||||
调用：`web_execute_js` script直传JSON字符串（工具层自动识别对象格式，走WS→background.js cmd路由）
|
||||
```js
|
||||
// 直接传JSON字符串作为script参数，无需DOM操作
|
||||
web_execute_js script='{"cmd": "cookies"}'
|
||||
web_execute_js script='{"cmd": "tabs"}'
|
||||
web_execute_js script='{"cmd": "cdp", "tabId": N, "method": "...", "params": {...}}'
|
||||
web_execute_js script='{"cmd": "batch", "commands": [...]}'
|
||||
// 返回值直接是JSON结果
|
||||
```
|
||||
通信方式：⭐JSON字符串直传(首选) | TID DOM方式(TID元素+MutationObserver，web_scan/execute_js底层依赖)
|
||||
单命令：`{cmd:'tabs'}` | `{cmd:'cookies'}` | `{cmd:'cdp', tabId:N, method:'...', params:{...}}` | `{cmd:'management', method:'list|reload|disable|enable', extId:'...'}`
|
||||
- management：list返回所有扩展信息；reload/disable/enable需传extId
|
||||
- contentSettings：`{cmd:'contentSettings', type:'automaticDownloads', pattern:'https://*/*', setting:'allow'}`
|
||||
- 绕过Chrome"下载多个文件"对话框（该对话框会阻塞整个浏览器JS执行）
|
||||
- type可选：automaticDownloads/popups/notifications等；setting：allow/block/ask
|
||||
- ⚠CDP的Browser.setDownloadBehavior在扩展中不可用（chrome.debugger仅tab级），此为替代方案
|
||||
- ⭐batch混合：`{cmd:'batch', commands:[{cmd:'cookies'},{cmd:'tabs'},{cmd:'cdp',...},...]}`
|
||||
- 返回`{ok:true, results:[...]}`，一次请求多命令，CDP懒attach复用session
|
||||
- 子命令会自动继承外层batch的tabId（如cookies命令可正确获取当前页面URL）
|
||||
- `$N.path`引用第N个结果字段(0-indexed)，如`"nodeId":"$2.root.nodeId"`
|
||||
- ⚠batch前序命令失败时，后续`$N`引用会静默变成undefined；要检查results数组中每项的ok状态
|
||||
- 典型文件上传：getDocument(**depth:1**) → querySelector(`input[type=file]`) → setFileInputFiles
|
||||
- 思想：
|
||||
- 同一链路内保持nodeId来源一致，不混用querySelector路径与performSearch路径
|
||||
- 上传后前端框架可能不感知，必要时JS补发`input`/`change`事件
|
||||
- 上传前检查`input.accept`；多input时用accept/父容器语义区分
|
||||
- 等待元素优先用`DOM.performSearch('input[type=file]')`做轻量轮询
|
||||
- 瞬态input的核心是**缩短发现→setFileInputFiles时间窗**：优先同batch完成；再不行用DOM事件监听；猴子补丁仅作兜底思路
|
||||
- ⚠tabId：CDP默认sender.tab.id(当前注入页)，跨tab需显式tabId或先batch内tabs查
|
||||
- ⭐跨tab无需前台：指定tabId即可操作后台标签页
|
||||
|
||||
## CDP点击完整生命周期（✅已验证）
|
||||
- 通用点击需**三事件序列**：mouseMoved → mousePressed → mouseReleased（间隔50-100ms）
|
||||
- 省略mouseMoved会导致MUI Tooltip/Ant Design Dropdown等hover依赖组件失效
|
||||
- ⚠autofill释放是特例，只需mousePressed即可（见下方autofill章节）
|
||||
- ⭐**坐标系结论**：稳定状态下 CDP坐标 = `getBoundingClientRect()` 坐标，**无需修正**
|
||||
- ⚠**首次attach陷阱**：CDP debugger首次attach时Chrome弹出infobar("正在受自动化控制"，~20px高)，页面内容被推下
|
||||
- 如果在attach前测量坐标、attach后发送点击 → 坐标偏移！（之前Currency下拉失败的根因）
|
||||
- ✅**解决**：确保测量坐标在CDP已attach稳定之后（即infobar已出现后再getBoundingClientRect）
|
||||
- 实践：首次CDP操作前先发一个无害的`mouseMoved(0,0)`预热，之后坐标系就稳定了
|
||||
- ⭐**下拉框(Vue3 oxd-select等)CDP操作流程**：
|
||||
1. 获取select元素rect → CDP点击打开下拉
|
||||
2. 获取option元素rect → CDP点击选中（option是动态DOM，打开后才能测量）
|
||||
- 已验证：CDP点击对自定义下拉框有效，无isTrusted问题
|
||||
- ⚠**限制**：选项多时底部option超出视口，CDP坐标够不着→此时应优先vnode方案(见vue3_component_sop)
|
||||
- 坐标修正（页面有transform:scale/zoom时）：
|
||||
```js
|
||||
var scale = window.visualViewport ? window.visualViewport.scale : 1;
|
||||
var zoom = parseFloat(getComputedStyle(document.documentElement).zoom) || 1;
|
||||
var realX = x * zoom; var realY = y * zoom;
|
||||
```
|
||||
- iframe内元素CDP点击：坐标需合成 `finalX = iframeRect.x + elRect.x`
|
||||
- 跨域iframe拿不到contentDocument：
|
||||
- ⚠`Target.getTargets`/`Target.attachToTarget`在CDP桥中返回"Not allowed"(chrome.debugger权限限制)
|
||||
- ⭐**已验证方案**：`Page.getFrameTree`找iframe frameId → `Page.createIsolatedWorld({frameId})`获取contextId → `Runtime.evaluate({expression, contextId})`在iframe中执行JS
|
||||
- batch链式引用：`$0.frameTree.childFrames`遍历找url匹配的frame，`$1.executionContextId`传给evaluate
|
||||
- postMessage中继方案仅在content script已注入iframe时有效，第三方支付iframe通常无注入
|
||||
|
||||
## CDP文本输入（未验证，BBS#23）
|
||||
- `insertText`快但无key事件；受控组件需补dispatch `input`事件
|
||||
- 需完整键盘模拟时用`dispatchKeyEvent`逐键派发
|
||||
|
||||
## CDP DOM域穿透 closed Shadow DOM（未验证，BBS#24/#25）
|
||||
- `DOM.getDocument({depth:-1, pierce:true})` 穿透所有Shadow边界（含closed）
|
||||
- `DOM.querySelector({nodeId, selector})` 定位 → `DOM.getBoxModel({nodeId})` 取坐标
|
||||
- getBoxModel返回content八值[x1,y1,...x4,y4]，中心用**四点平均**：centerX=sum(x)/4, centerY=sum(y)/4
|
||||
- ⚠不能简化为对角线平均——元素有transform:rotate/skew时四点非矩形
|
||||
- querySelector**不能跨Shadow边界写组合选择器**，需分步：先找host再在其shadow内找子元素
|
||||
- ⚠nodeId在DOM变更后失效 → 用`backendNodeId`更稳定，或重新getDocument刷新
|
||||
|
||||
|
||||
## autofill获取与登录
|
||||
检测：web_scan输出input带`data-autofilled="true"`，value显示为受保护提示(非真实值，Chrome安全保护需点击释放)
|
||||
- ⚠**前置条件：必须先CDP `Page.bringToFront` 切tab到前台**，Chrome仅在前台tab释放autofill保护值，后台tab物理点击无效
|
||||
- ⭐**一键释放与登录**：bringToFront → mousePressed点任一字段(无需Released，一个释放全页) → 等500ms → 补input/change事件 → 点登录
|
||||
|
||||
## 验证码/页面视觉截图
|
||||
- ⭐首选CDP截图：`Page.captureScreenshot`(format:'png')→返回base64，无需前台/后台tab也行，全页高清
|
||||
- 验证码canvas/img：JS `canvas.toDataURL()` 直接拿base64最干净
|
||||
|
||||
## simphtml与TMWebDriver调试
|
||||
- simphtml调试必须通过`code_run`注入JS到真实浏览器（Python端无法模拟DOM）
|
||||
- `d=TMWebDriver()`, `d.set_session('url_pattern')`, `d.execute_js(code)` → 返回`{'data': value}`
|
||||
- simphtml：`str(simphtml.optimize_html_for_tokens(html))` — 返回BS4 Tag需str()
|
||||
|
||||
## 连不上排查
|
||||
web_scan失败时按序排查（自动检测优先，用户参与放最后）：
|
||||
①浏览器没开？→检查浏览器进程是否在跑(tasklist/ps)，没有则启动并打开正常URL（⚠about:blank等内部页不加载扩展）
|
||||
②WS后台挂了？→本机18766端口没监听即dead→手动**后台持续运行**`from TMWebDriver import TMWebDriver; TMWebDriver()`起master
|
||||
③扩展没装？→读Chrome用户目录下`Secure Preferences`→`extensions.settings`中找`path`含`tmwd_cdp_bridge`的条目
|
||||
找到→扩展已装，排查其他原因；没找到→走web_setup_sop
|
||||
④以上都正常仍连不上→请求用户协助
|
||||
@@ -0,0 +1,204 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
UI元素检测 - 基于 OmniParser-v2.0 的 icon_detect YOLO 模型 + RapidOCR
|
||||
|
||||
首次使用前必须准备对应权重；这是 OmniParser-v2.0 的 icon_detect YOLO 模型。
|
||||
若缺失模型文件，新用户/AI 应搜索并下载 OmniParser-v2.0 icon_detect YOLO 权重。
|
||||
|
||||
用法:
|
||||
from ui_detect import detect
|
||||
elements = detect("screenshot.png") # 默认match模式
|
||||
elements = detect(pil_image) # 支持PIL.Image
|
||||
elements = detect(img, mode='crop') # crop备选
|
||||
返回: [{'bbox':[x1,y1,x2,y2], 'type':'icon'|'text', 'label':str|None, 'confidence':float}]
|
||||
模式: match=YOLO+全图OCR IoU匹配(推荐,1.2s,无文字图标label=None可VLM保底) | crop=拼接crop OCR(备选,更精确,2.3s)
|
||||
依赖: ultralytics, rapidocr-onnxruntime, pillow, numpy
|
||||
"""
|
||||
from pathlib import Path
|
||||
from PIL import Image, ImageDraw
|
||||
import numpy as np
|
||||
import json, urllib.request, subprocess, sys, time
|
||||
|
||||
#print('[UI DETECT] 截图分析后必须使用物理坐标，ljqCtrl也使用物理坐标！')
|
||||
|
||||
DEFAULT_MODEL = str(Path(__file__).resolve().parent.parent / 'temp' / 'weights' / 'icon_detect' / 'model.pt')
|
||||
|
||||
try:
|
||||
from rapidocr_onnxruntime import RapidOCR
|
||||
_ocr = RapidOCR()
|
||||
except ImportError: _ocr = None
|
||||
|
||||
_YOLO = None
|
||||
_YOLO_PORT = 31876
|
||||
|
||||
def _yolo_local(image_path, conf=0.25):
|
||||
global _YOLO
|
||||
if _YOLO is None:
|
||||
from ultralytics import YOLO
|
||||
_YOLO = YOLO(DEFAULT_MODEL)
|
||||
res = _YOLO(image_path, conf=conf, verbose=False)
|
||||
boxes = []
|
||||
for r in res:
|
||||
for b in r.boxes:
|
||||
x1, y1, x2, y2 = map(int, b.xyxy[0].cpu().numpy())
|
||||
boxes.append([x1, y1, x2, y2, float(b.conf[0])])
|
||||
return boxes
|
||||
|
||||
|
||||
def _ping_yolo_daemon():
|
||||
try: return urllib.request.urlopen(f'http://127.0.0.1:{_YOLO_PORT}/ping', timeout=0.1).read() == b'ui_detect_yolo'
|
||||
except Exception: return False
|
||||
|
||||
def _yolo(image_path, conf=0.25):
|
||||
"""YOLO检测 → list of [x1,y1,x2,y2,conf]；默认模型走跨进程daemon cache，失败回退本地"""
|
||||
if not _ping_yolo_daemon():
|
||||
kw = {'creationflags': getattr(subprocess, 'CREATE_NO_WINDOW', 0)} if sys.platform == 'win32' else {}
|
||||
subprocess.Popen([sys.executable, __file__, '--yolo-daemon'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, **kw)
|
||||
for _ in range(15):
|
||||
if _ping_yolo_daemon(): break
|
||||
time.sleep(0.5)
|
||||
try:
|
||||
data = json.dumps({'path': str(image_path), 'conf': conf}).encode('utf-8')
|
||||
req = urllib.request.Request(f'http://127.0.0.1:{_YOLO_PORT}/yolo', data=data, headers={'Content-Type': 'application/json'})
|
||||
return json.loads(urllib.request.urlopen(req, timeout=3).read().decode('utf-8'))['boxes']
|
||||
except Exception: return _yolo_local(image_path, conf)
|
||||
|
||||
def _ocr_full(image_path):
|
||||
"""全图OCR → list of [x1,y1,x2,y2,text,conf]"""
|
||||
if not _ocr: return []
|
||||
result, _ = _ocr(image_path)
|
||||
if not result: return []
|
||||
out = []
|
||||
for bbox, text, conf in result:
|
||||
xs = [p[0] for p in bbox]; ys = [p[1] for p in bbox]
|
||||
out.append([int(min(xs)), int(min(ys)), int(max(xs)), int(max(ys)), text, conf])
|
||||
return out
|
||||
|
||||
def _ocr_crops_batch(img, yolo_boxes):
|
||||
"""批量OCR：将所有YOLO框crop垂直拼接为一张图，一次OCR，按y坐标映射回各box → {box_idx: text}"""
|
||||
if not _ocr or not yolo_boxes: return {}
|
||||
|
||||
crops, offsets = [], [] # offsets: [(y_off, orig_x1, orig_y1, box_idx)]
|
||||
max_w, y_cursor = 0, 0
|
||||
for idx, (x1, y1, x2, y2, _) in enumerate(yolo_boxes):
|
||||
crop = img.crop((x1, y1, x2, y2))
|
||||
w, h = crop.size
|
||||
max_w = max(max_w, w)
|
||||
crops.append(crop)
|
||||
offsets.append((y_cursor, x1, y1, idx))
|
||||
y_cursor += h
|
||||
if max_w == 0: return {}
|
||||
# 垂直拼接
|
||||
stitched = Image.new('RGB', (max_w, y_cursor), (255, 255, 255))
|
||||
for i, crop in enumerate(crops):
|
||||
stitched.paste(crop, (0, offsets[i][0]))
|
||||
|
||||
result, _ = _ocr(np.array(stitched))
|
||||
if not result: return {}
|
||||
# 映射：OCR框中心y → 归属的crop
|
||||
labels = {}
|
||||
for bbox, text, _ in result:
|
||||
cy = sum(p[1] for p in bbox) / len(bbox)
|
||||
for y_off, ox1, oy1, idx in offsets:
|
||||
h = yolo_boxes[idx][3] - yolo_boxes[idx][1]
|
||||
if y_off <= cy < y_off + h:
|
||||
old = labels.get(idx)
|
||||
labels[idx] = (old + ' ' + text) if old else text
|
||||
break
|
||||
return labels
|
||||
|
||||
def _iou(a, b):
|
||||
"""计算两个bbox的交集占b面积的比例(包含率)"""
|
||||
x1, y1, x2, y2 = max(a[0],b[0]), max(a[1],b[1]), min(a[2],b[2]), min(a[3],b[3])
|
||||
inter = max(0, x2-x1) * max(0, y2-y1)
|
||||
area_b = (b[2]-b[0]) * (b[3]-b[1])
|
||||
return inter / area_b if area_b > 0 else 0
|
||||
|
||||
def detect(image_path, mode='match', conf=0.25, iou_thresh=0.5):
|
||||
"""
|
||||
统一检测入口，返回元素列表:
|
||||
[{'bbox':[x1,y1,x2,y2], 'type':'icon'|'text', 'label':str|None, 'confidence':float}]
|
||||
mode: 'match' = YOLO+全图OCR空间匹配(推荐, 快) | 'crop' = YOLO+拼接OCR(备选, 更精确)
|
||||
支持 image_path: str 路径 或 PIL.Image 对象
|
||||
"""
|
||||
# 归一化：PIL Image → 临时文件
|
||||
if isinstance(image_path, Image.Image):
|
||||
import tempfile, os
|
||||
tmp = tempfile.NamedTemporaryFile(suffix='.png', delete=False)
|
||||
image_path.save(tmp.name)
|
||||
image_path = tmp.name
|
||||
img = Image.open(image_path)
|
||||
|
||||
yolo_boxes = _yolo(image_path, conf)
|
||||
elements = []
|
||||
|
||||
if mode == 'crop':
|
||||
# YOLO元素批量OCR（拼接一次推理）
|
||||
labels_map = _ocr_crops_batch(img, yolo_boxes)
|
||||
for idx, (x1, y1, x2, y2, c) in enumerate(yolo_boxes):
|
||||
elements.append({'bbox': [x1,y1,x2,y2], 'type': 'icon', 'label': labels_map.get(idx), 'confidence': c})
|
||||
# 补充：全图OCR找未被覆盖的纯文本
|
||||
for ox1, oy1, ox2, oy2, text, oc in _ocr_full(image_path):
|
||||
covered = any(_iou([x1,y1,x2,y2,_,__], [ox1,oy1,ox2,oy2]) > iou_thresh
|
||||
for x1,y1,x2,y2,_,__ in [(b[0],b[1],b[2],b[3],0,0) for b in yolo_boxes])
|
||||
if not covered:
|
||||
elements.append({'bbox': [ox1,oy1,ox2,oy2], 'type': 'text', 'label': text, 'confidence': oc})
|
||||
|
||||
elif mode == 'match':
|
||||
ocr_items = _ocr_full(image_path)
|
||||
matched_ocr = set()
|
||||
for x1, y1, x2, y2, c in yolo_boxes:
|
||||
label = None
|
||||
for i, (ox1, oy1, ox2, oy2, text, oc) in enumerate(ocr_items):
|
||||
if _iou([x1,y1,x2,y2], [ox1,oy1,ox2,oy2]) > iou_thresh:
|
||||
label = text; matched_ocr.add(i); break
|
||||
elements.append({'bbox': [x1,y1,x2,y2], 'type': 'icon', 'label': label, 'confidence': c})
|
||||
# 未匹配的OCR作为独立text元素
|
||||
for i, (ox1, oy1, ox2, oy2, text, oc) in enumerate(ocr_items):
|
||||
if i not in matched_ocr:
|
||||
elements.append({'bbox': [ox1,oy1,ox2,oy2], 'type': 'text', 'label': text, 'confidence': oc})
|
||||
#if [x for x in elements if x['label'] is None]: print('[TIPS] crop grid + VLM to identify target no text icon if needed')
|
||||
print('[TIPS] UI DETECT contains OCR, no need to run OCR again!')
|
||||
return elements
|
||||
|
||||
def visualize_for_debug(image_path, elements, output_path=None):
|
||||
"""Only use when user wants to DEBUG!"""
|
||||
from PIL import ImageFont
|
||||
img = Image.open(image_path)
|
||||
draw = ImageDraw.Draw(img)
|
||||
try:
|
||||
font = ImageFont.truetype("msyh.ttc", 14)
|
||||
except:
|
||||
font = ImageFont.load_default()
|
||||
for el in elements:
|
||||
x1, y1, x2, y2 = el['bbox']
|
||||
color = 'red' if el['type'] == 'icon' else 'blue'
|
||||
draw.rectangle([x1, y1, x2, y2], outline=color, width=2)
|
||||
tag = el.get('label') or f"{el['confidence']:.2f}"
|
||||
draw.text((x1, y1-16), tag[:15], fill=color, font=font)
|
||||
if output_path: img.save(output_path)
|
||||
return img
|
||||
|
||||
def _serve_yolo_daemon():
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
class H(BaseHTTPRequestHandler):
|
||||
def log_message(self, *args): pass
|
||||
def handle_one_request(self): self.server.last=time.time(); return super().handle_one_request()
|
||||
def do_GET(self):
|
||||
if self.path == '/ping':
|
||||
self.send_response(200); self.end_headers(); self.wfile.write(b'ui_detect_yolo')
|
||||
else:
|
||||
self.send_response(404); self.end_headers()
|
||||
def do_POST(self):
|
||||
try:
|
||||
d = json.loads(self.rfile.read(int(self.headers.get('Content-Length', 0))))
|
||||
body = json.dumps({'boxes': _yolo_local(d['path'], d.get('conf', 0.25))}).encode('utf-8')
|
||||
self.send_response(200); self.end_headers(); self.wfile.write(body)
|
||||
except Exception as e:
|
||||
body = json.dumps({'error': repr(e)}).encode('utf-8')
|
||||
self.send_response(500); self.end_headers(); self.wfile.write(body)
|
||||
s=HTTPServer(('127.0.0.1', _YOLO_PORT), H); s.timeout=60; s.last=time.time()
|
||||
while time.time()-s.last < 3600: s.handle_request()
|
||||
|
||||
if __name__ == '__main__' and '--yolo-daemon' in sys.argv:
|
||||
_serve_yolo_daemon()
|
||||
@@ -0,0 +1,181 @@
|
||||
# GA UltraPlan SOP
|
||||
## 1. Protocol: start and continue
|
||||
### What this is
|
||||
UltraPlan is Python-scripted multi-agent orchestration. The main agent designs phases, prompts, fan-out/fan-in, and stop/continue decisions; subagents do task-facing work.
|
||||
### Opt-in only
|
||||
Start UltraPlan only when the user explicitly says `ultraplan`, `UltraPlan`, or `ultraplan mode`. If not opted in, do not start it; at most mention it is available.
|
||||
### First move
|
||||
Once opted in, the next substantive action is writing and running the first script.
|
||||
Before the first `plan(...)`, do not inspect source, tests, logs, imports, file listings, pages, or APIs for the task itself.
|
||||
Allowed pre-launch work: record objective/constraints, confirm cwd is GA `temp/`, write the minimal script.
|
||||
### File and cwd contract
|
||||
Scripts are plain Python files under GA `temp/`; run them with cwd = `temp/`.
|
||||
Reference repo files from `temp/`, e.g. `../assets/...`; never place UltraPlan scripts in the repo code tree.
|
||||
Every script starts with the real API contract and a shared artifact directory:
|
||||
```python
|
||||
import os, sys
|
||||
sys.path.append("..")
|
||||
from assets.ga_ultraplan import plan, phase, parallel, mapchain
|
||||
RUN_DIR = os.path.abspath("ultraplan_<stable_slug>")
|
||||
plan(RUN_DIR)
|
||||
ARTIFACT_DIR = RUN_DIR
|
||||
```
|
||||
`plan(...)` must be the first UltraPlan statement; defining `RUN_DIR` before it is allowed. If import/plan fails, diagnose only cwd/path/import/daemon startup, not the user task.
|
||||
### Same-plan continuation
|
||||
For one user objective, every later script reuses the exact same `RUN_DIR` and `plan(RUN_DIR)`.
|
||||
Round 2/3/etc. are new scripts under the same plan/work directory, not new plans. Continuation changes only phases/prompts/archetype.
|
||||
A finished script is not proof the task is finished: read reducer/report paths, then answer, ask, apply a completed result, or launch the next same-plan script.
|
||||
### Delegation boundary
|
||||
Do not solve the task outside UltraPlan. Do not perform task discovery, implementation, review, or verification in main chat.
|
||||
The main agent may read outputs only to supervise and decide the next script.
|
||||
Exactly one agent is the UltraPlan orchestrator for one objective. The orchestrator may read this SOP; ordinary workers must not be told to read UltraPlan SOP, start UltraPlan, design phases, or delegate.
|
||||
If the orchestrator itself is a subagent, give only objective, constraints, output budget, and permission to use UltraPlan; do not paste SOP, prescribe phases, or tell it which SOP files to read. It chooses context reads.
|
||||
Worker prompts are job tickets, not mini-SOPs: role, exact scope, inputs, allowed/forbidden actions, evidence, short output shape, stop condition.
|
||||
Every worker prompt must include the boundary when relevant: `Do not start UltraPlan. Do not delegate. If decomposition is needed, report blocker only.`
|
||||
|
||||
## 2. Core mental model
|
||||
### Why orchestrate
|
||||
Assume a strong single executor can handle long documents, complex code, and coherent multi-file edits. Do not orchestrate merely because work looks large.
|
||||
Orchestration is mainly omission control: missing items, angles, hypotheses, evidence, checks, or residual improvements. Hunt is the special hard-search case: one direct attempt may hit many dead ends before a viable proof/root cause/solution appears.
|
||||
### Three decisions
|
||||
1. Problem class: Explore, Sweep, Hunt, or Improve.
|
||||
2. Omission risk: unknown-list discovery or known-list ownership.
|
||||
3. Topology: one executor, parallel width, phase/loop depth, pipeline, barrier, reducer.
|
||||
### Parallel is only for omission control
|
||||
Use parallel in exactly two cases:
|
||||
- Unknown-list discovery: the item list is not known; split by meaningful search lenses, paths, evidence sources, representations, failure modes, or counterexamples. Evidence sources may include local code, logs/tests, live reproduction, user artifacts, and web/Google research when external ecosystem knowledge may reveal known issues, API limits, prior incidents, or platform constraints.
|
||||
- Known-list ownership: the item list is known, independent, and AI-sized; assign ownership so no item is skipped.
|
||||
Do not parallelize coherent execution. If the task is clear, bounded, coherent, and in one capable agent's comfort zone, use one executor.
|
||||
### Width vs depth
|
||||
Parallel width: independent angles/items may surface different omissions.
|
||||
Phase/loop depth: later search depends on earlier findings, reduction, verification, or dead ends. Use phases/loops for find -> dedupe/rank -> verify/refute -> search residuals until dry.
|
||||
### Choose by main risk
|
||||
Explore: space unknown; risk is missing angles/items.
|
||||
Sweep: known independent list; risk is missing items/status.
|
||||
Hunt: cause/solution/proof unknown or hard; risk is wrong path/dead ends.
|
||||
Improve: existing artifact; risk is residual defects/opportunities after execution.
|
||||
Design/Integrator are support moves: design contracts prevent divergent parallel output; integrators restore global coherence after parallel work.
|
||||
|
||||
## 3. Tool semantics and output discipline
|
||||
`phase(name, desc="")` is visible structure. Name the current archetype and reducer boundary.
|
||||
`parallel(tasks, max_workers=None, **data)` runs independent tasks and returns result paths in input order. Default concurrency is engine-chosen; omit `max_workers` in examples unless there is a real reason.
|
||||
Task forms: tuple/list `(desc, prompt)` or dict with `desc`, `prompt`, `data`, `llm_no`, `timeout`.
|
||||
Subagent calls return `.out.txt` paths. Later prompts should reference paths and tell workers to read/tail only what they need.
|
||||
`mapchain(items, step1, step2, ...)` runs steps sequentially per item and items concurrently. `{item}` is original item; `{previous}` is the prior step result path.
|
||||
A `parallel(...)` between stages is a barrier. Use it only when the next stage needs cross-result dedupe, ranking, shared context, or early-exit. If each item can continue independently, use `mapchain`.
|
||||
Workers return plain text, not JSON by default, but it must be reducer-readable: stable IDs, evidence paths/quotes, verdict/status, risk, next action.
|
||||
Brevity rule: be as short as practical. Main chat reports only status, blocker, next action. Workers/reducers/verifiers include necessary evidence but no padding.
|
||||
Forbid filler: no background essay, copied prompt, SOP recap, chain-of-thought prose, unsupported impression, or vague `done`.
|
||||
Reducers compare rather than concatenate: accept, reject, dedupe, rank, expose contradictions, state coverage bounds, and recommend stop/continue/next archetype.
|
||||
|
||||
## 4. Prompt contract
|
||||
A worker prompt must be executable without follow-up.
|
||||
State: role, exact scope, input paths/items, artifact directory, allowed sources/tools, evidence standard, concise output shape, stop condition, and exclusions. If web/Google search is allowed, say so explicitly; require URLs/source names, distinguish sourced facts vs local evidence vs hypotheses, and map every external finding to a task hypothesis, discriminator, mitigation, or verification step.
|
||||
Tell workers what not to do when overlap is harmful. Tell verifiers whether to confirm, refute, reproduce, compare, or inspect local formatting.
|
||||
Prefer file paths over pasted long context. Give only the state needed for that worker.
|
||||
If a worker creates or edits files, require saving them under `ARTIFACT_DIR` and returning paths.
|
||||
If an operation is risky or irreversible, the prompt must stop before doing it unless the user already approved it.
|
||||
|
||||
## 5. Archetypes
|
||||
### Explore
|
||||
Use Explore when the space is unknown and choosing one path too early would bias the task.
|
||||
Fan out by lenses, not by fake dependencies: architecture, failure modes, data/evidence sources, constraints, user intent, external web/official/forum evidence, reproduction route, counterexample route, test surface, style risk. Use web search only when outside knowledge may change the map; forbid generic background research.
|
||||
Each explorer returns lens, covered area, findings/frontiers, evidence, unknowns, and dead ends.
|
||||
Reducer builds the map: accepted facts, rejected claims, promising frontiers, missing lenses, contradictions, and next archetype.
|
||||
Stop Explore when the reducer can name a bounded Execute, Hunt, Improve, or Sweep.
|
||||
Research/report tasks often start with material collection plus Explore of different research paths; save gathered material as files, then synthesize a report and Improve it. In engineering/debugging tasks, web research is a collector lens feeding a reducer, not the final artifact unless the user asked for a research report.
|
||||
### Hunt
|
||||
Use Hunt for uncertain root cause, high-stakes claim validation, or hard solution/proof search.
|
||||
Typical flow: collect evidence surface -> synthesize facts/timeline/contradictions -> generate diverse hypotheses/approaches -> rank by evidence/value/cost/verifiability -> verify selected candidates.
|
||||
Fan out by non-overlapping blades or evidence sources: local code/static path, logs/errors/tests, reproduction behavior, recent changes, dependency edges, external web/official/forum evidence, constraints, weird angles, alternate representation, counterexamples. Use web research when known ecosystem/platform failures may be missing from local evidence; it must return cited mechanisms and discriminators, not a background essay.
|
||||
Each hunter returns candidate/approach ID, evidence, confidence, why distinct, why plausible, how to verify, and dead ends.
|
||||
Verification becomes Sweep only after there is a known independent hypothesis list.
|
||||
If all attempts fail, record rejected paths, exclude repeats, change blades/representation, and Hunt again. Dead ends are progress.
|
||||
### Improve
|
||||
Use Improve when there is an existing artifact to fix, simplify, optimize, rewrite, polish, or decide among alternatives.
|
||||
Improve is an outer loop, not one edit: find opportunities/residual risks -> reduce/prioritize -> execute selected change -> verify/test -> search residuals -> repeat.
|
||||
Do not start Improve with mechanical fixed lenses. Ask what omissions matter for this artifact, then choose search lenses only when each lens can uniquely find something.
|
||||
Default execution is one AI executor for small/coupled/coherent work. Sweep only when there is a known independent item list; Hunt if cause/option is unknown.
|
||||
Example: for a single-file/coherent rewrite, use one executor, then run real tests (see Verification shapes: discover existing tests/demos/CLI usage or build minimal real ones, not import-only smoke), then optionally use unknown-list discovery to find remaining regressions, style issues, or missed simplifications.
|
||||
After execution, verify. If coverage is uncertain, use phase/loop depth: find plausible untested failures -> dedupe/rank -> verify/refute -> search residuals until dry. Use parallel width only for distinct discovery lenses.
|
||||
For important changes, use adversarial verify: ask workers to refute the result, not merely agree.
|
||||
Stop only when no material improvement remains or remaining items are tiny/unsafe.
|
||||
### Sweep
|
||||
Use Sweep when the item list is known, items are truly independent, and coverage/status matters.
|
||||
Classic case: download games A/B/C/D. Each item has its own search/download/verify route; parallel ownership prevents forgetting D and isolates blockers.
|
||||
Use `mapchain` for per-item inspect -> act -> verify when each item can progress without global waiting.
|
||||
Each item report includes item ID, action/result, evidence, status, unresolved risk, skipped condition, and blocker.
|
||||
Reducer reports total, covered, omitted, failed, accepted findings, rejected findings, and coverage bounds.
|
||||
Do not call every large dataset a Sweep. 12,000 correlated rows for analysis usually need one data-analysis executor/script, not 12,000 AI workers. Sweep is for AI-sized independent items, often few-to-dozens; for huge independent batches, sample/pilot then script/shard with explicit bounds.
|
||||
If several items fail similarly, reduce failures and switch to Hunt for common cause; if one item is hard, make that item a Hunt.
|
||||
|
||||
## 6. Composition rules and edge cases
|
||||
### Design before parallel construction
|
||||
If independent artifacts require shared style/terminology/format, first Explore/Design a compact contract, then Sweep construction, then one integrator pass.
|
||||
Example: add Troubleshooting sections to 12 independent docs pages. Contract first; page workers then write under contract; integrator unifies style and catches hallucinations.
|
||||
But high-coherence artifacts with small per-unit edits usually stay single-executor. Example: add one conclusion sentence to each non-title slide in a 40-slide PPT; narrative continuity is global, so one executor writes. Sweep is suitable only for local checks such as missing sentence, overflow, or layout errors.
|
||||
### Verification shapes
|
||||
Verify is not always Sweep. For single coherent artifacts, verification often uses unknown-list discovery: find possible problems, reduce them, verify/refute, then search residuals until no material issue remains.
|
||||
Use parallel verification only when distinct lenses can find different omissions. Otherwise use one verifier plus real tests.
|
||||
Decouple verification lenses by evidence source, not by sub-checklists of one method. Splitting one static read into "check API", "check parity", "check side effects" is fake parallelism: same method, same files, overlapping output. Real independent sources are usually: static analysis (read code, no run), real execution (run actual tests), and quality-vs-intent (does it meet the task's goal, e.g. simpler/cleaner). Open a parallel lens only when its evidence source is genuinely different.
|
||||
Real execution must be genuine, not import-only smoke. Optimize for finding real breakage, not for finishing cheap. The runner first discovers existing entry points (test suites, example/demo scripts, README/CLI usage, callable public APIs) and runs the relevant ones; if none cover the change, it builds minimal but real tests that exercise the changed behavior. Record exact commands and stdout/stderr/stack. Never report pass for behavior that was not actually run; mark it blocked with the concrete reason (e.g. needs live window/GPU/network) and what would unblock it.
|
||||
Sweep verification fits known local independent checks: each file has required logging format, each slide has no overflow, each downloaded game opens.
|
||||
High-stakes claims use Hunt-style validation: collect evidence, generate alternatives, verify/refute, and block confident answers if coverage is weak.
|
||||
### Research/report shape
|
||||
Research is usually not a primitive archetype. Use collection/exploration to gather materials, parallel paths for distinct search strategies, a reducer/synthesizer for the report, then Improve to remove synthesis scars, gaps, weak evidence, and style problems.
|
||||
Final chat should summarize what was produced and where files/materials are, not paste huge gathered content.
|
||||
### Multi-round continuation
|
||||
Do not write one giant script when the next phase depends on reduced results.
|
||||
After each script, read only reducer/report outputs needed to decide: answer, ask, apply completed result, or launch the next same-plan script.
|
||||
The next script's archetype comes from the reducer: Explore if the map is still unknown, Hunt for candidates, Improve for chosen artifact, Sweep for known items, Verify for high-stakes claims.
|
||||
Never restart outside UltraPlan or rename the plan because the first script finished; rename only for a different user objective.
|
||||
|
||||
## 7. Scale, failure, and bounds
|
||||
Scale to the request. Quick check uses small fan-out; comprehensive audit uses broader blades, stronger verification, and explicit coverage bounds.
|
||||
Prefer engine-chosen concurrency. More agents are worse when prompts overlap; improve decomposition before tuning execution knobs.
|
||||
Use `timeout` for risky or slow probes and require workers to report partial progress.
|
||||
If you sample, top-N, time-box, skip retries, exclude a subsystem, or hit a tool failure, make the bound visible in reducer output and final answer.
|
||||
If a worker fails, inspect its `.out.txt` or error path, then retry with narrower scope, longer timeout, different tool, or different archetype. Do not repeat a failed prompt unchanged.
|
||||
If reducers expose contradictions, launch targeted verification to resolve conflicts with evidence.
|
||||
If coverage is too weak, do not answer confidently; run another same-plan script or ask the user to choose cost/coverage.
|
||||
|
||||
## 8. Classic patterns
|
||||
Use these as recognition anchors, not rigid templates:
|
||||
1. Improve existing artifact/code: Improve loop. If execution is coherent, one executor changes it; real tests follow (discover existing tests/demos/CLI usage or build minimal real ones, not import-only smoke); use unknown-list discovery only to find residual regressions, missed simplifications, style issues, or weak tests; repeat until dry.
|
||||
2. Root cause / unsafe conclusion: Hunt. Collect evidence first (single collector if narrow; parallel collectors only for distinct evidence sources) -> synthesize -> find hypotheses/counterexamples -> verify/refute -> record dead ends and continue if unresolved.
|
||||
3. Many known independent deliverables: Sweep. Example: download A/B/C/D games. Each item gets ownership because sequential work often forgets items; methods may differ per item. Reducer tracks status/blockers.
|
||||
4. Large correlated data: not Sweep per row. 12000 rows needing analysis is one coherent data-analysis execution; Sweep only independent AI-sized subsets or residual problem cases.
|
||||
5. Research/report: parallel only for distinct search paths/sources because materials may be missed; synthesize with one writer/integrator; Improve then searches evidence gaps, synthesis scars, style problems, and missing perspectives.
|
||||
6. Simple coherent code change across modest files: one executor may do it; add Sweep only for known local checks such as per-file log format, then optional residual discovery for style/tests.
|
||||
7. Single file or high-coherence artifact verification: not Sweep. Use tests and unknown-list problem discovery; use parallel only if distinct lenses can find different omissions.
|
||||
8. Design-then-Sweep construction: when items are independent but style must match, first Explore/Design a contract, then Sweep item work, then one integrator pass.
|
||||
9. PPT/narrative conclusion edits: usually one executor for coherence; Sweep may check local layout/format only, not write each page when cross-page flow matters.
|
||||
|
||||
## 9. Minimal shapes
|
||||
```python
|
||||
BOUNDARY = "Do not start UltraPlan. Do not delegate. If decomposition is needed, report blocker only."
|
||||
ART = f"Save any artifacts under {ARTIFACT_DIR}; return paths."
|
||||
|
||||
with phase("Improve coherent artifact", "single executor -> real tests -> residual search"):
|
||||
result = parallel([("Executor", f"Apply the focused change. Keep coherence. Run real tests: find existing tests/demos/CLI usage and run them, or build minimal real ones; record commands and output; do not claim pass for unrun behavior. {ART} Return evidence/blockers. Be concise. {BOUNDARY}")])[0]
|
||||
|
||||
with phase("Find residuals", "only if coverage is uncertain"):
|
||||
# Fill only meaningful lenses; leave empty for one verifier.
|
||||
residual_lenses = []
|
||||
residuals = parallel(residual_lenses) if residual_lenses else parallel([
|
||||
("Verifier", f"Inspect {result}. Find blockers/residuals only. {ART} Return evidence and stop/continue. Be concise. {BOUNDARY}")])
|
||||
|
||||
with phase("Reduce/decide", "dedupe, verify/refute, continue/stop"):
|
||||
next_move = parallel([("Reducer", f"Use {result} and {residuals}. Return accepted/rejected, artifact paths, evidence, next action. Be concise. {BOUNDARY}")])[0]
|
||||
|
||||
with phase("Hunt", "evidence -> hypotheses -> verification plan"):
|
||||
evidence = parallel(evidence_collectors) # each collector prompt includes ART and BOUNDARY
|
||||
hypotheses = parallel(hypothesis_blades) # each hunter prompt includes ART and BOUNDARY
|
||||
ranked = parallel([("Reducer", f"Use {evidence} and {hypotheses}. Rank candidates and verification steps. Return artifact paths. Be concise. {BOUNDARY}")])[0]
|
||||
|
||||
with phase("Sweep known independent items", "per-item ownership and status"):
|
||||
reports = mapchain(items,
|
||||
("Inspect {item}", "Inspect only {item}. Save artifacts under {artifact_dir}; return paths, ID, evidence, action, risk. Be concise. " + BOUNDARY),
|
||||
("Act/verify {previous}", "Use {previous}. Save artifacts under {artifact_dir}; return paths, ID, status, evidence, blocker. Be concise. " + BOUNDARY),
|
||||
artifact_dir=ARTIFACT_DIR)
|
||||
```
|
||||
@@ -0,0 +1,120 @@
|
||||
import base64, requests, sys, os
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
# ============ 用户配置区（从 template 拷贝后只需改这里）============
|
||||
# GA自举提示：
|
||||
# mykey.py配置变量名不固定；默认值失败时，枚举候选变量名，并用最小vision请求实验探测哪组配置可用。
|
||||
# 只打印变量名/字段名/model/apibase域名路径/HTTP状态码/错误类型，禁止打印完整dict和apikey/token。
|
||||
# 常见格式：
|
||||
# xxx_config = {"apibase": "https://.../v1", "apikey": "...", "model": "...", "proxy": None}
|
||||
# 不同中转的 apibase/endpoint 可能不同，按实际状态码和响应结构修正。
|
||||
CLAUDE_CONFIG_KEY = 'claude_config141' # mykey.py 中 Claude 配置的变量名
|
||||
OPENAI_CONFIG_KEY = 'oai_config1' # mykey.py 中 OpenAI 配置的变量名
|
||||
MODELSCOPE_API_KEY = '' # 直接填你的 ModelScope token
|
||||
DEFAULT_BACKEND = 'claude' # 默认后端: 'claude' / 'openai' / 'modelscope'
|
||||
# =================================================================
|
||||
|
||||
MODELSCOPE_API_BASE = 'https://api-inference.modelscope.cn'
|
||||
MODELSCOPE_MODEL = 'Qwen/Qwen3-VL-235B-A22B-Instruct'
|
||||
|
||||
_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
for _p in [os.path.join(_DIR, '..'), os.path.join(_DIR, '../..')]:
|
||||
if _p not in sys.path: sys.path.insert(0, _p)
|
||||
|
||||
def ask_vision(image_input, prompt="详细描述这张图片的内容", timeout=60, max_pixels=1440000, backend=DEFAULT_BACKEND):
|
||||
try:
|
||||
b64 = _prepare_image(image_input, max_pixels)
|
||||
except Exception as e:
|
||||
return f"Error: 图片处理失败 - {type(e).__name__}: {e}"
|
||||
try:
|
||||
if backend == 'claude':
|
||||
return _call_claude(b64, prompt, timeout)
|
||||
elif backend == 'openai':
|
||||
mk = _load_config()
|
||||
cfg = getattr(mk, OPENAI_CONFIG_KEY)
|
||||
return _call_openai_compat(
|
||||
b64, prompt, timeout,
|
||||
apibase=cfg['apibase'], apikey=cfg['apikey'], model=cfg['model'], proxy=cfg.get('proxy')
|
||||
)
|
||||
elif backend == 'modelscope':
|
||||
return _call_openai_compat(
|
||||
b64, prompt, timeout,
|
||||
apibase=MODELSCOPE_API_BASE, apikey=MODELSCOPE_API_KEY, model=MODELSCOPE_MODEL, proxy=None
|
||||
)
|
||||
else: return f"Error: 未知backend '{backend}'，可选: claude, openai, modelscope"
|
||||
except requests.exceptions.Timeout:
|
||||
return f"Error: 请求超时 (>{timeout}s)"
|
||||
except requests.exceptions.RequestException as e:
|
||||
return f"Error: API请求失败 - {type(e).__name__}: {e}"
|
||||
except (KeyError, ValueError) as e:
|
||||
return f"Error: 响应解析失败 - {e}"
|
||||
|
||||
# ===================== 以下为内部实现 =====================
|
||||
|
||||
def _prepare_image(image_input, max_pixels=1440000):
|
||||
"""加载+缩放+base64编码，返回b64字符串"""
|
||||
from PIL import Image
|
||||
if isinstance(image_input, Image.Image):
|
||||
img = image_input
|
||||
elif isinstance(image_input, (str, Path)):
|
||||
img = Image.open(image_input)
|
||||
else:
|
||||
raise TypeError(f"image_input 必须是文件路径或PIL Image，实际: {type(image_input).__name__}")
|
||||
w, h = img.size
|
||||
if w * h > max_pixels:
|
||||
scale = (max_pixels / (w * h)) ** 0.5
|
||||
new_w, new_h = int(w * scale), int(h * scale)
|
||||
img = img.resize((new_w, new_h), Image.Resampling.LANCZOS)
|
||||
print(f" 📐 缩放: {w}×{h} → {new_w}×{new_h}")
|
||||
if img.mode in ('RGBA', 'LA', 'P'):
|
||||
rgb = Image.new('RGB', img.size, (255, 255, 255))
|
||||
rgb.paste(img, mask=img.split()[-1] if img.mode == 'RGBA' else None)
|
||||
img = rgb
|
||||
buf = BytesIO()
|
||||
img.save(buf, format='JPEG', quality=80, optimize=True)
|
||||
b64 = base64.b64encode(buf.getvalue()).decode('utf-8')
|
||||
print(f" 📦 Base64: {len(buf.getvalue())/1024:.1f}KB")
|
||||
return b64
|
||||
|
||||
def _load_config():
|
||||
import mykey
|
||||
return mykey
|
||||
|
||||
def _call_claude(b64, prompt, timeout, max_tokens=1024):
|
||||
mk = _load_config()
|
||||
cfg = getattr(mk, CLAUDE_CONFIG_KEY)
|
||||
resp = requests.post(
|
||||
cfg['apibase'] + '/v1/messages', # endpoint按中转实际情况改：有的apibase已含/v1，或路径不同
|
||||
json={'model': cfg['model'], 'max_tokens': max_tokens, 'messages': [{
|
||||
'role': 'user',
|
||||
'content': [
|
||||
{'type': 'image', 'source': {'type': 'base64', 'media_type': 'image/jpeg', 'data': b64}},
|
||||
{'type': 'text', 'text': prompt}
|
||||
]
|
||||
}]},
|
||||
headers={'x-api-key': cfg['apikey'], 'anthropic-version': '2023-06-01', 'content-type': 'application/json'},
|
||||
timeout=timeout
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()['content'][0]['text']
|
||||
|
||||
def _call_openai_compat(b64, prompt, timeout, *, apibase, apikey, model, proxy=None):
|
||||
proxies = {'https': proxy, 'http': proxy} if proxy else None
|
||||
resp = requests.post(
|
||||
apibase.rstrip('/') + '/v1/chat/completions', # endpoint按中转实际情况改：有的apibase已含/v1，或路径不同
|
||||
json={'model': model, 'messages': [{
|
||||
'role': 'user',
|
||||
'content': [
|
||||
{'type': 'text', 'text': prompt},
|
||||
{'type': 'image_url', 'image_url': {'url': f'data:image/jpeg;base64,{b64}'}}
|
||||
]
|
||||
}]},
|
||||
headers={'Authorization': f"Bearer {apikey}", 'Content-Type': 'application/json'},
|
||||
proxies=proxies, timeout=timeout
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()['choices'][0]['message']['content']
|
||||
|
||||
if __name__ == '__main__':
|
||||
pass
|
||||
@@ -0,0 +1,23 @@
|
||||
# Vision API SOP
|
||||
|
||||
## ⚠️ 前置规则（必须遵守）
|
||||
|
||||
1. **先枚举窗口**：调用 vision 前必须先用 `pygetwindow` 枚举窗口标题，确认目标窗口存在且已激活到前台。窗口不存在就不要截图。
|
||||
2. **🚫 禁止全屏截图**：必须先利用ljqCtrl截取窗口区域。能截局部（如标题栏）就不截整窗口，能截窗口就绝不全屏。全屏截图在任何场景下都不允许。
|
||||
3. **能不用 vision 就不用**：如果窗口标题/本地 OCR（`ocr_utils.py`）能获取所需信息，就不要调用 vision API，省 token 且更可靠。Vision 是最后手段。
|
||||
|
||||
## 快速用法
|
||||
|
||||
```python
|
||||
from vision_api import ask_vision
|
||||
result = ask_vision(image, prompt="描述图片内容", timeout=60, max_pixels=1_440_000)
|
||||
# image: 文件路径(str/Path) 或 PIL Image
|
||||
# backend: 'claude'(默认) | 'openai' | 'modelscope'
|
||||
# 返回 str：成功为模型回复，失败为 'Error: ...'
|
||||
```
|
||||
|
||||
## 如果没有 `vision_api.py`，初次构建vision能力
|
||||
|
||||
1. 复制 `memory/vision_api.template.py` → `memory/vision_api.py`
|
||||
2. 只改头部"用户配置区"：去 `mykey.py` 里扫描变量名（⚠️ 只看名字，禁止输出 apikey 值），尝试找能用配置名填入 `CLAUDE_CONFIG_KEY` / `OPENAI_CONFIG_KEY`，`DEFAULT_BACKEND` 选后端，并测试
|
||||
3. 保底：没有可用 config 时去 `https://modelscope.cn/my/myaccesstoken` 申请 token 填入 `MODELSCOPE_API_KEY`
|
||||
@@ -0,0 +1,200 @@
|
||||
# Vue 3 自定义组件 JS 操作 SOP
|
||||
|
||||
## 问题
|
||||
Vue 3 自定义组件（如 OxdSelect）通过 `addEventListener` 绑定事件，JS `dispatchEvent` 产生的事件 `isTrusted: false`，组件不响应。
|
||||
- `element.click()` 无效（组件可能绑定 mousedown 而非 click）
|
||||
- `dispatchEvent(new MouseEvent('mousedown'))` 无效（isTrusted:false）
|
||||
- `element.focus()` 不触发 Vue 绑定的 focus handler
|
||||
|
||||
## 解决方案：直接操作 Vue 组件实例
|
||||
|
||||
### 1. 获取 Vue 3 根入口
|
||||
```javascript
|
||||
const rootVnode = document.getElementById('app')._vnode;
|
||||
```
|
||||
|
||||
### 2. 遍历 vnode 树匹配 DOM 元素
|
||||
```javascript
|
||||
function findCompByEl(vnode, targetEl, depth = 0) {
|
||||
if (depth > 50 || !vnode) return null;
|
||||
const comp = vnode.component;
|
||||
if (comp) {
|
||||
if (comp.vnode?.el === targetEl || comp.subTree?.el === targetEl) return comp;
|
||||
if (comp.vnode?.el?.contains?.(targetEl)) {
|
||||
const result = findCompByEl(comp.subTree, targetEl, depth + 1);
|
||||
if (result) return result;
|
||||
return comp;
|
||||
}
|
||||
const subResult = findCompByEl(comp.subTree, targetEl, depth + 1);
|
||||
if (subResult) return subResult;
|
||||
}
|
||||
if (vnode.children && Array.isArray(vnode.children)) {
|
||||
for (const child of vnode.children) {
|
||||
const result = findCompByEl(child, targetEl, depth + 1);
|
||||
if (result) return result;
|
||||
}
|
||||
}
|
||||
if (vnode.dynamicChildren) {
|
||||
for (const child of vnode.dynamicChildren) {
|
||||
const result = findCompByEl(child, targetEl, depth + 1);
|
||||
if (result) return result;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
```
|
||||
|
||||
### 3. 调用组件方法
|
||||
```javascript
|
||||
// 目标DOM的parentElement通常是组件根元素
|
||||
const comp = findCompByEl(rootVnode, targetElement.parentElement);
|
||||
const ctx = comp.proxy;
|
||||
|
||||
// 查看可用方法
|
||||
Object.keys(ctx).filter(k => !k.startsWith('_') && !k.startsWith('$'));
|
||||
|
||||
// Select 类组件：直接调用 onSelect
|
||||
ctx.onSelect({id: 'USD', label: 'United States Dollar'});
|
||||
|
||||
// 获取选项列表
|
||||
ctx.computedOptions; // [{id, label, _selected}, ...]
|
||||
```
|
||||
|
||||
## 组件层级注意
|
||||
- **展示层**（如 OxdSelectText）：只有 onToggle/onFocus/onBlur，调用无实际效果
|
||||
- **逻辑层**（如 OxdSelectInput，是展示层的父组件）：有 openDropdown/onSelect/computedOptions/onCloseDropdown
|
||||
- 定位逻辑层：用 `targetElement.parentElement` 而非 targetElement 本身
|
||||
|
||||
### 弹窗内 Select 同样纯 JS 优先（已验证）
|
||||
- 弹窗（`.oxd-dialog-sheet`）内的 `.oxd-select-text` 用循环向上查找同样能命中 `OxdSelectInput`，`onSelect` 正常工作。
|
||||
- 不需要 CDP 兜底。仅当循环 8 层仍找不到组件时才考虑 CDP 打开+JS 点 option。
|
||||
|
||||
### 循环向上查找模式（推荐）
|
||||
单层 `parentElement` 可能不够，用循环更健壮：
|
||||
```javascript
|
||||
function findSelectComp(selectTextEl) {
|
||||
for (let el = selectTextEl, up = 0; el && up < 8; el = el.parentElement, up++) {
|
||||
const comp = findCompByEl(rootVnode, el);
|
||||
if (comp?.proxy?.onSelect && comp.proxy.computedOptions?.length) return comp;
|
||||
}
|
||||
return null; // 找不到再考虑CDP兜底
|
||||
}
|
||||
```
|
||||
|
||||
## 普通 Input/Textarea 操作（nativeSetter）
|
||||
|
||||
Vue 3 的 `v-model` 监听 input 事件，直接 `el.value = x` 不触发响应式。需用原型 setter：
|
||||
|
||||
```javascript
|
||||
// Input
|
||||
const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value').set;
|
||||
setter.call(inputEl, '新值');
|
||||
inputEl.dispatchEvent(new Event('input', {bubbles: true}));
|
||||
inputEl.dispatchEvent(new Event('change', {bubbles: true}));
|
||||
|
||||
// Textarea
|
||||
const taSetter = Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, 'value').set;
|
||||
taSetter.call(textareaEl, '内容');
|
||||
textareaEl.dispatchEvent(new Event('input', {bubbles: true}));
|
||||
```
|
||||
|
||||
### Date Input 特殊处理
|
||||
日期组件通常有 blur 校验，需要 focus→赋值→blur 完整链：
|
||||
```javascript
|
||||
dateInput.focus();
|
||||
setter.call(dateInput, '2026-08-05');
|
||||
dateInput.dispatchEvent(new Event('input', {bubbles: true}));
|
||||
dateInput.dispatchEvent(new Event('change', {bubbles: true}));
|
||||
dateInput.dispatchEvent(new Event('blur', {bubbles: true}));
|
||||
```
|
||||
|
||||
### Button
|
||||
普通 `.click()` 即可，Vue 3 不检查 button click 的 isTrusted。
|
||||
|
||||
### File Upload (input[type="file"])
|
||||
浏览器安全模型禁止JS直接 `input.value='path'`，但可用 DataTransfer API 构造 FileList：
|
||||
```javascript
|
||||
const fileInput = document.querySelector('input[type="file"]');
|
||||
const content = '文件内容';
|
||||
const file = new File([content], 'filename.txt', { type: 'text/plain', lastModified: Date.now() });
|
||||
const dt = new DataTransfer();
|
||||
dt.items.add(file);
|
||||
fileInput.files = dt.files; // Chrome 62+ 支持
|
||||
fileInput.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
fileInput.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
```
|
||||
- 适用于任何框架（非Vue3特有），纯浏览器API
|
||||
- 可构造任意类型文件（Blob/ArrayBuffer均可传入File构造器）
|
||||
- ⚠ CDP `DOM.setFileInputFiles` 只设files属性不触发事件（Chrome通用行为），DataTransfer+dispatch是唯一纯JS方案
|
||||
- ⚠ 确保弹窗/容器已打开再querySelector，否则input不在DOM中
|
||||
|
||||
## 泛化到其他 Vue3 站点（未逐一验证，思路层面）
|
||||
|
||||
本 SOP 的核心方法（根 vnode → findCompByEl → proxy）是 Vue3 通用的，但具体方法名/属性名因 UI 库而异。
|
||||
|
||||
面对陌生 Vue3 站点的探测思路：
|
||||
|
||||
1. **确认是 Vue3** — `document.getElementById('app')?.__vue_app__` 存在即可
|
||||
2. **定位目标 DOM** — 用选择器找到要操作的元素（如某个 select wrapper）
|
||||
3. **从 DOM 反查组件** — 用 findCompByEl 从目标元素及其父级向上找，拿到 component
|
||||
4. **探测组件能力** — 拿到 comp 后查看：
|
||||
- `Object.keys(comp.proxy.$options.methods || {})` → 组件方法名
|
||||
- `Object.keys(comp.props || {})` → props
|
||||
- `Object.keys(comp.setupState || {})` → setup 暴露的响应式数据和函数
|
||||
- 重点找类似 onSelect/handleSelect/select/setValue 的方法，以及 options/items/computedOptions 之类的选项列表
|
||||
5. **试调** — 找到疑似选中方法后，传入选项对象试调，观察 DOM 是否更新
|
||||
6. **选项格式** — 不同库的 option 结构不同（可能是 `{id, label}` 也可能是 `{value, text}` 或纯字符串），从选项列表数据中取一个完整对象传入即可
|
||||
|
||||
注意事项：
|
||||
- 有些库用 `emits` 而非 methods，选中逻辑可能在父组件而非子组件
|
||||
- 有些库 prod build 会 minify 方法名，此时 setupState 里的 key 可能是短名，需结合行为猜测
|
||||
- Composition API 组件的逻辑主要在 setupState 而非 $options.methods
|
||||
- 如果 proxy 上找不到方法，试试 `comp.exposed`（`<script setup>` 用 defineExpose 暴露的）
|
||||
|
||||
## Vue 富文本编辑器操作
|
||||
|
||||
### 核心原则
|
||||
1. **禁止只改 DOM** — `innerHTML` 不触发编辑器内部 model 更新，提交时数据丢失
|
||||
2. **优先找编辑器实例调原生 API** — 唯一稳路径：
|
||||
- Quill: `el.__quill.setText()` / `.clipboard.dangerouslyPasteHTML()`
|
||||
- Tiptap: `el.__tiptap.commands.setContent()` 或 Vue ref `.editor.commands.setContent()`
|
||||
- TinyMCE: `tinymce.get(id).setContent()` 或 `tinymce.activeEditor.setContent()`
|
||||
- WangEditor: `el.__wangEditor.setHtml()` 或 Vue ref `.editorRef.setHtml()`
|
||||
- CKEditor: `editor.setData()`
|
||||
3. **次选 `innerHTML + InputEvent`** — 对简单 Vue wrapper 有效（wrapper 监听 input 并 emit），复杂编辑器不保证
|
||||
4. **兜底 CDP `Input.insertText`** — 绕过 `isTrusted` 检查，等同物理输入
|
||||
5. **验证标准是"提交对了"不是"看到了"** — 拦截 fetch/XHR 看 payload，或读 `editor.getHTML()`
|
||||
|
||||
### 编辑器实例查找路径（按优先级）
|
||||
1. DOM 私有字段: `el.__quill`, `el.__tiptap`, `el.cmView`(CodeMirror)
|
||||
2. Vue 组件 setupState/exposed: `comp.setupState.editor`, `comp.exposed.editor`
|
||||
3. 全局变量: `window.editor`, `tinymce.editors[0]`
|
||||
4. Quill 静态方法: `Quill.find(el)`
|
||||
|
||||
### 编辑器类型识别
|
||||
- `.ql-editor` → Quill
|
||||
- `.ProseMirror` → Tiptap / ProseMirror
|
||||
- `.tox-edit-area` / `iframe` → TinyMCE
|
||||
- `.w-e-text-container` → WangEditor
|
||||
- `.ck-editor__editable` → CKEditor 5
|
||||
- `.cm-editor` → CodeMirror 6
|
||||
|
||||
### 避坑
|
||||
- Element Plus Select 选项被 Teleport 到 body，不在组件 DOM 子树内，要从 `document.querySelectorAll('.el-select-dropdown__item')` 全局找
|
||||
- 编辑器可能在 iframe 内（TinyMCE 默认），需 `iframe.contentDocument.body` 操作
|
||||
- 提交时数据来源可能不是 Vue state，而是编辑器实例现取 `getHTML()`，所以必须改编辑器 model
|
||||
- debounce：有些 wrapper 用 debounce 同步到 v-model，改完后等 300-500ms 再验证
|
||||
- Pinia/Vuex：表单数据可能在 store 里而非组件 data，需找到 store 直接赋值
|
||||
|
||||
## 适用场景
|
||||
- Vue 3 自定义 Select/Dropdown/Autocomplete 组件 → vnode 实例方法
|
||||
- Vue 3 普通 Input/Textarea（v-model）→ nativeSetter + input 事件
|
||||
- Date 组件 → nativeSetter + focus/blur 链
|
||||
- File Upload → DataTransfer + change 事件
|
||||
- 需要绕过 isTrusted 检查的场景
|
||||
- **Vue 3 富文本编辑器（Quill/Tiptap/TinyMCE/WangEditor/CKEditor）→ 编辑器实例 API**
|
||||
|
||||
## 验证于
|
||||
- OrangeHRM (opensource-demo.orangehrmlive.com) Vue 3 + OXD 组件库
|
||||
- 本地 Vue3 + Element Plus + 模拟 Quill/Tiptap 富文本靶场 (2026-05-09)
|
||||
- 2026-05-08
|
||||
@@ -0,0 +1,25 @@
|
||||
# Web 工具链初始化执行 SOP
|
||||
|
||||
若 web_scan 和 web_execute_js 已测试可用，无需执行此 SOP。
|
||||
仅供初始安装时，code_run 可用但 web 工具尚未配置的场景。
|
||||
|
||||
## 目标
|
||||
在仅具备系统级权限（code_run）时，建立 Web 交互能力（web_scan / web_execute_js）。
|
||||
|
||||
## 前置：检测浏览器
|
||||
|
||||
## 安装 tmwd_cdp_bridge 扩展
|
||||
扩展路径: `../assets/tmwd_cdp_bridge/`（MV3 Chrome 扩展，含 CDP debugger + scripting + cookie 能力）
|
||||
|
||||
### 自动打开扩展管理页
|
||||
`chrome://extensions` 无法通过命令行或 JS 打开，需用剪贴板+地址栏方案
|
||||
|
||||
### 安装步骤（chrome扩展页难以自动化）
|
||||
1. 打开扩展管理页，开启「开发者模式」
|
||||
2. 点击「加载已解压的扩展程序」，选择 `assets/tmwd_cdp_bridge/` 目录，或让用户直接拖入
|
||||
3. 显示“错误”不用管，一般只是因为还没连上GA
|
||||
|
||||
## 验证
|
||||
⚠ web_scan 显示「没有可用标签页」不一定是扩展没装好，可能是浏览器未打开或只有 blank 页。
|
||||
此时禁止乱试，先用 `start "" "https://www.baidu.com"` 打开一个正常页面，再 `web_scan` 确认。
|
||||
若仍不可用，无法自动探测默认浏览器是哪个、插件装在了哪个浏览器、或是否已安装——此时请求用户协助。
|
||||
Reference in New Issue
Block a user